@sienklogic/plan-build-run 2.18.1 → 2.19.1

package/plugins/pbr/scripts/lib/local-llm/shadow.js

@@ -0,0 +1,60 @@
+'use strict';
+
+const { logAgreement } = require('./metrics');
+
+/**
+ * Fire-and-forget shadow comparison.
+ * When shadow_mode is enabled, runs localResultFn() in the background and
+ * logs agreement/disagreement with frontierResult — but ALWAYS returns frontierResult.
+ *
+ * @param {object} config - resolved config from resolveConfig()
+ * @param {string} planningDir - path to the .planning directory
+ * @param {string} operationType - e.g. 'artifact_classification'
+ * @param {Function} localResultFn - async function that returns the local LLM result
+ * @param {*} frontierResult - the result already returned to the caller (never changed)
+ * @param {string} [sessionId] - current session identifier
+ * @returns {*} frontierResult — unchanged
+ */
+function runShadow(config, planningDir, operationType, localResultFn, frontierResult, sessionId) {
+  // Shadow off or LLM disabled — return immediately
+  if (!config.advanced || !config.advanced.shadow_mode) {
+    return frontierResult;
+  }
+  if (!config.enabled) {
+    return frontierResult;
+  }
+
+  // Fire-and-forget: never propagates errors, never affects frontierResult
+  Promise.resolve()
+    .then(async () => {
+      let localValue;
+      try {
+        const raw = await localResultFn();
+        localValue = typeof raw === 'string' ? raw : JSON.stringify(raw);
+      } catch (_) {
+        // Local call failed — log as disagreement
+        localValue = null;
+      }
+
+      const frontierStr =
+        typeof frontierResult === 'string' ? frontierResult : JSON.stringify(frontierResult);
+      const localStr = localValue != null ? localValue.trim() : null;
+      const agrees = localStr !== null && localStr === frontierStr.trim();
+
+      logAgreement(planningDir, {
+        timestamp: new Date().toISOString(),
+        operation: operationType,
+        session_id: sessionId || 'unknown',
+        agrees,
+        local_result: localStr,
+        frontier_result: frontierStr
+      });
+    })
+    .catch(() => {
+      // Swallow all errors — shadow must never throw
+    });
+
+  return frontierResult;
+}
+
+module.exports = { runShadow };

package/plugins/pbr/scripts/lib/local-llm/threshold-tuner.js

@@ -0,0 +1,118 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+// --- Constants ---
+
+/** Minimum shadow log entries per operation before suggesting an adjustment */
+const MIN_SAMPLES = 20;
+
+/** Step size for each threshold adjustment */
+const ADJUST_STEP = 0.05;
+
+/** Clamp floor for suggested threshold */
+const THRESHOLD_MIN = 0.5;
+
+/** Clamp ceiling for suggested threshold */
+const THRESHOLD_MAX = 0.99;
+
+/**
+ * Failure rate above which local LLM is considered too unreliable.
+ * Suggests raising the confidence_threshold so fewer calls are routed locally.
+ */
+const HIGH_FAILURE_RATE = 0.20;
+
+/**
+ * Failure rate below which local LLM is considered very reliable.
+ * Suggests lowering the confidence_threshold so more calls are routed locally.
+ */
+const LOW_FAILURE_RATE = 0.05;
+
+/**
+ * Reads the shadow agreement log and returns advisory threshold adjustments
+ * per operation type.
+ *
+ * Only emits a suggestion for an operation when it has >= MIN_SAMPLES entries.
+ * All suggestions are ±ADJUST_STEP clamped to [THRESHOLD_MIN, THRESHOLD_MAX].
+ * Never writes to config — purely advisory.
+ *
+ * @param {string} planningDir - Absolute path to the .planning directory
+ * @param {number} currentThreshold - Current confidence_threshold from config
+ * @returns {Array<{operation: string, current: number, suggested: number, sample_count: number, agreement_rate: number}>}
+ */
+function computeThresholdAdjustments(planningDir, currentThreshold) {
+  try {
+    const shadowLogPath = path.join(planningDir, 'logs', 'local-llm-shadow.jsonl');
+
+    if (!fs.existsSync(shadowLogPath)) {
+      return [];
+    }
+
+    const raw = fs.readFileSync(shadowLogPath, 'utf8');
+    const lines = raw.split('\n').filter(l => l.trim().length > 0);
+
+    // Parse lines, skip unparseable
+    const entries = [];
+    for (const line of lines) {
+      try {
+        const parsed = JSON.parse(line);
+        if (parsed && typeof parsed === 'object') {
+          entries.push(parsed);
+        }
+      } catch (_e) {
+        // Skip malformed lines
+      }
+    }
+
+    // Group by operation
+    const groups = {};
+    for (const entry of entries) {
+      const op = entry.operation;
+      if (!op) continue;
+      if (!groups[op]) {
+        groups[op] = { count: 0, agrees: 0 };
+      }
+      groups[op].count += 1;
+      if (entry.agrees === true) {
+        groups[op].agrees += 1;
+      }
+    }
+
+    // Build suggestions for operations with enough samples
+    const suggestions = [];
+    for (const [operation, stats] of Object.entries(groups)) {
+      if (stats.count < MIN_SAMPLES) continue;
+
+      const agreementRate = stats.agrees / stats.count;
+      const failureRate = 1 - agreementRate;
+
+      let suggested;
+      if (failureRate > HIGH_FAILURE_RATE) {
+        // Local is too unreliable — raise threshold (fewer local calls)
+        suggested = Math.min(THRESHOLD_MAX, currentThreshold + ADJUST_STEP);
+      } else if (failureRate < LOW_FAILURE_RATE) {
+        // Local is very reliable — lower threshold (more local calls)
+        suggested = Math.max(THRESHOLD_MIN, currentThreshold - ADJUST_STEP);
+      } else {
+        // Within acceptable range — no change
+        suggested = currentThreshold;
+      }
+
+      suggestions.push({
+        operation,
+        current: currentThreshold,
+        suggested,
+        sample_count: stats.count,
+        agreement_rate: agreementRate
+      });
+    }
+
+    return suggestions;
+  } catch (_e) {
+    // Never throws
+    return [];
+  }
+}
+
+module.exports = { computeThresholdAdjustments };

package/plugins/pbr/scripts/lib/migrate.js

@@ -0,0 +1,298 @@
+/**
+ * lib/migrate.cjs -- Schema migration for Plan-Build-Run config.json.
+ *
+ * Tracks config.json schema version and applies sequential migrations
+ * to bring outdated configs up to the current version.
+ *
+ * Usage:
+ *   const { applyMigrations, CURRENT_SCHEMA_VERSION } = require('./migrate');
+ *   const result = await applyMigrations(planningDir, { dryRun: false });
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { atomicWrite } = require('./core');
+
+/** The current schema version supported by this version of PBR. */
+const CURRENT_SCHEMA_VERSION = 3;
+
+/**
+ * Migration registry. Each entry describes one schema version step.
+ * Migrations MUST be listed in ascending `from` order.
+ *
+ * @type {Array<{ from: number, to: number, description: string, migrate: function }>}
+ */
+const MIGRATIONS = [
+  {
+    from: 0,
+    to: 1,
+    description: 'Add schema_version field',
+    migrate(config) {
+      config.schema_version = 1;
+    }
+  },
+  {
+    from: 1,
+    to: 2,
+    description: 'Normalize GSD config format to PBR structure',
+    migrate(config) {
+      // mode: "yolo" -> "autonomous"
+      if (config.mode === 'yolo') config.mode = 'autonomous';
+
+      // depth: old names -> new names
+      const depthMap = { lean: 'quick', balanced: 'standard', thorough: 'comprehensive' };
+      if (config.depth && depthMap[config.depth]) {
+        config.depth = depthMap[config.depth];
+      }
+
+      // parallelization: boolean -> object
+      if (typeof config.parallelization === 'boolean') {
+        config.parallelization = { enabled: config.parallelization };
+      }
+
+      // branching_strategy (flat) -> git.branching (nested)
+      if (config.branching_strategy !== undefined) {
+        if (!config.git) config.git = {};
+        config.git.branching = config.branching_strategy;
+        delete config.branching_strategy;
+      }
+
+      // Top-level feature booleans -> features object
+      const featureMap = {
+        research: 'research_phase',
+        plan_checker: 'plan_checking',
+        verifier: 'goal_verification',
+      };
+      if (!config.features) config.features = {};
+      for (const [oldKey, newKey] of Object.entries(featureMap)) {
+        if (config[oldKey] !== undefined) {
+          config.features[newKey] = config[oldKey];
+          delete config[oldKey];
+        }
+      }
+
+      // Top-level commit_docs -> planning.commit_docs
+      if (config.commit_docs !== undefined) {
+        if (!config.planning) config.planning = {};
+        config.planning.commit_docs = config.commit_docs;
+        delete config.commit_docs;
+      }
+
+      // Top-level search_gitignored -> planning.search_gitignored
+      if (config.search_gitignored !== undefined) {
+        if (!config.planning) config.planning = {};
+        config.planning.search_gitignored = config.search_gitignored;
+        delete config.search_gitignored;
+      }
+
+      // Top-level model_profile (string) -> delete (not a PBR field, depth replaces it)
+      if (config.model_profile !== undefined) {
+        // Map old model_profile to depth if depth not set
+        if (!config.depth) {
+          const profileToDepth = { quality: 'comprehensive', balanced: 'standard', budget: 'quick' };
+          config.depth = profileToDepth[config.model_profile] || 'standard';
+        }
+        delete config.model_profile;
+      }
+
+      // Clean up GSD-only fields
+      const gsdOnlyFields = ['brave_search', 'phase_branch_template', 'milestone_branch_template'];
+      for (const f of gsdOnlyFields) {
+        if (config[f] !== undefined) delete config[f];
+      }
+
+      // workflow object: keep if present, it maps well to PBR
+      // (workflow.research, workflow.plan_check, etc. are valid in both)
+
+      // Ensure version is set
+      if (!config.version) config.version = 2;
+
+      config.schema_version = 2;
+    }
+  },
+  {
+    from: 2,
+    to: 3,
+    description: 'Add 1M context adaptation config sections',
+    migrate(config) {
+      // Add new workflow properties with defaults (do NOT overwrite existing)
+      if (!config.workflow) config.workflow = {};
+      const wfDefaults = {
+        inline_execution: false,
+        inline_max_tasks: 2,
+        inline_context_cap_pct: 40,
+        phase_boundary_clear: 'off',
+        autonomous: false,
+        speculative_planning: false,
+        phase_replay: false
+      };
+      for (const [key, val] of Object.entries(wfDefaults)) {
+        if (config.workflow[key] === undefined) config.workflow[key] = val;
+      }
+
+      // Add planning.multi_phase
+      if (!config.planning) config.planning = {};
+      if (config.planning.multi_phase === undefined) config.planning.multi_phase = false;
+
+      // Add gates.checkpoint_auto_resolve
+      if (!config.gates) config.gates = {};
+      if (config.gates.checkpoint_auto_resolve === undefined) {
+        config.gates.checkpoint_auto_resolve = 'none';
+      }
+
+      // Add new top-level sections with defaults
+      if (!config.intel) {
+        config.intel = { enabled: false, auto_update: false, inject_on_start: false };
+      }
+      if (!config.context_ledger) {
+        config.context_ledger = { enabled: false, stale_after_minutes: 60 };
+      }
+      if (!config.learnings) {
+        config.learnings = { enabled: false, read_depth: 3 };
+      }
+      if (!config.verification) {
+        config.verification = { confidence_gate: false, confidence_threshold: 1.0 };
+      }
+      if (!config.context_budget) {
+        config.context_budget = { threshold_curve: 'linear' };
+      }
+
+      config.schema_version = 3;
+    }
+  }
+];
+
+/**
+ * Detect the current schema version from a config object.
+ * Returns 0 if schema_version is absent or non-numeric.
+ *
+ * @param {object} config - Parsed config.json object
+ * @returns {number} Detected schema version
+ */
+function detectSchemaVersion(config) {
+  const v = config && config.schema_version;
+  if (typeof v === 'number' && Number.isFinite(v)) return v;
+  return 0;
+}
+
+/**
+ * Return the ordered list of migrations needed to go from `fromVersion` to `toVersion`.
+ * Returns an empty array if versions are equal.
+ * Throws if fromVersion > toVersion (downgrade not supported).
+ *
+ * @param {number} fromVersion - Current schema version
+ * @param {number} toVersion - Target schema version
+ * @returns {Array} Ordered migrations to apply
+ */
+function getMigrationPath(fromVersion, toVersion) {
+  if (fromVersion > toVersion) {
+    throw new Error(`Cannot downgrade schema from version ${fromVersion} to ${toVersion}`);
+  }
+  if (fromVersion === toVersion) return [];
+  return MIGRATIONS.filter(m => m.from >= fromVersion && m.to <= toVersion);
+}
+
+/**
+ * Apply pending migrations to config.json in planningDir.
+ *
+ * Options:
+ *   dryRun {boolean} -- If true, simulate migration without writing files (default: false)
+ *   force  {boolean} -- Reserved for future use (default: false)
+ *
+ * Returns:
+ *   { migrated: false, version: N }                          -- already current
+ *   { migrated: false, message: string }                     -- future version, no-op
+ *   { migrated: true, fromVersion, toVersion, applied, backupPath } -- success
+ *   { error: string }                                        -- failure
+ *
+ * @param {string} planningDir - Path to .planning directory
+ * @param {object} [options] - Options { dryRun, force }
+ * @returns {Promise<object>} Result object
+ */
+async function applyMigrations(planningDir, options) {
+  const opts = options || {};
+  const dryRun = opts.dryRun === true;
+
+  const configPath = path.join(planningDir, 'config.json');
+
+  // Load config.json
+  if (!fs.existsSync(configPath)) {
+    return { error: 'config.json not found in ' + planningDir };
+  }
+
+  let config;
+  try {
+    config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
+  } catch (e) {
+    return { error: 'config.json is not valid JSON: ' + e.message };
+  }
+
+  const currentVersion = detectSchemaVersion(config);
+
+  // Future version -- don't touch it
+  if (currentVersion > CURRENT_SCHEMA_VERSION) {
+    return {
+      migrated: false,
+      version: currentVersion,
+      message: `config.json schema_version (${currentVersion}) is newer than this PBR version supports (${CURRENT_SCHEMA_VERSION}). No migration applied.`
+    };
+  }
+
+  // Already current
+  if (currentVersion === CURRENT_SCHEMA_VERSION) {
+    return { migrated: false, version: currentVersion };
+  }
+
+  // Determine migrations to apply
+  const migrations = getMigrationPath(currentVersion, CURRENT_SCHEMA_VERSION);
+  if (migrations.length === 0) {
+    return { migrated: false, version: currentVersion };
+  }
+
+  // Clone config for mutation
+  const updatedConfig = JSON.parse(JSON.stringify(config));
+
+  // Apply each migration in sequence
+  const applied = [];
+  for (const m of migrations) {
+    m.migrate(updatedConfig);
+    applied.push(m.description);
+  }
+
+  if (dryRun) {
+    return {
+      migrated: true,
+      fromVersion: currentVersion,
+      toVersion: CURRENT_SCHEMA_VERSION,
+      applied,
+      dryRun: true
+    };
+  }
+
+  // Create backup
+  const backupDir = path.join(planningDir, '.migration-backup');
+  const backupPath = path.join(backupDir, 'config.json.bak');
+  if (!fs.existsSync(backupDir)) {
+    fs.mkdirSync(backupDir, { recursive: true });
+  }
+  fs.copyFileSync(configPath, backupPath);
+
+  // Write updated config atomically
+  atomicWrite(configPath, JSON.stringify(updatedConfig, null, 2));
+
+  return {
+    migrated: true,
+    fromVersion: currentVersion,
+    toVersion: CURRENT_SCHEMA_VERSION,
+    applied,
+    backupPath
+  };
+}
+
+module.exports = {
+  CURRENT_SCHEMA_VERSION,
+  MIGRATIONS,
+  detectSchemaVersion,
+  getMigrationPath,
+  applyMigrations
+};