@sienklogic/plan-build-run 2.1.0 → 2.1.1

package/plugins/pbr/scripts/local-llm/health.js

@@ -0,0 +1,220 @@
+/* global fetch, AbortSignal */
+'use strict';
+
+const WARMUP_TIMEOUT_MS = 120000;
+
+/**
+ * Merges the raw local_llm config block with defaults.
+ * @param {object|undefined} rawConfig
+ * @returns {object} Fully-defaulted local_llm config
+ */
+function resolveConfig(rawConfig) {
+  return {
+    enabled: rawConfig != null && rawConfig.enabled != null ? rawConfig.enabled : false,
+    provider: (rawConfig && rawConfig.provider) || 'ollama',
+    endpoint: (rawConfig && rawConfig.endpoint) || 'http://localhost:11434',
+    model: (rawConfig && rawConfig.model) || 'qwen2.5-coder:7b',
+    timeout_ms: (rawConfig && rawConfig.timeout_ms) || 3000,
+    max_retries: rawConfig != null && rawConfig.max_retries != null ? rawConfig.max_retries : 1,
+    fallback: (rawConfig && rawConfig.fallback) || 'frontier',
+    routing_strategy: (rawConfig && rawConfig.routing_strategy) || 'local_first',
+    features: Object.assign(
+      {
+        artifact_classification: true,
+        task_validation: true,
+        plan_adequacy: false,
+        gap_detection: false,
+        context_summarization: false,
+        source_scoring: false,
+        commit_classification: true,
+        test_triage: true,
+        file_intent_classification: true
+      },
+      (rawConfig && rawConfig.features) || {}
+    ),
+    metrics: Object.assign(
+      {
+        enabled: true,
+        log_file: '.planning/logs/local-llm-metrics.jsonl',
+        show_session_summary: true,
+        frontier_token_rate: 3.0
+      },
+      (rawConfig && rawConfig.metrics) || {}
+    ),
+    advanced: Object.assign(
+      {
+        confidence_threshold: 0.9,
+        max_input_tokens: 2000,
+        keep_alive: '30m',
+        num_ctx: 4096,
+        disable_after_failures: 3,
+        shadow_mode: false
+      },
+      (rawConfig && rawConfig.advanced) || {}
+    )
+  };
+}
+
+/**
+ * Checks availability of the configured Ollama instance and model.
+ * Always resolves — never rejects.
+ * @param {object} config - resolved config from resolveConfig()
+ * @returns {Promise<object>} Structured health status
+ */
+async function checkHealth(config) {
+  try {
+    if (!config.enabled) {
+      return { available: false, reason: 'disabled', model: null, version: null };
+    }
+
+    const timeoutShort = 3000;
+    const timeoutModel = 5000;
+
+    // Step 1 — Check server reachable
+    try {
+      const res = await fetch(config.endpoint + '/', {
+        signal: AbortSignal.timeout(timeoutShort)
+      });
+      const body = await res.text().catch(() => '');
+      if (!body.includes('Ollama')) {
+        return {
+          available: false,
+          reason: 'not_running',
+          detail: 'Ollama is not running. Start with: ollama serve',
+          model: null,
+          version: null
+        };
+      }
+    } catch (err) {
+      const isConnRefused =
+        (err.cause && err.cause.code === 'ECONNREFUSED') ||
+        (err.message && err.message.includes('ECONNREFUSED'));
+      const isTimeout = err.name === 'TimeoutError' || err.name === 'AbortError';
+      if (isConnRefused || isTimeout) {
+        return {
+          available: false,
+          reason: 'not_running',
+          detail: 'Ollama is not running. Start with: ollama serve',
+          model: null,
+          version: null
+        };
+      }
+      throw err;
+    }
+
+    // Step 2 — Check version (non-fatal)
+    let version = null;
+    try {
+      const res = await fetch(config.endpoint + '/api/version', {
+        signal: AbortSignal.timeout(timeoutShort)
+      });
+      const data = await res.json();
+      version = data.version || null;
+    } catch (_) {
+      version = null;
+    }
+
+    // Step 3 — Check model available
+    try {
+      const res = await fetch(config.endpoint + '/v1/models', {
+        signal: AbortSignal.timeout(timeoutModel)
+      });
+      const data = await res.json();
+      const modelList = (data.data || []).map((m) => m.id || '');
+      const baseModel = config.model.split(':')[0];
+      const found = modelList.some((m) => m.startsWith(baseModel));
+      if (!found) {
+        return {
+          available: false,
+          reason: 'model_missing',
+          detail: 'Run: ollama pull ' + config.model,
+          model: null,
+          version
+        };
+      }
+    } catch (_err) {
+      return {
+        available: false,
+        reason: 'model_missing',
+        detail: 'Run: ollama pull ' + config.model,
+        model: null,
+        version
+      };
+    }
+
+    // Step 4 — Detect GPU error (sleep/wake CUDA bug)
+    let warm = false;
+    try {
+      const res = await fetch(config.endpoint + '/v1/chat/completions', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          model: config.model,
+          messages: [{ role: 'user', content: '{"status":"ok"}' }],
+          max_tokens: 10,
+          num_ctx: 512
+        }),
+        signal: AbortSignal.timeout(timeoutModel)
+      });
+
+      if (!res.ok) {
+        const errBody = await res.text().catch(() => '');
+        if (res.status === 500 && (errBody.includes('GPU') || errBody.includes('CUDA'))) {
+          return {
+            available: false,
+            reason: 'gpu_error',
+            detail: 'GPU error detected. Restart Ollama: ollama serve',
+            model: config.model,
+            version
+          };
+        }
+        // Non-GPU HTTP error — treat as available but cold
+        warm = false;
+      } else {
+        warm = true;
+      }
+    } catch (err) {
+      const isTimeout = err.name === 'TimeoutError' || err.name === 'AbortError';
+      if (isTimeout) {
+        warm = false; // cold start in progress — skip, don't block
+      } else {
+        warm = false;
+      }
+    }
+
+    return { available: true, warm, reason: 'ok', model: config.model, version };
+  } catch (_err) {
+    return {
+      available: false,
+      reason: 'unknown_error',
+      detail: _err.message,
+      model: null,
+      version: null
+    };
+  }
+}
+
+/**
+ * Fire-and-forget warm-up request. Callers should NOT await this.
+ * @param {object} config - resolved config from resolveConfig()
+ */
+async function warmUp(config) {
+  try {
+    await fetch(config.endpoint + '/v1/chat/completions', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        model: config.model,
+        messages: [{ role: 'user', content: '{"status":"ready"}' }],
+        max_tokens: 10,
+        num_ctx: 512,
+        keep_alive: config.advanced.keep_alive
+      }),
+      signal: AbortSignal.timeout(WARMUP_TIMEOUT_MS)
+    });
+  } catch (_) {
+    // Swallow all errors silently — fire and forget
+  }
+}
+
+module.exports = { resolveConfig, checkHealth, warmUp };

package/plugins/pbr/scripts/local-llm/metrics.js

@@ -0,0 +1,340 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const MAX_ENTRIES = 200;
+
+/**
+ * Appends a metric entry to the JSONL log file.
+ * Rotates to keep only the last 200 entries when the log exceeds MAX_ENTRIES lines.
+ * Swallows all errors silently — metrics must never crash hooks.
+ *
+ * @param {string} planningDir - path to the .planning directory
+ * @param {object} entry - metric entry object
+ * @param {string} entry.session_id
+ * @param {string} entry.timestamp
+ * @param {string} entry.operation
+ * @param {string} entry.model
+ * @param {number} entry.latency_ms
+ * @param {number} entry.tokens_used_local
+ * @param {number} entry.tokens_saved_frontier
+ * @param {string} entry.result
+ * @param {boolean} entry.fallback_used
+ * @param {number} entry.confidence
+ */
+function logMetric(planningDir, entry) {
+  try {
+    const logsDir = path.join(planningDir, 'logs');
+    const logFile = path.join(logsDir, 'local-llm-metrics.jsonl');
+
+    fs.mkdirSync(logsDir, { recursive: true });
+
+    // Update lifetime totals BEFORE appending to JSONL so seeding
+    // (which reads the JSONL) doesn't double-count this entry
+    try {
+      updateLifetimeTotals(logsDir, entry);
+    } catch (_) {
+      // Totals update failure is non-fatal
+    }
+
+    fs.appendFileSync(logFile, JSON.stringify(entry) + '\n', 'utf8');
+
+    // Rotate if over MAX_ENTRIES
+    try {
+      const contents = fs.readFileSync(logFile, 'utf8');
+      const lines = contents.split(/\r?\n/).filter((l) => l.trim() !== '');
+      if (lines.length > MAX_ENTRIES) {
+        const trimmed = lines.slice(lines.length - MAX_ENTRIES);
+        fs.writeFileSync(logFile, trimmed.join('\n') + '\n', 'utf8');
+      }
+    } catch (_) {
+      // Rotation failure is non-fatal
+    }
+  } catch (_) {
+    // Swallow all errors silently
+  }
+}
+
+/**
+ * Atomically increments the lifetime-totals.json running counters.
+ * This file persists across JSONL log rotations so lifetime metrics never plateau.
+ *
+ * @param {string} logsDir - path to the .planning/logs directory
+ * @param {object} entry - the metric entry being logged
+ */
+function updateLifetimeTotals(logsDir, entry) {
+  const totalsFile = path.join(logsDir, 'lifetime-totals.json');
+  let totals = null;
+
+  try {
+    totals = JSON.parse(fs.readFileSync(totalsFile, 'utf8'));
+  } catch (_) {
+    // File doesn't exist yet or is corrupt — seed from existing JSONL
+    totals = seedTotalsFromJsonl(logsDir);
+  }
+
+  totals.total_calls = (totals.total_calls || 0) + 1;
+  totals.fallback_count = (totals.fallback_count || 0) + (entry.fallback_used ? 1 : 0);
+  totals.tokens_saved = (totals.tokens_saved || 0) + (entry.tokens_saved_frontier || 0);
+  totals.total_latency_ms = (totals.total_latency_ms || 0) + (entry.latency_ms || 0);
+
+  fs.writeFileSync(totalsFile, JSON.stringify(totals) + '\n', 'utf8');
+}
+
+/**
+ * Seeds lifetime totals by scanning the existing JSONL log.
+ * Called once when lifetime-totals.json doesn't exist yet (migration from pre-totals installs).
+ * Returns the accumulated totals from whatever entries remain in the JSONL.
+ *
+ * @param {string} logsDir - path to the .planning/logs directory
+ * @returns {{ total_calls: number, fallback_count: number, tokens_saved: number, total_latency_ms: number }}
+ */
+function seedTotalsFromJsonl(logsDir) {
+  const seed = { total_calls: 0, fallback_count: 0, tokens_saved: 0, total_latency_ms: 0 };
+  try {
+    const logFile = path.join(logsDir, 'local-llm-metrics.jsonl');
+    const contents = fs.readFileSync(logFile, 'utf8');
+    const lines = contents.split(/\r?\n/).filter((l) => l.trim() !== '');
+    for (const line of lines) {
+      try {
+        const e = JSON.parse(line);
+        seed.total_calls += 1;
+        seed.fallback_count += e.fallback_used ? 1 : 0;
+        seed.tokens_saved += e.tokens_saved_frontier || 0;
+        seed.total_latency_ms += e.latency_ms || 0;
+      } catch (_) {
+        // Skip malformed lines
+      }
+    }
+  } catch (_) {
+    // No JSONL file — start at zero
+  }
+  return seed;
+}
+
+/**
+ * Reads metric entries from the JSONL log that occurred at or after sessionStartTime.
+ *
+ * @param {string} planningDir - path to the .planning directory
+ * @param {string|Date} sessionStartTime - ISO string or Date
+ * @returns {object[]} Array of matching metric entry objects
+ */
+function readSessionMetrics(planningDir, sessionStartTime) {
+  try {
+    const logFile = path.join(planningDir, 'logs', 'local-llm-metrics.jsonl');
+    const contents = fs.readFileSync(logFile, 'utf8');
+    const startMs = new Date(sessionStartTime).getTime();
+
+    return contents
+      .split(/\r?\n/)
+      .filter((l) => l.trim() !== '')
+      .map((l) => {
+        try {
+          return JSON.parse(l);
+        } catch (_) {
+          return null;
+        }
+      })
+      .filter((e) => e !== null)
+      .filter((e) => {
+        try {
+          return new Date(e.timestamp).getTime() >= startMs;
+        } catch (_) {
+          return false;
+        }
+      });
+  } catch (_) {
+    return [];
+  }
+}
+
+/**
+ * Summarizes an array of metric entries.
+ *
+ * @param {object[]} entries
+ * @param {number} [frontierTokenRate=3.0] - cost per million tokens in USD
+ * @returns {{ total_calls: number, fallback_count: number, avg_latency_ms: number, tokens_saved: number, cost_saved_usd: number }}
+ */
+function summarizeMetrics(entries, frontierTokenRate) {
+  if (!entries || entries.length === 0) {
+    return {
+      total_calls: 0,
+      fallback_count: 0,
+      avg_latency_ms: 0,
+      tokens_saved: 0,
+      cost_saved_usd: 0
+    };
+  }
+
+  const rate = frontierTokenRate != null ? frontierTokenRate : 3.0;
+  const total_calls = entries.length;
+  const fallback_count = entries.filter((e) => e.fallback_used).length;
+  const totalLatency = entries.reduce((sum, e) => sum + (e.latency_ms || 0), 0);
+  const avg_latency_ms = total_calls > 0 ? totalLatency / total_calls : 0;
+  const tokens_saved = entries.reduce((sum, e) => sum + (e.tokens_saved_frontier || 0), 0);
+  const cost_saved_usd = tokens_saved * (rate / 1_000_000);
+
+  return { total_calls, fallback_count, avg_latency_ms, tokens_saved, cost_saved_usd };
+}
+
+/**
+ * Computes lifetime aggregate metrics by reading all entries from the JSONL log.
+ * No date filter — reads everything. Adds a by_operation breakdown keyed by operation.
+ *
+ * @param {string} planningDir - path to the .planning directory
+ * @param {number} [frontierTokenRate=3.0] - cost per million tokens in USD
+ * @returns {{ total_calls: number, fallback_count: number, avg_latency_ms: number, tokens_saved: number, cost_saved_usd: number, by_operation: object }}
+ */
+function computeLifetimeMetrics(planningDir, frontierTokenRate) {
+  const zero = {
+    total_calls: 0,
+    fallback_count: 0,
+    avg_latency_ms: 0,
+    tokens_saved: 0,
+    cost_saved_usd: 0,
+    by_operation: {}
+  };
+
+  try {
+    const rate = frontierTokenRate != null ? frontierTokenRate : 3.0;
+    const logsDir = path.join(planningDir, 'logs');
+    const totalsFile = path.join(logsDir, 'lifetime-totals.json');
+
+    // Primary path: read from lifetime-totals.json (survives log rotation)
+    let totals = null;
+    try {
+      totals = JSON.parse(fs.readFileSync(totalsFile, 'utf8'));
+    } catch (_) {
+      // No totals file — fall back to JSONL scan (migration path for existing installs)
+    }
+
+    // Build by_operation from the JSONL (only covers recent entries, but still useful)
+    const by_operation = {};
+    const logFile = path.join(logsDir, 'local-llm-metrics.jsonl');
+    try {
+      const contents = fs.readFileSync(logFile, 'utf8');
+      const entries = contents
+        .split(/\r?\n/)
+        .filter((l) => l.trim() !== '')
+        .map((l) => {
+          try {
+            return JSON.parse(l);
+          } catch (_) {
+            return null;
+          }
+        })
+        .filter((e) => e !== null);
+
+      for (const e of entries) {
+        const op = e.operation || 'unknown';
+        if (!by_operation[op]) {
+          by_operation[op] = { calls: 0, fallbacks: 0, tokens_saved: 0 };
+        }
+        by_operation[op].calls += 1;
+        if (e.fallback_used) by_operation[op].fallbacks += 1;
+        by_operation[op].tokens_saved += e.tokens_saved_frontier || 0;
+      }
+
+      // If no totals file, compute from JSONL (legacy/migration path)
+      if (!totals) {
+        if (entries.length === 0) return zero;
+        const total_calls = entries.length;
+        const fallback_count = entries.filter((e) => e.fallback_used).length;
+        const totalLatency = entries.reduce((sum, e) => sum + (e.latency_ms || 0), 0);
+        const avg_latency_ms = total_calls > 0 ? totalLatency / total_calls : 0;
+        const tokens_saved = entries.reduce((sum, e) => sum + (e.tokens_saved_frontier || 0), 0);
+        const cost_saved_usd = tokens_saved * (rate / 1_000_000);
+        return { total_calls, fallback_count, avg_latency_ms, tokens_saved, cost_saved_usd, by_operation };
+      }
+    } catch (_) {
+      // No JSONL file — if we have totals, continue; otherwise return zero
+      if (!totals) return zero;
+    }
+
+    // Use lifetime totals for the headline numbers
+    const total_calls = totals.total_calls || 0;
+    const fallback_count = totals.fallback_count || 0;
+    const tokens_saved = totals.tokens_saved || 0;
+    const total_latency_ms = totals.total_latency_ms || 0;
+    const avg_latency_ms = total_calls > 0 ? total_latency_ms / total_calls : 0;
+    const cost_saved_usd = tokens_saved * (rate / 1_000_000);
+
+    return { total_calls, fallback_count, avg_latency_ms, tokens_saved, cost_saved_usd, by_operation };
+  } catch (_) {
+    return zero;
+  }
+}
+
+/**
+ * Formats a metrics aggregate (output of summarizeMetrics) into a human-readable one-liner.
+ *
+ * @param {object} summary - output of summarizeMetrics()
+ * @param {string} [model] - optional model name
+ * @returns {string}
+ */
+function formatSessionSummary(summary, model) {
+  if (!summary || summary.total_calls === 0) {
+    return 'Local LLM: no calls this session';
+  }
+
+  const { total_calls, fallback_count, avg_latency_ms, tokens_saved, cost_saved_usd } = summary;
+
+  let costStr = '';
+  if (cost_saved_usd > 0) {
+    costStr = ` ($${cost_saved_usd.toFixed(2)})`;
+  }
+
+  let fallbackStr = '';
+  if (fallback_count > 0) {
+    fallbackStr = `, ${fallback_count} fallback(s)`;
+  }
+
+  let modelStr = '';
+  if (model) {
+    modelStr = ` [${model}]`;
+  }
+
+  const avgMs = Math.round(avg_latency_ms);
+
+  return `Local LLM: ${total_calls} calls, ~${tokens_saved} frontier tokens saved${costStr}, avg ${avgMs}ms${fallbackStr}${modelStr}`;
+}
+
+/**
+ * Appends a shadow comparison entry to the shadow JSONL log file.
+ * Rotates to keep only the last 200 entries. Swallows all errors silently.
+ *
+ * @param {string} planningDir - path to the .planning directory
+ * @param {object} entry - shadow comparison entry object
+ * @param {string} entry.timestamp
+ * @param {string} entry.operation
+ * @param {string} entry.session_id
+ * @param {boolean} entry.agrees
+ * @param {string|null} entry.local_result
+ * @param {string} entry.frontier_result
+ */
+function logAgreement(planningDir, entry) {
+  try {
+    const logsDir = path.join(planningDir, 'logs');
+    const logFile = path.join(logsDir, 'local-llm-shadow.jsonl');
+
+    fs.mkdirSync(logsDir, { recursive: true });
+    fs.appendFileSync(logFile, JSON.stringify(entry) + '\n', 'utf8');
+
+    // Rotate if over MAX_ENTRIES
+    try {
+      const contents = fs.readFileSync(logFile, 'utf8');
+      const lines = contents.split(/\r?\n/).filter((l) => l.trim() !== '');
+      if (lines.length > MAX_ENTRIES) {
+        const trimmed = lines.slice(lines.length - MAX_ENTRIES);
+        fs.writeFileSync(logFile, trimmed.join('\n') + '\n', 'utf8');
+      }
+    } catch (_) {
+      // Rotation failure is non-fatal
+    }
+  } catch (_) {
+    // Swallow all errors silently
+  }
+}
+
+module.exports = { logMetric, readSessionMetrics, summarizeMetrics, computeLifetimeMetrics, formatSessionSummary, logAgreement, updateLifetimeTotals, seedTotalsFromJsonl };

package/plugins/pbr/scripts/local-llm/operations/classify-artifact.js

@@ -0,0 +1,76 @@
+'use strict';
+
+const { complete, tryParseJSON, isDisabled } = require('../client');
+const { logMetric } = require('../metrics');
+const { route } = require('../router');
+
+/**
+ * Classifies a PLAN.md or SUMMARY.md artifact using the local LLM.
+ *
+ * @param {object} config - resolved local_llm config block
+ * @param {string} planningDir - path to the .planning directory
+ * @param {string} content - file content to classify
+ * @param {string} fileType - 'PLAN' or 'SUMMARY'
+ * @param {string} [sessionId] - optional session identifier for metrics
+ * @returns {Promise<{ classification: string, confidence: number, reason: string, latency_ms: number, fallback_used: boolean }|null>}
+ */
+async function classifyArtifact(config, planningDir, content, fileType, sessionId) {
+  if (!config.enabled || !config.features.artifact_classification) return null;
+  if (isDisabled('artifact-classification', config.advanced.disable_after_failures)) return null;
+
+  const maxChars = (config.advanced.max_input_tokens || 1024) * 4;
+  const truncatedContent = content.length > maxChars ? content.slice(0, maxChars) : content;
+
+  let prompt;
+  if (fileType === 'PLAN') {
+    prompt =
+      'Classify this PLAN.md as stub, partial, or complete. A stub has placeholder tasks or missing required XML elements. A partial has some tasks filled but action/verify/done are vague. A complete has all tasks with specific steps, executable verify commands, and observable done conditions. Respond with JSON: {"classification": "stub"|"partial"|"complete", "confidence": 0.0-1.0, "reason": "one sentence"}\n\nContent:\n' +
+      truncatedContent;
+  } else if (fileType === 'SUMMARY') {
+    prompt =
+      'Classify this SUMMARY.md as substantive or thin. Substantive means it has specific artifact paths, commit hashes, and observable outcomes. Thin means vague or placeholder content. Respond with JSON: {"classification": "substantive"|"thin", "confidence": 0.0-1.0, "reason": "one sentence"}\n\nContent:\n' +
+      truncatedContent;
+  } else {
+    return null;
+  }
+
+  try {
+    const result = await route(config, prompt, 'artifact-classification', (logprobs) =>
+      complete(config, prompt, 'artifact-classification', { logprobs })
+    );
+    if (result === null) return null;
+    const parsed = tryParseJSON(result.content);
+    if (!parsed.ok) return null;
+
+    const validPlanClassifications = ['stub', 'partial', 'complete'];
+    const validSummaryClassifications = ['substantive', 'thin'];
+    const validValues = fileType === 'PLAN' ? validPlanClassifications : validSummaryClassifications;
+    if (!parsed.data.classification || !validValues.includes(parsed.data.classification)) return null;
+
+    const metricEntry = {
+      session_id: sessionId || 'unknown',
+      timestamp: new Date().toISOString(),
+      operation: 'artifact-classification',
+      model: config.model,
+      latency_ms: result.latency_ms,
+      tokens_used_local: result.tokens,
+      tokens_saved_frontier: 420,
+      result: parsed.data.classification,
+      fallback_used: false,
+      confidence: parsed.data.confidence || 0.9
+    };
+    logMetric(planningDir, metricEntry);
+
+    return {
+      classification: parsed.data.classification,
+      confidence: parsed.data.confidence || 0.9,
+      reason: parsed.data.reason || '',
+      latency_ms: result.latency_ms,
+      fallback_used: false
+    };
+  } catch (_) {
+    return null;
+  }
+}
+
+module.exports = { classifyArtifact };