@sienklogic/plan-build-run 2.22.2 → 2.23.0

package/plugins/pbr/scripts/local-llm/health.js

@@ -0,0 +1,217 @@
+/* global fetch, AbortSignal */
+'use strict';
+
+const WARMUP_TIMEOUT_MS = 120000;
+
+/**
+ * Merges the raw local_llm config block with defaults.
+ * @param {object|undefined} rawConfig
+ * @returns {object} Fully-defaulted local_llm config
+ */
+function resolveConfig(rawConfig) {
+  return {
+    enabled: rawConfig != null && rawConfig.enabled != null ? rawConfig.enabled : false,
+    provider: (rawConfig && rawConfig.provider) || 'ollama',
+    endpoint: (rawConfig && rawConfig.endpoint) || 'http://localhost:11434',
+    model: (rawConfig && rawConfig.model) || 'qwen2.5-coder:7b',
+    timeout_ms: (rawConfig && rawConfig.timeout_ms) || 3000,
+    max_retries: rawConfig != null && rawConfig.max_retries != null ? rawConfig.max_retries : 1,
+    fallback: (rawConfig && rawConfig.fallback) || 'frontier',
+    routing_strategy: (rawConfig && rawConfig.routing_strategy) || 'local_first',
+    features: Object.assign(
+      {
+        artifact_classification: true,
+        task_validation: true,
+        plan_adequacy: false,
+        gap_detection: false,
+        context_summarization: false,
+        source_scoring: false
+      },
+      (rawConfig && rawConfig.features) || {}
+    ),
+    metrics: Object.assign(
+      {
+        enabled: true,
+        log_file: '.planning/logs/local-llm-metrics.jsonl',
+        show_session_summary: true,
+        frontier_token_rate: 3.0
+      },
+      (rawConfig && rawConfig.metrics) || {}
+    ),
+    advanced: Object.assign(
+      {
+        confidence_threshold: 0.9,
+        max_input_tokens: 2000,
+        keep_alive: '30m',
+        num_ctx: 4096,
+        disable_after_failures: 3,
+        shadow_mode: false
+      },
+      (rawConfig && rawConfig.advanced) || {}
+    )
+  };
+}
+
+/**
+ * Checks availability of the configured Ollama instance and model.
+ * Always resolves — never rejects.
+ * @param {object} config - resolved config from resolveConfig()
+ * @returns {Promise<object>} Structured health status
+ */
+async function checkHealth(config) {
+  try {
+    if (!config.enabled) {
+      return { available: false, reason: 'disabled', model: null, version: null };
+    }
+
+    const timeoutShort = 3000;
+    const timeoutModel = 5000;
+
+    // Step 1 — Check server reachable
+    try {
+      const res = await fetch(config.endpoint + '/', {
+        signal: AbortSignal.timeout(timeoutShort)
+      });
+      const body = await res.text().catch(() => '');
+      if (!body.includes('Ollama')) {
+        return {
+          available: false,
+          reason: 'not_running',
+          detail: 'Ollama is not running. Start with: ollama serve',
+          model: null,
+          version: null
+        };
+      }
+    } catch (err) {
+      const isConnRefused =
+        (err.cause && err.cause.code === 'ECONNREFUSED') ||
+        (err.message && err.message.includes('ECONNREFUSED'));
+      const isTimeout = err.name === 'TimeoutError' || err.name === 'AbortError';
+      if (isConnRefused || isTimeout) {
+        return {
+          available: false,
+          reason: 'not_running',
+          detail: 'Ollama is not running. Start with: ollama serve',
+          model: null,
+          version: null
+        };
+      }
+      throw err;
+    }
+
+    // Step 2 — Check version (non-fatal)
+    let version = null;
+    try {
+      const res = await fetch(config.endpoint + '/api/version', {
+        signal: AbortSignal.timeout(timeoutShort)
+      });
+      const data = await res.json();
+      version = data.version || null;
+    } catch (_) {
+      version = null;
+    }
+
+    // Step 3 — Check model available
+    try {
+      const res = await fetch(config.endpoint + '/v1/models', {
+        signal: AbortSignal.timeout(timeoutModel)
+      });
+      const data = await res.json();
+      const modelList = (data.data || []).map((m) => m.id || '');
+      const baseModel = config.model.split(':')[0];
+      const found = modelList.some((m) => m.startsWith(baseModel));
+      if (!found) {
+        return {
+          available: false,
+          reason: 'model_missing',
+          detail: 'Run: ollama pull ' + config.model,
+          model: null,
+          version
+        };
+      }
+    } catch (_err) {
+      return {
+        available: false,
+        reason: 'model_missing',
+        detail: 'Run: ollama pull ' + config.model,
+        model: null,
+        version
+      };
+    }
+
+    // Step 4 — Detect GPU error (sleep/wake CUDA bug)
+    let warm = false;
+    try {
+      const res = await fetch(config.endpoint + '/v1/chat/completions', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          model: config.model,
+          messages: [{ role: 'user', content: '{"status":"ok"}' }],
+          max_tokens: 10,
+          num_ctx: 512
+        }),
+        signal: AbortSignal.timeout(timeoutModel)
+      });
+
+      if (!res.ok) {
+        const errBody = await res.text().catch(() => '');
+        if (res.status === 500 && (errBody.includes('GPU') || errBody.includes('CUDA'))) {
+          return {
+            available: false,
+            reason: 'gpu_error',
+            detail: 'GPU error detected. Restart Ollama: ollama serve',
+            model: config.model,
+            version
+          };
+        }
+        // Non-GPU HTTP error — treat as available but cold
+        warm = false;
+      } else {
+        warm = true;
+      }
+    } catch (err) {
+      const isTimeout = err.name === 'TimeoutError' || err.name === 'AbortError';
+      if (isTimeout) {
+        warm = false; // cold start in progress — skip, don't block
+      } else {
+        warm = false;
+      }
+    }
+
+    return { available: true, warm, reason: 'ok', model: config.model, version };
+  } catch (_err) {
+    return {
+      available: false,
+      reason: 'unknown_error',
+      detail: _err.message,
+      model: null,
+      version: null
+    };
+  }
+}
+
+/**
+ * Fire-and-forget warm-up request. Callers should NOT await this.
+ * @param {object} config - resolved config from resolveConfig()
+ */
+async function warmUp(config) {
+  try {
+    await fetch(config.endpoint + '/v1/chat/completions', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        model: config.model,
+        messages: [{ role: 'user', content: '{"status":"ready"}' }],
+        max_tokens: 10,
+        num_ctx: 512,
+        keep_alive: config.advanced.keep_alive
+      }),
+      signal: AbortSignal.timeout(WARMUP_TIMEOUT_MS)
+    });
+  } catch (_) {
+    // Swallow all errors silently — fire and forget
+  }
+}
+
+module.exports = { resolveConfig, checkHealth, warmUp };

package/plugins/pbr/scripts/local-llm/metrics.js

@@ -0,0 +1,252 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const MAX_ENTRIES = 200;
+
+/**
+ * Appends a metric entry to the JSONL log file.
+ * Rotates to keep only the last 200 entries when the log exceeds MAX_ENTRIES lines.
+ * Swallows all errors silently — metrics must never crash hooks.
+ *
+ * @param {string} planningDir - path to the .planning directory
+ * @param {object} entry - metric entry object
+ * @param {string} entry.session_id
+ * @param {string} entry.timestamp
+ * @param {string} entry.operation
+ * @param {string} entry.model
+ * @param {number} entry.latency_ms
+ * @param {number} entry.tokens_used_local
+ * @param {number} entry.tokens_saved_frontier
+ * @param {string} entry.result
+ * @param {boolean} entry.fallback_used
+ * @param {number} entry.confidence
+ */
+function logMetric(planningDir, entry) {
+  try {
+    const logsDir = path.join(planningDir, 'logs');
+    const logFile = path.join(logsDir, 'local-llm-metrics.jsonl');
+
+    fs.mkdirSync(logsDir, { recursive: true });
+    fs.appendFileSync(logFile, JSON.stringify(entry) + '\n', 'utf8');
+
+    // Rotate if over MAX_ENTRIES
+    try {
+      const contents = fs.readFileSync(logFile, 'utf8');
+      const lines = contents.split(/\r?\n/).filter((l) => l.trim() !== '');
+      if (lines.length > MAX_ENTRIES) {
+        const trimmed = lines.slice(lines.length - MAX_ENTRIES);
+        fs.writeFileSync(logFile, trimmed.join('\n') + '\n', 'utf8');
+      }
+    } catch (_) {
+      // Rotation failure is non-fatal
+    }
+  } catch (_) {
+    // Swallow all errors silently
+  }
+}
+
+/**
+ * Reads metric entries from the JSONL log that occurred at or after sessionStartTime.
+ *
+ * @param {string} planningDir - path to the .planning directory
+ * @param {string|Date} sessionStartTime - ISO string or Date
+ * @returns {object[]} Array of matching metric entry objects
+ */
+function readSessionMetrics(planningDir, sessionStartTime) {
+  try {
+    const logFile = path.join(planningDir, 'logs', 'local-llm-metrics.jsonl');
+    const contents = fs.readFileSync(logFile, 'utf8');
+    const startMs = new Date(sessionStartTime).getTime();
+
+    return contents
+      .split(/\r?\n/)
+      .filter((l) => l.trim() !== '')
+      .map((l) => {
+        try {
+          return JSON.parse(l);
+        } catch (_) {
+          return null;
+        }
+      })
+      .filter((e) => e !== null)
+      .filter((e) => {
+        try {
+          return new Date(e.timestamp).getTime() >= startMs;
+        } catch (_) {
+          return false;
+        }
+      });
+  } catch (_) {
+    return [];
+  }
+}
+
+/**
+ * Summarizes an array of metric entries.
+ *
+ * @param {object[]} entries
+ * @param {number} [frontierTokenRate=3.0] - cost per million tokens in USD
+ * @returns {{ total_calls: number, fallback_count: number, avg_latency_ms: number, tokens_saved: number, cost_saved_usd: number }}
+ */
+function summarizeMetrics(entries, frontierTokenRate) {
+  if (!entries || entries.length === 0) {
+    return {
+      total_calls: 0,
+      fallback_count: 0,
+      avg_latency_ms: 0,
+      tokens_saved: 0,
+      cost_saved_usd: 0
+    };
+  }
+
+  const rate = frontierTokenRate != null ? frontierTokenRate : 3.0;
+  const total_calls = entries.length;
+  const fallback_count = entries.filter((e) => e.fallback_used).length;
+  const totalLatency = entries.reduce((sum, e) => sum + (e.latency_ms || 0), 0);
+  const avg_latency_ms = total_calls > 0 ? totalLatency / total_calls : 0;
+  const tokens_saved = entries.reduce((sum, e) => sum + (e.tokens_saved_frontier || 0), 0);
+  const cost_saved_usd = tokens_saved * (rate / 1_000_000);
+
+  return { total_calls, fallback_count, avg_latency_ms, tokens_saved, cost_saved_usd };
+}
+
+/**
+ * Computes lifetime aggregate metrics by reading all entries from the JSONL log.
+ * No date filter — reads everything. Adds a by_operation breakdown keyed by operation.
+ *
+ * @param {string} planningDir - path to the .planning directory
+ * @param {number} [frontierTokenRate=3.0] - cost per million tokens in USD
+ * @returns {{ total_calls: number, fallback_count: number, avg_latency_ms: number, tokens_saved: number, cost_saved_usd: number, by_operation: object }}
+ */
+function computeLifetimeMetrics(planningDir, frontierTokenRate) {
+  const zero = {
+    total_calls: 0,
+    fallback_count: 0,
+    avg_latency_ms: 0,
+    tokens_saved: 0,
+    cost_saved_usd: 0,
+    by_operation: {}
+  };
+
+  try {
+    const logFile = path.join(planningDir, 'logs', 'local-llm-metrics.jsonl');
+    let contents;
+    try {
+      contents = fs.readFileSync(logFile, 'utf8');
+    } catch (_) {
+      return zero;
+    }
+
+    const entries = contents
+      .split(/\r?\n/)
+      .filter((l) => l.trim() !== '')
+      .map((l) => {
+        try {
+          return JSON.parse(l);
+        } catch (_) {
+          return null;
+        }
+      })
+      .filter((e) => e !== null);
+
+    if (entries.length === 0) return zero;
+
+    const rate = frontierTokenRate != null ? frontierTokenRate : 3.0;
+    const total_calls = entries.length;
+    const fallback_count = entries.filter((e) => e.fallback_used).length;
+    const totalLatency = entries.reduce((sum, e) => sum + (e.latency_ms || 0), 0);
+    const avg_latency_ms = total_calls > 0 ? totalLatency / total_calls : 0;
+    const tokens_saved = entries.reduce((sum, e) => sum + (e.tokens_saved_frontier || 0), 0);
+    const cost_saved_usd = tokens_saved * (rate / 1_000_000);
+
+    const by_operation = {};
+    for (const e of entries) {
+      const op = e.operation || 'unknown';
+      if (!by_operation[op]) {
+        by_operation[op] = { calls: 0, fallbacks: 0, tokens_saved: 0 };
+      }
+      by_operation[op].calls += 1;
+      if (e.fallback_used) by_operation[op].fallbacks += 1;
+      by_operation[op].tokens_saved += e.tokens_saved_frontier || 0;
+    }
+
+    return { total_calls, fallback_count, avg_latency_ms, tokens_saved, cost_saved_usd, by_operation };
+  } catch (_) {
+    return zero;
+  }
+}
+
+/**
+ * Formats a metrics aggregate (output of summarizeMetrics) into a human-readable one-liner.
+ *
+ * @param {object} summary - output of summarizeMetrics()
+ * @param {string} [model] - optional model name
+ * @returns {string}
+ */
+function formatSessionSummary(summary, model) {
+  if (!summary || summary.total_calls === 0) {
+    return 'Local LLM: no calls this session';
+  }
+
+  const { total_calls, fallback_count, avg_latency_ms, tokens_saved, cost_saved_usd } = summary;
+
+  let costStr = '';
+  if (cost_saved_usd > 0) {
+    costStr = ` ($${cost_saved_usd.toFixed(2)})`;
+  }
+
+  let fallbackStr = '';
+  if (fallback_count > 0) {
+    fallbackStr = `, ${fallback_count} fallback(s)`;
+  }
+
+  let modelStr = '';
+  if (model) {
+    modelStr = ` [${model}]`;
+  }
+
+  const avgMs = Math.round(avg_latency_ms);
+
+  return `Local LLM: ${total_calls} calls, ~${tokens_saved} frontier tokens saved${costStr}, avg ${avgMs}ms${fallbackStr}${modelStr}`;
+}
+
+/**
+ * Appends a shadow comparison entry to the shadow JSONL log file.
+ * Rotates to keep only the last 200 entries. Swallows all errors silently.
+ *
+ * @param {string} planningDir - path to the .planning directory
+ * @param {object} entry - shadow comparison entry object
+ * @param {string} entry.timestamp
+ * @param {string} entry.operation
+ * @param {string} entry.session_id
+ * @param {boolean} entry.agrees
+ * @param {string|null} entry.local_result
+ * @param {string} entry.frontier_result
+ */
+function logAgreement(planningDir, entry) {
+  try {
+    const logsDir = path.join(planningDir, 'logs');
+    const logFile = path.join(logsDir, 'local-llm-shadow.jsonl');
+
+    fs.mkdirSync(logsDir, { recursive: true });
+    fs.appendFileSync(logFile, JSON.stringify(entry) + '\n', 'utf8');
+
+    // Rotate if over MAX_ENTRIES
+    try {
+      const contents = fs.readFileSync(logFile, 'utf8');
+      const lines = contents.split(/\r?\n/).filter((l) => l.trim() !== '');
+      if (lines.length > MAX_ENTRIES) {
+        const trimmed = lines.slice(lines.length - MAX_ENTRIES);
+        fs.writeFileSync(logFile, trimmed.join('\n') + '\n', 'utf8');
+      }
+    } catch (_) {
+      // Rotation failure is non-fatal
+    }
+  } catch (_) {
+    // Swallow all errors silently
+  }
+}
+
+module.exports = { logMetric, readSessionMetrics, summarizeMetrics, computeLifetimeMetrics, formatSessionSummary, logAgreement };

package/plugins/pbr/scripts/local-llm/operations/classify-artifact.js

@@ -0,0 +1,76 @@
+'use strict';
+
+const { complete, tryParseJSON, isDisabled } = require('../client');
+const { logMetric } = require('../metrics');
+const { route } = require('../router');
+
+/**
+ * Classifies a PLAN.md or SUMMARY.md artifact using the local LLM.
+ *
+ * @param {object} config - resolved local_llm config block
+ * @param {string} planningDir - path to the .planning directory
+ * @param {string} content - file content to classify
+ * @param {string} fileType - 'PLAN' or 'SUMMARY'
+ * @param {string} [sessionId] - optional session identifier for metrics
+ * @returns {Promise<{ classification: string, confidence: number, reason: string, latency_ms: number, fallback_used: boolean }|null>}
+ */
+async function classifyArtifact(config, planningDir, content, fileType, sessionId) {
+  if (!config.enabled || !config.features.artifact_classification) return null;
+  if (isDisabled('artifact-classification', config.advanced.disable_after_failures)) return null;
+
+  const maxChars = (config.advanced.max_input_tokens || 1024) * 4;
+  const truncatedContent = content.length > maxChars ? content.slice(0, maxChars) : content;
+
+  let prompt;
+  if (fileType === 'PLAN') {
+    prompt =
+      'Classify this PLAN.md as stub, partial, or complete. A stub has placeholder tasks or missing required XML elements. A partial has some tasks filled but action/verify/done are vague. A complete has all tasks with specific steps, executable verify commands, and observable done conditions. Respond with JSON: {"classification": "stub"|"partial"|"complete", "confidence": 0.0-1.0, "reason": "one sentence"}\n\nContent:\n' +
+      truncatedContent;
+  } else if (fileType === 'SUMMARY') {
+    prompt =
+      'Classify this SUMMARY.md as substantive or thin. Substantive means it has specific artifact paths, commit hashes, and observable outcomes. Thin means vague or placeholder content. Respond with JSON: {"classification": "substantive"|"thin", "confidence": 0.0-1.0, "reason": "one sentence"}\n\nContent:\n' +
+      truncatedContent;
+  } else {
+    return null;
+  }
+
+  try {
+    const result = await route(config, prompt, 'artifact-classification', (logprobs) =>
+      complete(config, prompt, 'artifact-classification', { logprobs })
+    );
+    if (result === null) return null;
+    const parsed = tryParseJSON(result.content);
+    if (!parsed.ok) return null;
+
+    const validPlanClassifications = ['stub', 'partial', 'complete'];
+    const validSummaryClassifications = ['substantive', 'thin'];
+    const validValues = fileType === 'PLAN' ? validPlanClassifications : validSummaryClassifications;
+    if (!parsed.data.classification || !validValues.includes(parsed.data.classification)) return null;
+
+    const metricEntry = {
+      session_id: sessionId || 'unknown',
+      timestamp: new Date().toISOString(),
+      operation: 'artifact-classification',
+      model: config.model,
+      latency_ms: result.latency_ms,
+      tokens_used_local: result.tokens,
+      tokens_saved_frontier: 420,
+      result: parsed.data.classification,
+      fallback_used: false,
+      confidence: parsed.data.confidence || 0.9
+    };
+    logMetric(planningDir, metricEntry);
+
+    return {
+      classification: parsed.data.classification,
+      confidence: parsed.data.confidence || 0.9,
+      reason: parsed.data.reason || '',
+      latency_ms: result.latency_ms,
+      fallback_used: false
+    };
+  } catch (_) {
+    return null;
+  }
+}
+
+module.exports = { classifyArtifact };

package/plugins/pbr/scripts/local-llm/operations/classify-error.js

@@ -0,0 +1,75 @@
+'use strict';
+
+const { complete, tryParseJSON, isDisabled } = require('../client');
+const { logMetric } = require('../metrics');
+const { route } = require('../router');
+
+const ERROR_CATEGORIES = [
+  'connection_refused',
+  'timeout',
+  'missing_output',
+  'wrong_output_format',
+  'permission_error',
+  'unknown'
+];
+
+/**
+ * Classifies an agent error into one of 6 categories using the local LLM.
+ *
+ * @param {object} config - resolved local_llm config block
+ * @param {string} planningDir - path to the .planning directory
+ * @param {string} errorText - the error message or stack trace
+ * @param {string} [agentType] - the agent type that produced the error
+ * @param {string} [sessionId] - optional session identifier for metrics
+ * @returns {Promise<{ category: string, confidence: number, latency_ms: number, fallback_used: boolean }|null>}
+ */
+async function classifyError(config, planningDir, errorText, agentType, sessionId) {
+  if (!config.enabled) return null;
+  if (isDisabled('error-classification', config.advanced.disable_after_failures)) return null;
+
+  const truncatedError = errorText.length > 500 ? errorText.slice(0, 500) : errorText;
+
+  const prompt =
+    'Classify this agent error into one category. Categories: connection_refused (network/ECONNREFUSED), timeout (operation timed out), missing_output (expected file/artifact not found), wrong_output_format (output exists but malformed), permission_error (filesystem/permission issue), unknown (none of the above). Respond with JSON: {"category": "<one of the 6>", "confidence": 0.0-1.0}\n\nAgent: ' +
+    (agentType || 'unknown') +
+    '\nError: ' +
+    truncatedError;
+
+  try {
+    const result = await route(config, prompt, 'error-classification', (logprobs) =>
+      complete(config, prompt, 'error-classification', { logprobs })
+    );
+    if (result === null) return null;
+    const parsed = tryParseJSON(result.content);
+    if (!parsed.ok) return null;
+
+    const category = ERROR_CATEGORIES.includes(parsed.data.category)
+      ? parsed.data.category
+      : 'unknown';
+
+    const metricEntry = {
+      session_id: sessionId || 'unknown',
+      timestamp: new Date().toISOString(),
+      operation: 'error-classification',
+      model: config.model,
+      latency_ms: result.latency_ms,
+      tokens_used_local: result.tokens,
+      tokens_saved_frontier: 120,
+      result: category,
+      fallback_used: false,
+      confidence: parsed.data.confidence || 0.9
+    };
+    logMetric(planningDir, metricEntry);
+
+    return {
+      category,
+      confidence: parsed.data.confidence || 0.9,
+      latency_ms: result.latency_ms,
+      fallback_used: false
+    };
+  } catch (_) {
+    return null;
+  }
+}
+
+module.exports = { classifyError, ERROR_CATEGORIES };

package/plugins/pbr/scripts/local-llm/operations/score-source.js

@@ -0,0 +1,72 @@
+'use strict';
+
+const { complete, tryParseJSON, isDisabled } = require('../client');
+const { logMetric } = require('../metrics');
+const { route } = require('../router');
+
+const SOURCE_LEVELS = ['S0', 'S1', 'S2', 'S3', 'S4', 'S5', 'S6'];
+
+/**
+ * Scores a research source on the S0-S6 credibility scale using the local LLM.
+ *
+ * S0=local prior research, S1=live MCP docs, S2=official docs, S3=official GitHub,
+ * S4=verified WebSearch (2+ sources), S5=unverified WebSearch, S6=training knowledge.
+ *
+ * @param {object} config - resolved local_llm config block
+ * @param {string} planningDir - path to the .planning directory
+ * @param {string} sourceText - text content from the source
+ * @param {string} sourceUrl - URL or identifier for the source
+ * @param {string} [sessionId] - optional session identifier for metrics
+ * @returns {Promise<{ level: string, confidence: number, reason: string, latency_ms: number, fallback_used: boolean }|null>}
+ */
+async function scoreSource(config, planningDir, sourceText, sourceUrl, sessionId) {
+  if (!config.enabled) return null;
+  if (!config.features || !config.features.source_scoring) return null;
+  if (isDisabled('source-scoring', config.advanced.disable_after_failures)) return null;
+
+  const maxChars = (config.advanced.max_input_tokens || 1024) * 4;
+  const truncated = sourceText.length > maxChars ? sourceText.slice(0, maxChars) : sourceText;
+
+  const prompt =
+    'Score this research source on the S0-S6 credibility scale. S0=local prior research, S1=live MCP docs, S2=official docs, S3=official GitHub, S4=verified WebSearch (2+ sources), S5=unverified WebSearch, S6=training knowledge. Respond with JSON: {"level": "S0"-"S6", "confidence": 0.0-1.0, "reason": "one sentence"}\n\nURL: ' +
+    sourceUrl +
+    '\nContent excerpt:\n' +
+    truncated;
+
+  try {
+    const result = await route(config, prompt, 'source-scoring', (logprobs) =>
+      complete(config, prompt, 'source-scoring', { logprobs })
+    );
+    if (result === null) return null;
+    const parsed = tryParseJSON(result.content);
+    if (!parsed.ok) return null;
+
+    const level = SOURCE_LEVELS.includes(parsed.data.level) ? parsed.data.level : 'S6';
+
+    const metricEntry = {
+      session_id: sessionId || 'unknown',
+      timestamp: new Date().toISOString(),
+      operation: 'source-scoring',
+      model: config.model,
+      latency_ms: result.latency_ms,
+      tokens_used_local: result.tokens,
+      tokens_saved_frontier: 80,
+      result: level,
+      fallback_used: false,
+      confidence: parsed.data.confidence || 0.9
+    };
+    logMetric(planningDir, metricEntry);
+
+    return {
+      level,
+      confidence: parsed.data.confidence || 0.9,
+      reason: parsed.data.reason || '',
+      latency_ms: result.latency_ms,
+      fallback_used: false
+    };
+  } catch (_) {
+    return null;
+  }
+}
+
+module.exports = { scoreSource, SOURCE_LEVELS };