@sienklogic/plan-build-run 2.18.1 → 2.19.1

package/plugins/pbr/scripts/lib/local-llm/client.js

@@ -0,0 +1,237 @@
+/* global fetch, AbortSignal, performance */
+'use strict';
+
+const {
+  isDisabledPersistent,
+  recordFailurePersistent,
+  resetCircuitPersistent
+} = require('../circuit-state');
+
+// Circuit breaker: Map<operationType, { failures: number, disabled: boolean }>
+const circuitState = new Map();
+
+/**
+ * Attempts to parse JSON from text that may be raw JSON or wrapped in a markdown code block.
+ * @param {string} text
+ * @returns {{ ok: true, data: any } | { ok: false, raw: string }}
+ */
+function tryParseJSON(text) {
+  // Attempt 1: direct parse
+  try {
+    const data = JSON.parse(text);
+    return { ok: true, data };
+  } catch (_) {
+    // fall through
+  }
+
+  // Attempt 2: extract from markdown code block
+  const codeBlockMatch = text.match(/```(?:json)?\s*\n?([\s\S]*?)\n?```/);
+  if (codeBlockMatch) {
+    try {
+      const data = JSON.parse(codeBlockMatch[1].trim());
+      return { ok: true, data };
+    } catch (_) {
+      // fall through
+    }
+  }
+
+  // Attempt 3: find first {...}
+  const objectMatch = text.match(/\{[\s\S]*\}/);
+  if (objectMatch) {
+    try {
+      const data = JSON.parse(objectMatch[0]);
+      return { ok: true, data };
+    } catch (_) {
+      // fall through
+    }
+  }
+
+  return { ok: false, raw: text };
+}
+
+/**
+ * Maps an error to one of 5 canonical types.
+ * @param {Error} err
+ * @returns {{ type: string, message: string }}
+ */
+function categorizeError(err) {
+  if (
+    (err.cause && err.cause.code === 'ECONNREFUSED') ||
+    (err.message && err.message.includes('ECONNREFUSED'))
+  ) {
+    return { type: 'ECONNREFUSED', message: err.message };
+  }
+  if (err.name === 'TimeoutError' || err.name === 'AbortError') {
+    return { type: 'timeout', message: err.message };
+  }
+  if (err.message && err.message.startsWith('HTTP ')) {
+    return { type: 'http_error', message: err.message };
+  }
+  if (err instanceof SyntaxError) {
+    return { type: 'json_parse', message: err.message };
+  }
+  return { type: 'wrong_answer', message: err.message };
+}
+
+/**
+ * Returns true if the circuit is open (operation should be skipped).
+ * @param {string} operationType
+ * @param {number} maxFailures
+ * @returns {boolean}
+ */
+function isDisabled(operationType, maxFailures) {
+  const entry = circuitState.get(operationType);
+  if (!entry) return false;
+  return entry.disabled || entry.failures >= maxFailures;
+}
+
+/**
+ * Records a failure for an operation type. Disables the circuit if maxFailures is reached.
+ * @param {string} operationType
+ * @param {number} maxFailures
+ */
+function recordFailure(operationType, maxFailures) {
+  const entry = circuitState.get(operationType) || { failures: 0, disabled: false };
+  entry.failures += 1;
+  if (entry.failures >= maxFailures) {
+    entry.disabled = true;
+  }
+  circuitState.set(operationType, entry);
+}
+
+/**
+ * Resets the circuit breaker for an operation type.
+ * @param {string} operationType
+ */
+function resetCircuit(operationType) {
+  circuitState.delete(operationType);
+}
+
+/**
+ * Sends a chat completion request to a local LLM endpoint with retry and circuit-breaker logic.
+ *
+ * @param {object} config - local_llm config block (resolved)
+ * @param {string} prompt - user message to send
+ * @param {string} operationType - operation identifier for circuit breaker tracking
+ * @param {object} [options={}] - optional parameters
+ * @param {boolean} [options.logprobs] - if true, request logprobs from the API
+ * @param {string} [options.planningDir] - optional .planning directory for persistent circuit breaker state
+ * @returns {Promise<{ content: string, latency_ms: number, tokens: number, logprobsData: Array<{token: string, logprob: number}>|null }>}
+ */
+async function complete(config, prompt, operationType, options = {}) {
+  const endpoint = config.endpoint || 'http://localhost:11434';
+  const model = config.model || 'qwen2.5-coder:7b';
+  const timeoutMs = config.timeout_ms || 3000;
+  const maxRetries = config.max_retries != null ? config.max_retries : 1;
+  const numCtx = (config.advanced && config.advanced.num_ctx) || 4096;
+  const keepAlive = (config.advanced && config.advanced.keep_alive) || '30m';
+  const maxFailures = (config.advanced && config.advanced.disable_after_failures) || 3;
+  const planningDir = options.planningDir || null;
+
+  // Check in-memory circuit first (fast path), then persistent state
+  if (isDisabled(operationType, maxFailures)) {
+    const err = new Error('Circuit open for operation: ' + operationType);
+    err.type = 'circuit_open';
+    throw err;
+  }
+  if (planningDir && isDisabledPersistent(planningDir, operationType, maxFailures)) {
+    const err = new Error('Circuit open for operation: ' + operationType);
+    err.type = 'circuit_open';
+    throw err;
+  }
+
+  const bodyObj = {
+    model,
+    messages: [
+      {
+        role: 'system',
+        content:
+          'You are a precise classification assistant. Always respond with valid JSON only. No explanations outside the JSON.'
+      },
+      { role: 'user', content: prompt }
+    ],
+    response_format: { type: 'json_object' },
+    temperature: 0.1,
+    max_tokens: 200,
+    keep_alive: keepAlive,
+    num_ctx: numCtx
+  };
+  if (options.logprobs === true) {
+    bodyObj.logprobs = true;
+    bodyObj.top_logprobs = 3;
+  }
+  const body = JSON.stringify(bodyObj);
+
+  const url = endpoint + '/v1/chat/completions';
+  const totalAttempts = maxRetries + 1;
+
+  let lastErr;
+  for (let attempt = 0; attempt < totalAttempts; attempt++) {
+    const start = performance.now();
+    try {
+      const res = await fetch(url, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body,
+        signal: AbortSignal.timeout(timeoutMs)
+      });
+
+      if (!res.ok) {
+        const errText = await res.text().catch(() => '');
+        throw new Error('HTTP ' + res.status + ': ' + errText);
+      }
+
+      const json = await res.json();
+      const content = json.choices[0].message.content;
+      const completionTokens = (json.usage && json.usage.completion_tokens) || 0;
+      const latency_ms = performance.now() - start;
+      const logprobsData = (options.logprobs && json.choices[0].logprobs)
+        ? json.choices[0].logprobs.content
+        : null;
+
+      return { content, latency_ms, tokens: completionTokens, logprobsData };
+    } catch (err) {
+      lastErr = err;
+      const isConnRefused =
+        (err.cause && err.cause.code === 'ECONNREFUSED') ||
+        (err.message && err.message.includes('ECONNREFUSED'));
+
+      if (isConnRefused) {
+        // Server not running — no point retrying
+        recordFailure(operationType, maxFailures);
+        if (planningDir) recordFailurePersistent(planningDir, operationType, maxFailures);
+        throw err;
+      }
+
+      const isTimeout = err.name === 'TimeoutError' || err.name === 'AbortError';
+      const isHttpError = err.message && err.message.startsWith('HTTP ');
+
+      if ((isTimeout || isHttpError) && attempt < totalAttempts - 1) {
+        // Retry
+        continue;
+      }
+
+      // Final attempt or non-retryable error
+      recordFailure(operationType, maxFailures);
+      if (planningDir) recordFailurePersistent(planningDir, operationType, maxFailures);
+      throw err;
+    }
+  }
+
+  // Should not reach here, but guard anyway
+  recordFailure(operationType, maxFailures);
+  if (planningDir) recordFailurePersistent(planningDir, operationType, maxFailures);
+  throw lastErr;
+}
+
+module.exports = {
+  tryParseJSON,
+  categorizeError,
+  isDisabled,
+  recordFailure,
+  resetCircuit,
+  complete,
+  isDisabledPersistent,
+  recordFailurePersistent,
+  resetCircuitPersistent
+};

package/plugins/pbr/scripts/lib/local-llm/health.js

@@ -0,0 +1,12 @@
+'use strict';
+
+/**
+ * local-llm/health.cjs — Stub for LLM health/config subsystem.
+ * Full implementation deferred to v3 (ADV-01).
+ */
+
+function resolveConfig(config) {
+  return Object.assign({ enabled: false, provider: 'none' }, config || {});
+}
+
+module.exports = { resolveConfig };

package/plugins/pbr/scripts/lib/local-llm/index.js

@@ -0,0 +1,89 @@
+'use strict';
+
+/**
+ * local-llm/index.cjs — Local LLM subsystem stub for PBR v2.0.
+ *
+ * Full implementation deferred to v3 (ADV-01: Local LLM shadow mode).
+ * See REQUIREMENTS.md ADV-01 for v3 plans.
+ *
+ * This stub ensures the dispatcher does not crash when LLM commands are
+ * invoked. All functions return graceful "not available" responses instead
+ * of throwing errors.
+ */
+
+const STUB_MESSAGE = 'Local LLM subsystem not available in v2.0. See ADV-01 for v3 plans.';
+
+/**
+ * Check LLM availability and health.
+ * @returns {{ available: boolean, message: string }}
+ */
+function llmHealth() {
+  return { available: false, message: STUB_MESSAGE };
+}
+
+/**
+ * Get LLM subsystem status.
+ * @returns {{ status: string, message: string }}
+ */
+function llmStatus() {
+  return { status: 'disabled', message: STUB_MESSAGE };
+}
+
+/**
+ * Classify input using local LLM.
+ * @returns {{ error: string }}
+ */
+function llmClassify() {
+  return { error: STUB_MESSAGE };
+}
+
+/**
+ * Score a source file using local LLM.
+ * @returns {{ error: string }}
+ */
+function llmScoreSource() {
+  return { error: STUB_MESSAGE };
+}
+
+/**
+ * Classify an error using local LLM.
+ * @returns {{ error: string }}
+ */
+function llmClassifyError() {
+  return { error: STUB_MESSAGE };
+}
+
+/**
+ * Summarize content using local LLM.
+ * @returns {{ error: string }}
+ */
+function llmSummarize() {
+  return { error: STUB_MESSAGE };
+}
+
+/**
+ * Get LLM usage metrics.
+ * @returns {{ metrics: Array, message: string }}
+ */
+function llmMetrics() {
+  return { metrics: [], message: STUB_MESSAGE };
+}
+
+/**
+ * Adjust classification thresholds using local LLM.
+ * @returns {{ error: string }}
+ */
+function llmAdjustThresholds() {
+  return { error: STUB_MESSAGE };
+}
+
+module.exports = {
+  llmHealth,
+  llmStatus,
+  llmClassify,
+  llmScoreSource,
+  llmClassifyError,
+  llmSummarize,
+  llmMetrics,
+  llmAdjustThresholds
+};

package/plugins/pbr/scripts/lib/local-llm/metrics.js

@@ -0,0 +1,20 @@
+'use strict';
+
+/**
+ * local-llm/metrics.cjs — Stub for LLM metrics subsystem.
+ * Full implementation deferred to v3 (ADV-01).
+ */
+
+function computeLifetimeMetrics(_planningDir) {
+  return { total_calls: 0, tokens_saved: 0 };
+}
+
+function readSessionMetrics(_planningDir, _since) {
+  return [];
+}
+
+function summarizeMetrics(_entries) {
+  return { total_calls: 0, tokens_saved: 0 };
+}
+
+module.exports = { computeLifetimeMetrics, readSessionMetrics, summarizeMetrics };

package/plugins/pbr/scripts/lib/local-llm/operations/classify-artifact.js

@@ -0,0 +1,4 @@
+'use strict';
+// Stub: local-llm classify-artifact (deferred to v3 ADV-01)
+async function classifyArtifact() { return { classification: 'unknown', confidence: 0 }; }
+module.exports = { classifyArtifact };

package/plugins/pbr/scripts/lib/local-llm/operations/classify-commit.js

@@ -0,0 +1,4 @@
+'use strict';
+// Stub: local-llm classify-commit (deferred to v3 ADV-01)
+async function classifyCommit() { return { type: 'unknown', confidence: 0 }; }
+module.exports = { classifyCommit };

package/plugins/pbr/scripts/lib/local-llm/operations/classify-error.js

@@ -0,0 +1,4 @@
+'use strict';
+// Stub: local-llm classify-error (deferred to v3 ADV-01)
+async function classifyError() { return { category: 'unknown', confidence: 0 }; }
+module.exports = { classifyError };

package/plugins/pbr/scripts/lib/local-llm/operations/classify-file-intent.js

@@ -0,0 +1,4 @@
+'use strict';
+// Stub: local-llm classify-file-intent (deferred to v3 ADV-01)
+async function classifyFileIntent() { return { intent: 'unknown', confidence: 0 }; }
+module.exports = { classifyFileIntent };

package/plugins/pbr/scripts/lib/local-llm/operations/score-source.js

@@ -0,0 +1,72 @@
+'use strict';
+
+const { complete, tryParseJSON, isDisabled } = require('../client');
+const { logMetric } = require('../metrics');
+const { route } = require('../router');
+
+const SOURCE_LEVELS = ['S0', 'S1', 'S2', 'S3', 'S4', 'S5', 'S6'];
+
+/**
+ * Scores a research source on the S0-S6 credibility scale using the local LLM.
+ *
+ * S0=local prior research, S1=live MCP docs, S2=official docs, S3=official GitHub,
+ * S4=verified WebSearch (2+ sources), S5=unverified WebSearch, S6=training knowledge.
+ *
+ * @param {object} config - resolved local_llm config block
+ * @param {string} planningDir - path to the .planning directory
+ * @param {string} sourceText - text content from the source
+ * @param {string} sourceUrl - URL or identifier for the source
+ * @param {string} [sessionId] - optional session identifier for metrics
+ * @returns {Promise<{ level: string, confidence: number, reason: string, latency_ms: number, fallback_used: boolean }|null>}
+ */
+async function scoreSource(config, planningDir, sourceText, sourceUrl, sessionId) {
+  if (!config.enabled) return null;
+  if (!config.features || !config.features.source_scoring) return null;
+  if (isDisabled('source-scoring', config.advanced.disable_after_failures)) return null;
+
+  const maxChars = (config.advanced.max_input_tokens || 1024) * 4;
+  const truncated = sourceText.length > maxChars ? sourceText.slice(0, maxChars) : sourceText;
+
+  const prompt =
+    'Score this research source on the S0-S6 credibility scale. S0=local prior research, S1=live MCP docs, S2=official docs, S3=official GitHub, S4=verified WebSearch (2+ sources), S5=unverified WebSearch, S6=training knowledge. Respond with JSON: {"level": "S0"-"S6", "confidence": 0.0-1.0, "reason": "one sentence"}\n\nURL: ' +
+    sourceUrl +
+    '\nContent excerpt:\n' +
+    truncated;
+
+  try {
+    const result = await route(config, prompt, 'source-scoring', (logprobs) =>
+      complete(config, prompt, 'source-scoring', { logprobs })
+    );
+    if (result === null) return null;
+    const parsed = tryParseJSON(result.content);
+    if (!parsed.ok) return null;
+
+    const level = SOURCE_LEVELS.includes(parsed.data.level) ? parsed.data.level : 'S6';
+
+    const metricEntry = {
+      session_id: sessionId || 'unknown',
+      timestamp: new Date().toISOString(),
+      operation: 'source-scoring',
+      model: config.model,
+      latency_ms: result.latency_ms,
+      tokens_used_local: result.tokens,
+      tokens_saved_frontier: 80,
+      result: level,
+      fallback_used: false,
+      confidence: parsed.data.confidence || 0.9
+    };
+    logMetric(planningDir, metricEntry);
+
+    return {
+      level,
+      confidence: parsed.data.confidence || 0.9,
+      reason: parsed.data.reason || '',
+      latency_ms: result.latency_ms,
+      fallback_used: false
+    };
+  } catch (_) {
+    return null;
+  }
+}
+
+module.exports = { scoreSource, SOURCE_LEVELS };

package/plugins/pbr/scripts/lib/local-llm/operations/summarize-context.js

@@ -0,0 +1,62 @@
+'use strict';
+
+const { complete, isDisabled } = require('../client');
+const { logMetric } = require('../metrics');
+const { route } = require('../router');
+
+/**
+ * Produces a concise plain-text summary of project context using the local LLM.
+ *
+ * @param {object} config - resolved local_llm config block
+ * @param {string} planningDir - path to the .planning directory
+ * @param {string} contextText - project context text to summarize
+ * @param {number} [maxWords] - target word count for the summary (default 150)
+ * @param {string} [sessionId] - optional session identifier for metrics
+ * @returns {Promise<{ summary: string, latency_ms: number, fallback_used: boolean }|null>}
+ */
+async function summarizeContext(config, planningDir, contextText, maxWords, sessionId) {
+  if (!config.enabled) return null;
+  if (!config.features.context_summarization) return null;
+  if (isDisabled('context-summarization', config.advanced.disable_after_failures)) return null;
+
+  const maxChars = (config.advanced.max_input_tokens || 1024) * 4;
+  const truncated = contextText.length > maxChars ? contextText.slice(0, maxChars) : contextText;
+  const targetWords = maxWords || 150;
+
+  const prompt =
+    'Summarize the following project context in under ' +
+    targetWords +
+    ' words. Focus on: current phase goal, key decisions made, what is built so far, and what is still needed. Output plain text only — no JSON, no headings, no bullet points.\n\nContext:\n' +
+    truncated;
+
+  try {
+    const result = await route(config, prompt, 'context-summarization', (logprobs) =>
+      complete(config, prompt, 'context-summarization', { logprobs })
+    );
+    if (result === null) return null;
+
+    const metricEntry = {
+      session_id: sessionId || 'unknown',
+      timestamp: new Date().toISOString(),
+      operation: 'context-summarization',
+      model: config.model,
+      latency_ms: result.latency_ms,
+      tokens_used_local: result.tokens,
+      tokens_saved_frontier: 350,
+      result: 'summary',
+      fallback_used: false,
+      confidence: 0.9
+    };
+    logMetric(planningDir, metricEntry);
+
+    return {
+      summary: result.content.trim(),
+      latency_ms: result.latency_ms,
+      fallback_used: false
+    };
+  } catch (_) {
+    return null;
+  }
+}
+
+module.exports = { summarizeContext };

package/plugins/pbr/scripts/lib/local-llm/operations/triage-test-output.js

@@ -0,0 +1,12 @@
+'use strict';
+
+/**
+ * local-llm/operations/triage-test-output.cjs — Stub for test output triage.
+ * Full implementation deferred to v3 (ADV-01).
+ */
+
+async function triageTestOutput(_config, _planningDir, _output, _runner, _sessionId) {
+  return { category: 'unknown', confidence: 0, file_hint: null };
+}
+
+module.exports = { triageTestOutput };

package/plugins/pbr/scripts/lib/local-llm/operations/validate-task.js

@@ -0,0 +1,4 @@
+'use strict';
+// Stub: local-llm validate-task (deferred to v3 ADV-01)
+async function validateTask() { return { valid: true, confidence: 0 }; }
+module.exports = { validateTask };

package/plugins/pbr/scripts/lib/local-llm/router.js

@@ -0,0 +1,101 @@
+'use strict';
+
+const { runShadow } = require('./shadow');
+
+const COMPLEXITY_HIGH_THRESHOLD = 0.65;
+
+/**
+ * Scores the complexity of a prompt using a weighted surface heuristic.
+ * Returns a value in [0, 1] where higher means more complex.
+ *
+ * @param {string} prompt
+ * @returns {number}
+ */
+function scoreComplexity(prompt) {
+  const words = prompt.split(/\s+/).length;
+  const codeBlocks = (prompt.match(/```/g) || []).length / 2;
+  const constraints = (prompt.match(/\b(must|should|exactly|only|never|always)\b/gi) || []).length;
+  const reasoning = (prompt.match(/\b(why|explain|compare|analyze|reason|evaluate)\b/gi) || []).length;
+  const structuredOut = /\b(json|schema|yaml|frontmatter)\b/i.test(prompt) ? 1 : 0;
+  return Math.min(words / 500, 1.0) * 0.25 +
+         Math.min(codeBlocks / 3, 1.0) * 0.20 +
+         Math.min(constraints / 5, 1.0) * 0.20 +
+         Math.min(reasoning / 3, 1.0) * 0.20 +
+         structuredOut * 0.15;
+}
+
+/**
+ * Extracts a confidence score from logprobs data returned by the local LLM.
+ * Returns a value in [0, 1] or null if no logprobs data is available.
+ *
+ * @param {Array<{token: string, logprob: number}>|null|undefined} logprobsData
+ * @returns {number|null}
+ */
+function extractConfidence(logprobsData) {
+  if (!logprobsData || logprobsData.length === 0) return null;
+  const sum = logprobsData.reduce((acc, t) => acc + t.logprob, 0);
+  const avgLogprob = sum / logprobsData.length;
+  return Math.min(1, Math.max(0, Math.exp(avgLogprob)));
+}
+
+/**
+ * Routes a prompt through local LLM or signals caller to use frontier model.
+ * Returns the local LLM result if local is suitable, or null if caller should
+ * fall back to frontier. Never throws — all errors return null.
+ *
+ * @param {object} config - local_llm config block with routing_strategy and advanced settings
+ * @param {string} prompt - the prompt being routed
+ * @param {string} operationType - operation identifier
+ * @param {function(boolean): Promise<{content: string, logprobsData: Array|null}>} callLocalFn
+ *   Async function accepting a logprobs boolean, returns the local LLM result object.
+ * @param {string} [planningDir] - path to .planning directory; when provided enables shadow mode
+ * @param {Function} [frontierResultFn] - async function that calls the frontier model;
+ *   NOTE: parameter inversion vs shadow.js — here LOCAL has already run (it's the primary result)
+ *   and FRONTIER is the shadow. We pass frontierResultFn as shadow.js arg 4 (localResultFn slot)
+ *   so shadow.js calls it, and result.content as arg 5 (frontierResult slot, the committed result).
+ * @returns {Promise<{content: string, logprobsData: Array|null}|null>}
+ */
+async function route(config, prompt, operationType, callLocalFn, planningDir, frontierResultFn) {
+  try {
+    const routingStrategy = (config && config.routing_strategy) || 'local_first';
+    const confidenceThreshold = (config && config.advanced && config.advanced.confidence_threshold) || 0.9;
+
+    if (routingStrategy === 'quality_first') {
+      const score = scoreComplexity(prompt);
+      if (score >= 0.3) return null;
+      const result = await callLocalFn(false);
+      if (result !== null && planningDir && frontierResultFn) {
+        runShadow(config, planningDir, operationType, frontierResultFn, result.content);
+      }
+      return result;
+    }
+
+    if (routingStrategy === 'balanced') {
+      const score = scoreComplexity(prompt);
+      if (score > 0.45) return null;
+      const result = await callLocalFn(true);
+      const confidence = extractConfidence(result && result.logprobsData);
+      if (confidence === null || confidence < 0.75) return null;
+      if (result !== null && planningDir && frontierResultFn) {
+        runShadow(config, planningDir, operationType, frontierResultFn, result.content);
+      }
+      return result;
+    }
+
+    // Default: local_first
+    const score = scoreComplexity(prompt);
+    if (score > COMPLEXITY_HIGH_THRESHOLD) return null;
+    const result = await callLocalFn(true);
+    const confidence = extractConfidence(result && result.logprobsData);
+    if (confidence === null || confidence < confidenceThreshold) return null;
+    if (result !== null && planningDir && frontierResultFn) {
+      runShadow(config, planningDir, operationType, frontierResultFn, result.content);
+    }
+    return result;
+  } catch (_) {
+    return null;
+  }
+}
+
+module.exports = { route, scoreComplexity, extractConfidence };
+module.exports.COMPLEXITY_HIGH_THRESHOLD = COMPLEXITY_HIGH_THRESHOLD;