@sienklogic/plan-build-run 2.22.1 → 2.23.0

package/plugins/pbr/scripts/local-llm/client.js

@@ -0,0 +1,214 @@
+/* global fetch, AbortSignal, performance */
+'use strict';
+
+// Circuit breaker: Map<operationType, { failures: number, disabled: boolean }>
+const circuitState = new Map();
+
+/**
+ * Attempts to parse JSON from text that may be raw JSON or wrapped in a markdown code block.
+ * @param {string} text
+ * @returns {{ ok: true, data: any } | { ok: false, raw: string }}
+ */
+function tryParseJSON(text) {
+  // Attempt 1: direct parse
+  try {
+    const data = JSON.parse(text);
+    return { ok: true, data };
+  } catch (_) {
+    // fall through
+  }
+
+  // Attempt 2: extract from markdown code block
+  const codeBlockMatch = text.match(/```(?:json)?\s*\n?([\s\S]*?)\n?```/);
+  if (codeBlockMatch) {
+    try {
+      const data = JSON.parse(codeBlockMatch[1].trim());
+      return { ok: true, data };
+    } catch (_) {
+      // fall through
+    }
+  }
+
+  // Attempt 3: find first {...}
+  const objectMatch = text.match(/\{[\s\S]*\}/);
+  if (objectMatch) {
+    try {
+      const data = JSON.parse(objectMatch[0]);
+      return { ok: true, data };
+    } catch (_) {
+      // fall through
+    }
+  }
+
+  return { ok: false, raw: text };
+}
+
+/**
+ * Maps an error to one of 5 canonical types.
+ * @param {Error} err
+ * @returns {{ type: string, message: string }}
+ */
+function categorizeError(err) {
+  if (
+    (err.cause && err.cause.code === 'ECONNREFUSED') ||
+    (err.message && err.message.includes('ECONNREFUSED'))
+  ) {
+    return { type: 'ECONNREFUSED', message: err.message };
+  }
+  if (err.name === 'TimeoutError' || err.name === 'AbortError') {
+    return { type: 'timeout', message: err.message };
+  }
+  if (err.message && err.message.startsWith('HTTP ')) {
+    return { type: 'http_error', message: err.message };
+  }
+  if (err instanceof SyntaxError) {
+    return { type: 'json_parse', message: err.message };
+  }
+  return { type: 'wrong_answer', message: err.message };
+}
+
+/**
+ * Returns true if the circuit is open (operation should be skipped).
+ * @param {string} operationType
+ * @param {number} maxFailures
+ * @returns {boolean}
+ */
+function isDisabled(operationType, maxFailures) {
+  const entry = circuitState.get(operationType);
+  if (!entry) return false;
+  return entry.disabled || entry.failures >= maxFailures;
+}
+
+/**
+ * Records a failure for an operation type. Disables the circuit if maxFailures is reached.
+ * @param {string} operationType
+ * @param {number} maxFailures
+ */
+function recordFailure(operationType, maxFailures) {
+  const entry = circuitState.get(operationType) || { failures: 0, disabled: false };
+  entry.failures += 1;
+  if (entry.failures >= maxFailures) {
+    entry.disabled = true;
+  }
+  circuitState.set(operationType, entry);
+}
+
+/**
+ * Resets the circuit breaker for an operation type.
+ * @param {string} operationType
+ */
+function resetCircuit(operationType) {
+  circuitState.delete(operationType);
+}
+
+/**
+ * Sends a chat completion request to a local LLM endpoint with retry and circuit-breaker logic.
+ *
+ * @param {object} config - local_llm config block (resolved)
+ * @param {string} prompt - user message to send
+ * @param {string} operationType - operation identifier for circuit breaker tracking
+ * @param {object} [options={}] - optional parameters
+ * @param {boolean} [options.logprobs] - if true, request logprobs from the API
+ * @returns {Promise<{ content: string, latency_ms: number, tokens: number, logprobsData: Array<{token: string, logprob: number}>|null }>}
+ */
+async function complete(config, prompt, operationType, options = {}) {
+  const endpoint = config.endpoint || 'http://localhost:11434';
+  const model = config.model || 'qwen2.5-coder:7b';
+  const timeoutMs = config.timeout_ms || 3000;
+  const maxRetries = config.max_retries != null ? config.max_retries : 1;
+  const numCtx = (config.advanced && config.advanced.num_ctx) || 4096;
+  const keepAlive = (config.advanced && config.advanced.keep_alive) || '30m';
+  const maxFailures = (config.advanced && config.advanced.disable_after_failures) || 3;
+
+  if (isDisabled(operationType, maxFailures)) {
+    const err = new Error('Circuit open for operation: ' + operationType);
+    err.type = 'circuit_open';
+    throw err;
+  }
+
+  const bodyObj = {
+    model,
+    messages: [
+      {
+        role: 'system',
+        content:
+          'You are a precise classification assistant. Always respond with valid JSON only. No explanations outside the JSON.'
+      },
+      { role: 'user', content: prompt }
+    ],
+    response_format: { type: 'json_object' },
+    temperature: 0.1,
+    max_tokens: 200,
+    keep_alive: keepAlive,
+    num_ctx: numCtx
+  };
+  if (options.logprobs === true) {
+    bodyObj.logprobs = true;
+    bodyObj.top_logprobs = 3;
+  }
+  const body = JSON.stringify(bodyObj);
+
+  const url = endpoint + '/v1/chat/completions';
+  const totalAttempts = maxRetries + 1;
+
+  let lastErr;
+  for (let attempt = 0; attempt < totalAttempts; attempt++) {
+    const start = performance.now();
+    try {
+      const res = await fetch(url, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body,
+        signal: AbortSignal.timeout(timeoutMs)
+      });
+
+      if (!res.ok) {
+        const errText = await res.text().catch(() => '');
+        throw new Error('HTTP ' + res.status + ': ' + errText);
+      }
+
+      const json = await res.json();
+      const content = json.choices[0].message.content;
+      const completionTokens = (json.usage && json.usage.completion_tokens) || 0;
+      const latency_ms = performance.now() - start;
+      const logprobsData = (options.logprobs && json.choices[0].logprobs)
+        ? json.choices[0].logprobs.content
+        : null;
+
+      return { content, latency_ms, tokens: completionTokens, logprobsData };
+    } catch (err) {
+      lastErr = err;
+      const isConnRefused =
+        (err.cause && err.cause.code === 'ECONNREFUSED') ||
+        (err.message && err.message.includes('ECONNREFUSED'));
+
+      if (isConnRefused) {
+        // Server not running — no point retrying
+        recordFailure(operationType, maxFailures);
+        throw err;
+      }
+
+      const isTimeout = err.name === 'TimeoutError' || err.name === 'AbortError';
+      const isHttpError = err.message && err.message.startsWith('HTTP ');
+
+      if ((isTimeout || isHttpError) && attempt < totalAttempts - 1) {
+        // Retry
+        continue;
+      }
+
+      // Final attempt or non-retryable error
+      if (attempt === totalAttempts - 1) {
+        recordFailure(operationType, maxFailures);
+      } else {
+        recordFailure(operationType, maxFailures);
+      }
+      throw err;
+    }
+  }
+
+  // Should not reach here, but guard anyway
+  recordFailure(operationType, maxFailures);
+  throw lastErr;
+}
+
+module.exports = { tryParseJSON, categorizeError, isDisabled, recordFailure, resetCircuit, complete };

package/plugins/pbr/scripts/local-llm/health.js

@@ -0,0 +1,217 @@
+/* global fetch, AbortSignal */
+'use strict';
+
+const WARMUP_TIMEOUT_MS = 120000;
+
+/**
+ * Merges the raw local_llm config block with defaults.
+ * @param {object|undefined} rawConfig
+ * @returns {object} Fully-defaulted local_llm config
+ */
+function resolveConfig(rawConfig) {
+  return {
+    enabled: rawConfig != null && rawConfig.enabled != null ? rawConfig.enabled : false,
+    provider: (rawConfig && rawConfig.provider) || 'ollama',
+    endpoint: (rawConfig && rawConfig.endpoint) || 'http://localhost:11434',
+    model: (rawConfig && rawConfig.model) || 'qwen2.5-coder:7b',
+    timeout_ms: (rawConfig && rawConfig.timeout_ms) || 3000,
+    max_retries: rawConfig != null && rawConfig.max_retries != null ? rawConfig.max_retries : 1,
+    fallback: (rawConfig && rawConfig.fallback) || 'frontier',
+    routing_strategy: (rawConfig && rawConfig.routing_strategy) || 'local_first',
+    features: Object.assign(
+      {
+        artifact_classification: true,
+        task_validation: true,
+        plan_adequacy: false,
+        gap_detection: false,
+        context_summarization: false,
+        source_scoring: false
+      },
+      (rawConfig && rawConfig.features) || {}
+    ),
+    metrics: Object.assign(
+      {
+        enabled: true,
+        log_file: '.planning/logs/local-llm-metrics.jsonl',
+        show_session_summary: true,
+        frontier_token_rate: 3.0
+      },
+      (rawConfig && rawConfig.metrics) || {}
+    ),
+    advanced: Object.assign(
+      {
+        confidence_threshold: 0.9,
+        max_input_tokens: 2000,
+        keep_alive: '30m',
+        num_ctx: 4096,
+        disable_after_failures: 3,
+        shadow_mode: false
+      },
+      (rawConfig && rawConfig.advanced) || {}
+    )
+  };
+}
+
+/**
+ * Checks availability of the configured Ollama instance and model.
+ * Always resolves — never rejects.
+ * @param {object} config - resolved config from resolveConfig()
+ * @returns {Promise<object>} Structured health status
+ */
+async function checkHealth(config) {
+  try {
+    if (!config.enabled) {
+      return { available: false, reason: 'disabled', model: null, version: null };
+    }
+
+    const timeoutShort = 3000;
+    const timeoutModel = 5000;
+
+    // Step 1 — Check server reachable
+    try {
+      const res = await fetch(config.endpoint + '/', {
+        signal: AbortSignal.timeout(timeoutShort)
+      });
+      const body = await res.text().catch(() => '');
+      if (!body.includes('Ollama')) {
+        return {
+          available: false,
+          reason: 'not_running',
+          detail: 'Ollama is not running. Start with: ollama serve',
+          model: null,
+          version: null
+        };
+      }
+    } catch (err) {
+      const isConnRefused =
+        (err.cause && err.cause.code === 'ECONNREFUSED') ||
+        (err.message && err.message.includes('ECONNREFUSED'));
+      const isTimeout = err.name === 'TimeoutError' || err.name === 'AbortError';
+      if (isConnRefused || isTimeout) {
+        return {
+          available: false,
+          reason: 'not_running',
+          detail: 'Ollama is not running. Start with: ollama serve',
+          model: null,
+          version: null
+        };
+      }
+      throw err;
+    }
+
+    // Step 2 — Check version (non-fatal)
+    let version = null;
+    try {
+      const res = await fetch(config.endpoint + '/api/version', {
+        signal: AbortSignal.timeout(timeoutShort)
+      });
+      const data = await res.json();
+      version = data.version || null;
+    } catch (_) {
+      version = null;
+    }
+
+    // Step 3 — Check model available
+    try {
+      const res = await fetch(config.endpoint + '/v1/models', {
+        signal: AbortSignal.timeout(timeoutModel)
+      });
+      const data = await res.json();
+      const modelList = (data.data || []).map((m) => m.id || '');
+      const baseModel = config.model.split(':')[0];
+      const found = modelList.some((m) => m.startsWith(baseModel));
+      if (!found) {
+        return {
+          available: false,
+          reason: 'model_missing',
+          detail: 'Run: ollama pull ' + config.model,
+          model: null,
+          version
+        };
+      }
+    } catch (_err) {
+      return {
+        available: false,
+        reason: 'model_missing',
+        detail: 'Run: ollama pull ' + config.model,
+        model: null,
+        version
+      };
+    }
+
+    // Step 4 — Detect GPU error (sleep/wake CUDA bug)
+    let warm = false;
+    try {
+      const res = await fetch(config.endpoint + '/v1/chat/completions', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          model: config.model,
+          messages: [{ role: 'user', content: '{"status":"ok"}' }],
+          max_tokens: 10,
+          num_ctx: 512
+        }),
+        signal: AbortSignal.timeout(timeoutModel)
+      });
+
+      if (!res.ok) {
+        const errBody = await res.text().catch(() => '');
+        if (res.status === 500 && (errBody.includes('GPU') || errBody.includes('CUDA'))) {
+          return {
+            available: false,
+            reason: 'gpu_error',
+            detail: 'GPU error detected. Restart Ollama: ollama serve',
+            model: config.model,
+            version
+          };
+        }
+        // Non-GPU HTTP error — treat as available but cold
+        warm = false;
+      } else {
+        warm = true;
+      }
+    } catch (err) {
+      const isTimeout = err.name === 'TimeoutError' || err.name === 'AbortError';
+      if (isTimeout) {
+        warm = false; // cold start in progress — skip, don't block
+      } else {
+        warm = false;
+      }
+    }
+
+    return { available: true, warm, reason: 'ok', model: config.model, version };
+  } catch (_err) {
+    return {
+      available: false,
+      reason: 'unknown_error',
+      detail: _err.message,
+      model: null,
+      version: null
+    };
+  }
+}
+
+/**
+ * Fire-and-forget warm-up request. Callers should NOT await this.
+ * @param {object} config - resolved config from resolveConfig()
+ */
+async function warmUp(config) {
+  try {
+    await fetch(config.endpoint + '/v1/chat/completions', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        model: config.model,
+        messages: [{ role: 'user', content: '{"status":"ready"}' }],
+        max_tokens: 10,
+        num_ctx: 512,
+        keep_alive: config.advanced.keep_alive
+      }),
+      signal: AbortSignal.timeout(WARMUP_TIMEOUT_MS)
+    });
+  } catch (_) {
+    // Swallow all errors silently — fire and forget
+  }
+}
+
+module.exports = { resolveConfig, checkHealth, warmUp };

package/plugins/pbr/scripts/local-llm/metrics.js

@@ -0,0 +1,252 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const MAX_ENTRIES = 200;
+
+/**
+ * Appends a metric entry to the JSONL log file.
+ * Rotates to keep only the last 200 entries when the log exceeds MAX_ENTRIES lines.
+ * Swallows all errors silently — metrics must never crash hooks.
+ *
+ * @param {string} planningDir - path to the .planning directory
+ * @param {object} entry - metric entry object
+ * @param {string} entry.session_id
+ * @param {string} entry.timestamp
+ * @param {string} entry.operation
+ * @param {string} entry.model
+ * @param {number} entry.latency_ms
+ * @param {number} entry.tokens_used_local
+ * @param {number} entry.tokens_saved_frontier
+ * @param {string} entry.result
+ * @param {boolean} entry.fallback_used
+ * @param {number} entry.confidence
+ */
+function logMetric(planningDir, entry) {
+  try {
+    const logsDir = path.join(planningDir, 'logs');
+    const logFile = path.join(logsDir, 'local-llm-metrics.jsonl');
+
+    fs.mkdirSync(logsDir, { recursive: true });
+    fs.appendFileSync(logFile, JSON.stringify(entry) + '\n', 'utf8');
+
+    // Rotate if over MAX_ENTRIES
+    try {
+      const contents = fs.readFileSync(logFile, 'utf8');
+      const lines = contents.split(/\r?\n/).filter((l) => l.trim() !== '');
+      if (lines.length > MAX_ENTRIES) {
+        const trimmed = lines.slice(lines.length - MAX_ENTRIES);
+        fs.writeFileSync(logFile, trimmed.join('\n') + '\n', 'utf8');
+      }
+    } catch (_) {
+      // Rotation failure is non-fatal
+    }
+  } catch (_) {
+    // Swallow all errors silently
+  }
+}
+
+/**
+ * Reads metric entries from the JSONL log that occurred at or after sessionStartTime.
+ *
+ * @param {string} planningDir - path to the .planning directory
+ * @param {string|Date} sessionStartTime - ISO string or Date
+ * @returns {object[]} Array of matching metric entry objects
+ */
+function readSessionMetrics(planningDir, sessionStartTime) {
+  try {
+    const logFile = path.join(planningDir, 'logs', 'local-llm-metrics.jsonl');
+    const contents = fs.readFileSync(logFile, 'utf8');
+    const startMs = new Date(sessionStartTime).getTime();
+
+    return contents
+      .split(/\r?\n/)
+      .filter((l) => l.trim() !== '')
+      .map((l) => {
+        try {
+          return JSON.parse(l);
+        } catch (_) {
+          return null;
+        }
+      })
+      .filter((e) => e !== null)
+      .filter((e) => {
+        try {
+          return new Date(e.timestamp).getTime() >= startMs;
+        } catch (_) {
+          return false;
+        }
+      });
+  } catch (_) {
+    return [];
+  }
+}
+
+/**
+ * Summarizes an array of metric entries.
+ *
+ * @param {object[]} entries
+ * @param {number} [frontierTokenRate=3.0] - cost per million tokens in USD
+ * @returns {{ total_calls: number, fallback_count: number, avg_latency_ms: number, tokens_saved: number, cost_saved_usd: number }}
+ */
+function summarizeMetrics(entries, frontierTokenRate) {
+  if (!entries || entries.length === 0) {
+    return {
+      total_calls: 0,
+      fallback_count: 0,
+      avg_latency_ms: 0,
+      tokens_saved: 0,
+      cost_saved_usd: 0
+    };
+  }
+
+  const rate = frontierTokenRate != null ? frontierTokenRate : 3.0;
+  const total_calls = entries.length;
+  const fallback_count = entries.filter((e) => e.fallback_used).length;
+  const totalLatency = entries.reduce((sum, e) => sum + (e.latency_ms || 0), 0);
+  const avg_latency_ms = total_calls > 0 ? totalLatency / total_calls : 0;
+  const tokens_saved = entries.reduce((sum, e) => sum + (e.tokens_saved_frontier || 0), 0);
+  const cost_saved_usd = tokens_saved * (rate / 1_000_000);
+
+  return { total_calls, fallback_count, avg_latency_ms, tokens_saved, cost_saved_usd };
+}
+
+/**
+ * Computes lifetime aggregate metrics by reading all entries from the JSONL log.
+ * No date filter — reads everything. Adds a by_operation breakdown keyed by operation.
+ *
+ * @param {string} planningDir - path to the .planning directory
+ * @param {number} [frontierTokenRate=3.0] - cost per million tokens in USD
+ * @returns {{ total_calls: number, fallback_count: number, avg_latency_ms: number, tokens_saved: number, cost_saved_usd: number, by_operation: object }}
+ */
+function computeLifetimeMetrics(planningDir, frontierTokenRate) {
+  const zero = {
+    total_calls: 0,
+    fallback_count: 0,
+    avg_latency_ms: 0,
+    tokens_saved: 0,
+    cost_saved_usd: 0,
+    by_operation: {}
+  };
+
+  try {
+    const logFile = path.join(planningDir, 'logs', 'local-llm-metrics.jsonl');
+    let contents;
+    try {
+      contents = fs.readFileSync(logFile, 'utf8');
+    } catch (_) {
+      return zero;
+    }
+
+    const entries = contents
+      .split(/\r?\n/)
+      .filter((l) => l.trim() !== '')
+      .map((l) => {
+        try {
+          return JSON.parse(l);
+        } catch (_) {
+          return null;
+        }
+      })
+      .filter((e) => e !== null);
+
+    if (entries.length === 0) return zero;
+
+    const rate = frontierTokenRate != null ? frontierTokenRate : 3.0;
+    const total_calls = entries.length;
+    const fallback_count = entries.filter((e) => e.fallback_used).length;
+    const totalLatency = entries.reduce((sum, e) => sum + (e.latency_ms || 0), 0);
+    const avg_latency_ms = total_calls > 0 ? totalLatency / total_calls : 0;
+    const tokens_saved = entries.reduce((sum, e) => sum + (e.tokens_saved_frontier || 0), 0);
+    const cost_saved_usd = tokens_saved * (rate / 1_000_000);
+
+    const by_operation = {};
+    for (const e of entries) {
+      const op = e.operation || 'unknown';
+      if (!by_operation[op]) {
+        by_operation[op] = { calls: 0, fallbacks: 0, tokens_saved: 0 };
+      }
+      by_operation[op].calls += 1;
+      if (e.fallback_used) by_operation[op].fallbacks += 1;
+      by_operation[op].tokens_saved += e.tokens_saved_frontier || 0;
+    }
+
+    return { total_calls, fallback_count, avg_latency_ms, tokens_saved, cost_saved_usd, by_operation };
+  } catch (_) {
+    return zero;
+  }
+}
+
+/**
+ * Formats a metrics aggregate (output of summarizeMetrics) into a human-readable one-liner.
+ *
+ * @param {object} summary - output of summarizeMetrics()
+ * @param {string} [model] - optional model name
+ * @returns {string}
+ */
+function formatSessionSummary(summary, model) {
+  if (!summary || summary.total_calls === 0) {
+    return 'Local LLM: no calls this session';
+  }
+
+  const { total_calls, fallback_count, avg_latency_ms, tokens_saved, cost_saved_usd } = summary;
+
+  let costStr = '';
+  if (cost_saved_usd > 0) {
+    costStr = ` ($${cost_saved_usd.toFixed(2)})`;
+  }
+
+  let fallbackStr = '';
+  if (fallback_count > 0) {
+    fallbackStr = `, ${fallback_count} fallback(s)`;
+  }
+
+  let modelStr = '';
+  if (model) {
+    modelStr = ` [${model}]`;
+  }
+
+  const avgMs = Math.round(avg_latency_ms);
+
+  return `Local LLM: ${total_calls} calls, ~${tokens_saved} frontier tokens saved${costStr}, avg ${avgMs}ms${fallbackStr}${modelStr}`;
+}
+
+/**
+ * Appends a shadow comparison entry to the shadow JSONL log file.
+ * Rotates to keep only the last 200 entries. Swallows all errors silently.
+ *
+ * @param {string} planningDir - path to the .planning directory
+ * @param {object} entry - shadow comparison entry object
+ * @param {string} entry.timestamp
+ * @param {string} entry.operation
+ * @param {string} entry.session_id
+ * @param {boolean} entry.agrees
+ * @param {string|null} entry.local_result
+ * @param {string} entry.frontier_result
+ */
+function logAgreement(planningDir, entry) {
+  try {
+    const logsDir = path.join(planningDir, 'logs');
+    const logFile = path.join(logsDir, 'local-llm-shadow.jsonl');
+
+    fs.mkdirSync(logsDir, { recursive: true });
+    fs.appendFileSync(logFile, JSON.stringify(entry) + '\n', 'utf8');
+
+    // Rotate if over MAX_ENTRIES
+    try {
+      const contents = fs.readFileSync(logFile, 'utf8');
+      const lines = contents.split(/\r?\n/).filter((l) => l.trim() !== '');
+      if (lines.length > MAX_ENTRIES) {
+        const trimmed = lines.slice(lines.length - MAX_ENTRIES);
+        fs.writeFileSync(logFile, trimmed.join('\n') + '\n', 'utf8');
+      }
+    } catch (_) {
+      // Rotation failure is non-fatal
+    }
+  } catch (_) {
+    // Swallow all errors silently
+  }
+}
+
+module.exports = { logMetric, readSessionMetrics, summarizeMetrics, computeLifetimeMetrics, formatSessionSummary, logAgreement };