agentboss 0.1.0

package/server/llm/cli-runner.js

@@ -0,0 +1,249 @@
+/**
+ * LLM judge runner — spawns a local AI CLI for evaluation tasks.
+ *
+ * Detection order (first one found wins):
+ *   1. `opencode run -p "<prompt>"`
+ *   2. `claude -p "<prompt>"`
+ *
+ * Returns parsed JSON.  Failures (CLI missing / timeout / non-JSON
+ * output) resolve to `null` so callers can fall back to rule-based
+ * heuristics.  Never throws.
+ *
+ * @author Felix
+ */
+
+'use strict';
+
+const { spawn } = require('child_process');
+const { JUDGE_SENTINEL } = require('./judge-prompts');
+
+/**
+ * Prepend the JUDGE_SENTINEL to the prompt if it isn't already the very
+ * first line.  This is the last-line defence that guarantees *every*
+ * LLM call originating from aboss is recognisable when its session
+ * later gets re-imported by the ETL (see server/etl/judge-filter.js).
+ *
+ * Callers (e.g. buildE1Prompt / buildO1Prompt) already prepend the
+ * sentinel, but enforcing it here means any future caller — or any
+ * accidentally-omitted sentinel — still produces a tagged session
+ * rather than polluting the user's own work.
+ */
+function ensureSentinel(prompt) {
+  if (typeof prompt !== 'string') return prompt;
+  if (prompt.startsWith(JUDGE_SENTINEL)) return prompt;
+  return `${JUDGE_SENTINEL}（内部标记，忽略本行）\n${prompt}`;
+}
+
+// ---------------------------------------------------------------------------
+//  Detection
+// ---------------------------------------------------------------------------
+
+/**
+ * CLI candidates.
+ *
+ * `argv` builds the command-line args.  When `stdinPrompt: true`, the
+ * prompt is fed on STDIN instead of being inlined into argv — this is
+ * essential on Windows where the command-line cap is ~8 KB and our
+ * judge prompts run 10 KB+.
+ */
+const CANDIDATES = [
+  // opencode reads stdin when no positional arg is given (after `run`)
+  { name: 'opencode', bin: 'opencode', argv: () => ['run'],   stdinPrompt: true },
+  // claude -p reads stdin when -p is used without an inline prompt
+  { name: 'claude',   bin: 'claude',   argv: () => ['-p'],    stdinPrompt: true },
+];
+
+let _cachedCli = undefined; // null = detected none; obj = found
+
+/**
+ * Detect which CLI is available.  Tries `bin --version` for each candidate.
+ * Caches the result for the process lifetime.
+ *
+ * @returns {Promise<{name:string, bin:string, argv:Function}|null>}
+ */
+async function detectAvailableCli() {
+  if (_cachedCli !== undefined) return _cachedCli;
+  for (const c of CANDIDATES) {
+    if (await canSpawn(c.bin)) {
+      _cachedCli = c;
+      return c;
+    }
+  }
+  _cachedCli = null;
+  return null;
+}
+
+/** Reset the detection cache.  Mostly useful in tests / settings reload. */
+function _resetCache() { _cachedCli = undefined; }
+
+/**
+ * Try to spawn `bin --version`.  Resolves true on exit code 0.  Cross-
+ * platform: on Windows `bin` is resolved via PATH automatically by spawn.
+ */
+function canSpawn(bin) {
+  return new Promise((resolve) => {
+    let resolved = false;
+    const settle = (v) => { if (!resolved) { resolved = true; resolve(v); } };
+
+    try {
+      const proc = spawn(bin, ['--version'], {
+        stdio: 'ignore',
+        shell: process.platform === 'win32',
+      });
+      proc.on('error', () => settle(false));
+      proc.on('exit', (code) => settle(code === 0));
+      // hard timeout
+      setTimeout(() => { try { proc.kill('SIGKILL'); } catch {} settle(false); }, 5000);
+    } catch {
+      settle(false);
+    }
+  });
+}
+
+// ---------------------------------------------------------------------------
+//  Runner
+// ---------------------------------------------------------------------------
+
+/**
+ * Spawn the chosen CLI with the prompt, capture stdout, and try to parse
+ * it as JSON.  The caller's prompt should *demand* JSON output.
+ *
+ * Options:
+ *   timeoutMs (default 30_000)
+ *   maxBytes  (default 256 KB) — guard against runaway output
+ *
+ * Resolves:
+ *   { ok: true, data: any, raw: string, cli: 'opencode'|'claude' }
+ *   { ok: false, reason: 'no-cli' | 'timeout' | 'exit-non-zero' | 'bad-json' | 'spawn-error', raw?: string, error?: string }
+ *
+ * @param {Object} opts
+ * @returns {Promise<Object>}
+ */
+async function runJudge(opts = {}) {
+  const { prompt: rawPrompt, timeoutMs = 30_000, maxBytes = 256 * 1024 } = opts;
+  if (!rawPrompt || typeof rawPrompt !== 'string') {
+    return { ok: false, reason: 'no-prompt' };
+  }
+  // Stamp the sentinel onto every outbound prompt so the ETL can later
+  // recognise and discard the session this CLI call will create.
+  const prompt = ensureSentinel(rawPrompt);
+
+  const cli = await detectAvailableCli();
+  if (!cli) return { ok: false, reason: 'no-cli' };
+
+  return new Promise((resolve) => {
+    let resolved = false;
+    const settle = (v) => { if (!resolved) { resolved = true; resolve(v); } };
+
+    let proc;
+    try {
+      const useStdin = cli.stdinPrompt === true;
+      proc = spawn(cli.bin, cli.argv(prompt), {
+        stdio: [useStdin ? 'pipe' : 'ignore', 'pipe', 'pipe'],
+        shell: process.platform === 'win32',
+      });
+      if (useStdin && proc.stdin) {
+        proc.stdin.on('error', () => {}); // EPIPE if CLI exits early
+        proc.stdin.end(prompt, 'utf8');
+      }
+    } catch (err) {
+      return settle({ ok: false, reason: 'spawn-error', error: err.message });
+    }
+
+    let stdout = '';
+    let stderr = '';
+    let truncated = false;
+
+    proc.stdout.on('data', (chunk) => {
+      if (truncated) return;
+      stdout += chunk.toString('utf8');
+      if (stdout.length > maxBytes) {
+        stdout = stdout.slice(0, maxBytes);
+        truncated = true;
+        try { proc.kill('SIGKILL'); } catch {}
+      }
+    });
+    proc.stderr.on('data', (chunk) => { stderr += chunk.toString('utf8'); });
+
+    proc.on('error', (err) => settle({ ok: false, reason: 'spawn-error', error: err.message }));
+
+    proc.on('exit', (code) => {
+      if (code !== 0 && !truncated) {
+        return settle({ ok: false, reason: 'exit-non-zero', raw: stdout, error: stderr.slice(0, 500) });
+      }
+      const parsed = extractJson(stdout);
+      if (parsed === undefined) {
+        return settle({ ok: false, reason: 'bad-json', raw: stdout.slice(0, 500) });
+      }
+      settle({ ok: true, data: parsed, raw: stdout, cli: cli.name });
+    });
+
+    const t = setTimeout(() => {
+      try { proc.kill('SIGKILL'); } catch {}
+      settle({ ok: false, reason: 'timeout' });
+    }, timeoutMs);
+    proc.on('exit', () => clearTimeout(t));
+  });
+}
+
+/**
+ * Try to find a JSON value in raw stdout.  Tolerates leading log lines
+ * by scanning for the first { or [.  Returns the parsed value or
+ * undefined on failure.
+ */
+function extractJson(raw) {
+  if (!raw) return undefined;
+  // common case: stdout is pure JSON
+  const trimmed = raw.trim();
+  try { return JSON.parse(trimmed); } catch {}
+  // fall back: find first { or [
+  const i1 = trimmed.indexOf('{');
+  const i2 = trimmed.indexOf('[');
+  let start = -1;
+  if (i1 >= 0 && i2 >= 0) start = Math.min(i1, i2);
+  else if (i1 >= 0) start = i1;
+  else if (i2 >= 0) start = i2;
+  if (start < 0) return undefined;
+
+  // find matching last brace/bracket of the same kind
+  const open = trimmed[start];
+  const close = open === '{' ? '}' : ']';
+  const end = trimmed.lastIndexOf(close);
+  if (end < start) return undefined;
+  try { return JSON.parse(trimmed.slice(start, end + 1)); } catch {}
+  return undefined;
+}
+
+// ---------------------------------------------------------------------------
+//  Concurrency guard
+// ---------------------------------------------------------------------------
+
+let _inFlight = 0;
+const _waiters = [];
+const MAX_CONCURRENT = 2;
+
+/** Run `fn` under a 2-wide semaphore so we don't fork-bomb the CLI. */
+function withSlot(fn) {
+  return new Promise((resolve) => {
+    const start = async () => {
+      _inFlight++;
+      try { resolve(await fn()); }
+      finally {
+        _inFlight--;
+        const next = _waiters.shift();
+        if (next) next();
+      }
+    };
+    if (_inFlight < MAX_CONCURRENT) start();
+    else _waiters.push(start);
+  });
+}
+
+module.exports = {
+  detectAvailableCli,
+  runJudge,
+  withSlot,
+  // exported for tests
+  _resetCache,
+  extractJson,
+};

package/server/llm/judge-prompts.js

@@ -0,0 +1,179 @@
+/**
+ * Prompt templates for the LLM-judge runs used by E1 and O1.
+ *
+ * Each function returns a STRING prompt that demands strict JSON output.
+ * The runner enforces a 30s timeout and ~256 KB cap; we keep the input
+ * compact (only last N messages, truncated) so the CLI doesn't choke.
+ *
+ * @author Felix
+ */
+
+'use strict';
+
+/**
+ * Bump whenever the prompt output contract changes — cached judge
+ * results stamped with an older version are ignored and re-judged.
+ * v2: added per-field `details` scoring evidence.
+ * v4: dropped H1.reframe sub-indicator.
+ */
+const PROMPT_VERSION = 4;
+
+/**
+ * First line of every judge prompt.  The judge CLIs (opencode / claude)
+ * log each call as a session in their own data stores; the ETL uses this
+ * marker to recognise and skip those sessions so they don't get imported
+ * back as the user's own work (which would create a feedback loop).
+ */
+const JUDGE_SENTINEL = '[ABOSS-JUDGE]';
+
+const MAX_MESSAGES = 30;
+const MAX_LEN_PER_MSG = 600;
+
+/** Build a short, role-tagged transcript fragment. */
+function buildTranscript(messages) {
+  const slice = messages.slice(-MAX_MESSAGES);
+  return slice
+    .filter((m) => m.text)
+    .map((m) => {
+      const role = (m.role || '?').toUpperCase().padEnd(9);
+      const text = m.text.length > MAX_LEN_PER_MSG
+        ? m.text.slice(0, MAX_LEN_PER_MSG) + '…[truncated]'
+        : m.text;
+      return `[${role}] ${text}`;
+    })
+    .join('\n---\n');
+}
+
+/**
+ * E1 — AI Knowledge Coverage judge prompt.
+ *
+ * Asks the judge to score three dimensions in [0, 1].  Hands back a
+ * strict JSON object.
+ */
+function buildE1Prompt(messages, meta = {}) {
+  const transcript = buildTranscript(messages);
+  return `${JUDGE_SENTINEL}（内部标记，忽略本行）
+你是一名严格的 AI 协作审计员。下面是一段开发者与 AI 编程助手的对话片段（按时间序）。
+请评估这位 AI 助手在该会话中的"知识覆盖"表现，返回一个严格 JSON 对象，不要任何额外文字、解释或 markdown 代码块。
+
+字段定义（值域均为 0.0 - 1.0 之间的浮点数）：
+- domain_errors: 助手输出中存在领域错误（错误 API、概念混淆、虚构函数）的比例。越接近 0 越好。
+- staleness:     助手使用已废弃 / 已移除技术的次数，归一化（0=没有，1=非常严重）。越接近 0 越好。
+- best_practice: 助手输出符合当前最佳实践的比例。越接近 1 越好。
+
+同时必须在 details 中为每个字段给出打分依据：引用对话中的具体表现（哪条消息、什么内容），
+说明为什么打这个数值而不是更高或更低（例如"全程未发现虚构 API，仅第 12 条消息混淆了 X 与 Y，故 0.05 而非 0"）。
+没有证据就明确写"未发现相关证据"。禁止空泛措辞。
+
+如果对话太短或无法判断，对应数值字段返回 null，details 中说明无法判断的原因。
+
+会话上下文：
+- 模型：${meta.model || '未知'}
+- 项目：${meta.project || '未知'}
+- 任务难度：${meta.difficulty || '未知'} (1=琐碎 4=重型)
+
+对话片段：
+${transcript}
+
+只输出 JSON，形如：
+{"domain_errors": 0.1, "staleness": 0.0, "best_practice": 0.85,
+ "details": {"domain_errors": "打分依据，引用具体消息", "staleness": "…", "best_practice": "…"},
+ "rationale": "一句话总评"}`;
+}
+
+/**
+ * O1 — Output Quality judge prompt.
+ */
+function buildO1Prompt(messages, meta = {}) {
+  const transcript = buildTranscript(messages);
+  return `${JUDGE_SENTINEL}（内部标记，忽略本行）
+你是一名严格的代码与产出审计员。下面是一段开发者与 AI 编程助手的对话片段。
+请评估这位 AI 在该会话中的"输出质量"，返回严格 JSON 对象（无任何额外文字 / markdown）。
+
+字段（0.0 - 1.0 浮点）：
+- first_take:   助手输出可被一次采纳（无需修改）的比例。
+- code_style:   助手产出的代码规范度（命名、格式、注释、可读性）。
+- completeness: 助手是否考虑边界条件、错误处理、测试等完备性。
+
+同时必须在 details 中为每个字段给出打分依据：引用对话中的具体表现，
+说明为什么是这个数值而不是更高或更低（例如"8 次产出中 2 次被用户要求返工，故 first_take=0.75 而非 0.9"）。
+没有证据就明确写"未发现相关证据"。禁止空泛措辞。
+
+无法判断的字段返回 null，details 中说明原因。
+
+会话上下文：模型 ${meta.model || '未知'}，难度 ${meta.difficulty || '未知'}/4。
+
+对话片段：
+${transcript}
+
+只输出 JSON：
+{"first_take": 0.75, "code_style": 0.8, "completeness": 0.6,
+ "details": {"first_take": "打分依据，引用具体消息", "code_style": "…", "completeness": "…"},
+ "rationale": "一句话总评"}`;
+}
+
+/**
+ * Per-dimension L1–L4 anchors.  Kept compact: one line per level so the
+ * full rubric stays well under the CLI prompt budget.
+ */
+const RUBRIC = `
+H1 立意（把模糊需求收敛成可执行的精确问题的功力）
+  clarity 初始指令清晰度 · converge 收敛效率 · drift 方向稳定性
+  L4 开局即给出目标+约束+验收，几乎无需追问；L3 信息基本充分，少量澄清；
+  L2 需多轮补全；L1 一句话甩任务、反复改方向。
+H2 判断（不盲从、敢质疑、敢推翻 vs 橡皮图章式照单全收）
+  challenge 合理质疑 · override 该推翻时推翻 · accept_rate 采纳前是否有判断
+  L4 在该质疑处质疑、在合理处高效采纳，质疑有依据；L3 大体审视偶有盲从；
+  L2 多数直接采纳少量质疑；L1 几乎全程"好的/继续"式盲从。
+E1 知识（AI 对该技术栈的掌握）
+  domain_errors 领域错误（越少越好）· staleness 过时引用（越少越好）· best_practice 最佳实践
+  L4 无错误、无过时、全程最佳实践；L3 偶有小瑕疵；L2 多处问题；L1 频繁错误/过时。
+O1 产出（结果是否真的可用）
+  first_take 一次采纳 · code_style 代码规范 · completeness 边界/异常/测试完备
+  L4 几乎一次过且规范完备；L3 小修即可；L2 需明显返工；L1 大量返工或缺失完备性。
+`;
+
+/**
+ * Build the consolidated session-judge prompt.  Demands a single strict
+ * JSON object scoring every semantic sub-indicator with level (1-4 or
+ * null), confidence (0-1) and evidence (must cite specific messages).
+ *
+ * @param {Array<{role:string,text:string}>} messages
+ * @param {{difficulty?:number, model?:string, project?:string}} meta
+ * @returns {string}
+ */
+function buildSessionJudgePrompt(messages, meta = {}) {
+  const transcript = buildTranscript(messages);
+  return `${JUDGE_SENTINEL}（内部标记，忽略本行）
+你是一名严格、客观的"人机协作"审计员。下面是一段开发者与 AI 编程助手的对话片段（按时间序）。
+请按下面的 rubric，对四个维度的每个子指标给出 L1–L4 等级，返回**单个严格 JSON 对象**，
+不要任何额外文字、解释或 markdown 代码块。
+
+评分 rubric（难度越高合格线越宽松，本会话难度见下文）：
+${RUBRIC}
+
+每个子指标返回三元组：
+- level: 整数 1–4；证据不足或对话太短无法判断时返回 null（不要硬给低分）。
+- confidence: 0.0–1.0，你对该 level 的把握。
+- evidence: 打分依据，必须引用对话中的具体表现（哪条消息、什么内容），说明为何是这个等级
+  而非更高/更低。没有证据就写"未发现相关证据"。禁止空泛措辞。
+
+客观性要求（必须遵守）：
+1. 不因 AI 输出更长、更礼貌、更啰嗦而加分，只看实质内容。
+2. 证据不足或会话过短的子指标必须返回 level=null，而不是给低分。
+3. 任务难度低不代表做得差——难度已单独归一，不要二次惩罚简单任务。
+
+会话上下文：模型 ${meta.model || '未知'}，项目 ${meta.project || '未知'}，任务难度 ${meta.difficulty || '未知'}/4（1=琐碎 4=重型）。
+
+对话片段：
+${transcript}
+
+只输出 JSON，形如：
+{"H1":{"clarity":{"level":3,"confidence":0.8,"evidence":"…"},"converge":{"level":4,"confidence":0.9,"evidence":"…"},"drift":{"level":null,"confidence":0,"evidence":"无法判断，原因…"}},
+ "H2":{"challenge":{…},"override":{…},"accept_rate":{…}},
+ "E1":{"domain_errors":{…},"staleness":{…},"best_practice":{…}},
+ "O1":{"first_take":{…},"code_style":{…},"completeness":{…}},
+ "rationale":"一句话总评"}`;
+}
+
+module.exports = { buildE1Prompt, buildO1Prompt, buildSessionJudgePrompt, PROMPT_VERSION, JUDGE_SENTINEL };

package/server/llm/judge.js

@@ -0,0 +1,118 @@
+/**
+ * High-level LLM judge — bridges dimension scorers (E1, O1) to the
+ * cli-runner.  Handles:
+ *   • opt-in via user_settings.enable_llm_judge
+ *   • per-session cache via session_analysis.llm_judge_v2
+ *   • concurrency throttle (cli-runner.withSlot)
+ *   • fall-back signalling so dimension scorers can branch
+ *
+ * @author Felix
+ */
+
+'use strict';
+
+const { runJudge, detectAvailableCli, withSlot } = require('./cli-runner');
+const { buildSessionJudgePrompt, PROMPT_VERSION } = require('./judge-prompts');
+const { queryOne } = require('../db/queries');
+
+// ---------------------------------------------------------------------------
+//  Settings cache
+// ---------------------------------------------------------------------------
+
+let _settingsCache = null;
+let _settingsCacheAt = 0;
+const SETTINGS_TTL_MS = 10_000;
+
+function getSettings(db) {
+  const now = Date.now();
+  if (_settingsCache && now - _settingsCacheAt < SETTINGS_TTL_MS) {
+    return _settingsCache;
+  }
+  const rows = db.exec(
+    "SELECT key, value FROM user_settings WHERE key IN ('enable_llm_judge')"
+  );
+  const out = { enable_llm_judge: false };
+  if (rows[0]) {
+    for (const [k, v] of rows[0].values) {
+      if (k === 'enable_llm_judge') out.enable_llm_judge = String(v) === '1' || String(v).toLowerCase() === 'true';
+    }
+  }
+  _settingsCache = out;
+  _settingsCacheAt = now;
+  return out;
+}
+
+/** Public: force a settings reload (e.g. after PUT /api/settings). */
+function invalidateSettingsCache() {
+  _settingsCache = null;
+}
+
+// ---------------------------------------------------------------------------
+//  Per-session cache
+// ---------------------------------------------------------------------------
+
+/** Return the cached llm_judge_v2 JSON for a session or null. */
+function loadCache(db, sessionId) {
+  const row = queryOne(
+    db,
+    'SELECT llm_judge_v2 FROM session_analysis WHERE session_id = ?',
+    [sessionId]
+  );
+  if (!row || !row.llm_judge_v2) return null;
+  try { return JSON.parse(row.llm_judge_v2); }
+  catch { return null; }
+}
+
+// ---------------------------------------------------------------------------
+//  Public judge functions
+// ---------------------------------------------------------------------------
+
+/**
+ * Consolidated judge — one LLM call scoring H1/H2/E1/O1 for a session.
+ * Returns the parsed payload (stamped with v / msgCount / cli) or null
+ * when disabled, no CLI, or the call fails.  Cached in llm_judge_v2.
+ */
+async function judgeSession(db, session, messages, meta = {}) {
+  const settings = getSettings(db);
+  if (!settings.enable_llm_judge) return null;
+
+  const msgCount = messages.length;
+  const cache = loadCache(db, session.id);
+  if (cache && cache.v === PROMPT_VERSION && cache.msgCount === msgCount) return cache;
+
+  const cli = await detectAvailableCli();
+  if (!cli) return null;
+
+  const prompt = buildSessionJudgePrompt(messages, meta);
+  const result = await withSlot(() => runJudge({ prompt, timeoutMs: 90_000 }));
+  if (!result.ok || !result.data) return null;
+
+  return {
+    ...result.data,
+    v: PROMPT_VERSION,
+    msgCount,
+    cli: result.cli,
+    cachedAt: new Date().toISOString(),
+  };
+}
+
+/**
+ * One-shot pre-flight to surface whether a CLI is configured.  Used by
+ * the Settings page.
+ */
+async function diagnose() {
+  const cli = await detectAvailableCli();
+  return {
+    available: !!cli,
+    name: cli ? cli.name : null,
+  };
+}
+
+module.exports = {
+  judgeSession,
+  diagnose,
+  invalidateSettingsCache,
+  // re-export so callers don't need cli-runner directly
+  detectAvailableCli,
+  PROMPT_VERSION,
+};