agentboss 0.1.0

package/server/llm/advice.js

@@ -0,0 +1,384 @@
+/**
+ * High-level entry point for per-session AI advice generation.
+ *
+ * Pipeline (see docs/superpowers/specs/2026-06-13-session-advice-design.md §3.2):
+ *
+ *   1. settings gate          — reuses enable_llm_judge
+ *   2. cache check            — session_analysis.llm_advice with VERSION + msgCount keys
+ *   3. CLI detection          — opencode > claude
+ *   4. context assembly       — session row + messages + tool-call summary
+ *                                 (NO scores, NO sub-scores — see advice-prompt.js
+ *                                  header for why)
+ *   5. truncateContext        — keep prompt within 80 KB
+ *   6. runJudge under withSlot — 90 s timeout, JSON parsing, sentinel
+ *   7. persist                — UPDATE session_analysis SET llm_advice = ?
+ *
+ * Failures bubble up as `{ ok: false, reason }` so the API layer can map
+ * them onto HTTP codes; nothing in here throws on normal failures.
+ *
+ * @author Felix
+ */
+
+'use strict';
+
+const {
+  detectAvailableCli,
+  runJudge,
+  withSlot,
+} = require('./cli-runner');
+const {
+  ADVICE_PROMPT_VERSION,
+  buildAdvicePrompt,
+  truncateContext,
+} = require('./advice-prompt');
+const {
+  queryAll,
+  queryOne,
+  getSessionById,
+  getMessagesBySession,
+} = require('../db/queries');
+const { saveDb } = require('../db/connection');
+
+// ---------------------------------------------------------------------------
+//  Settings gate — reuse the existing LLM-judge toggle so users don't have
+//  to flip a second switch.  Tiny in-process cache (10 s) avoids hitting
+//  the DB for every button click.
+// ---------------------------------------------------------------------------
+
+let _settingsCache = null;
+let _settingsCacheAt = 0;
+const SETTINGS_TTL_MS = 10_000;
+
+function getSettings(db) {
+  const now = Date.now();
+  if (_settingsCache && now - _settingsCacheAt < SETTINGS_TTL_MS) {
+    return _settingsCache;
+  }
+  const rows = db.exec(
+    "SELECT key, value FROM user_settings WHERE key = 'enable_llm_judge'"
+  );
+  let enable = false;
+  if (rows[0]) {
+    for (const [, v] of rows[0].values) {
+      enable = String(v) === '1' || String(v).toLowerCase() === 'true';
+    }
+  }
+  _settingsCache = { enable_llm_judge: enable };
+  _settingsCacheAt = now;
+  return _settingsCache;
+}
+
+/** Public: drop the settings cache (used by PUT /api/settings). */
+function invalidateAdviceSettingsCache() {
+  _settingsCache = null;
+}
+
+// ---------------------------------------------------------------------------
+//  Cache load / store
+// ---------------------------------------------------------------------------
+
+/**
+ * Return the cached llm_advice JSON for a session, or null if missing /
+ * unparseable.  Does NOT enforce version/msgCount freshness — that's
+ * the caller's job (we want loadAdvice to be a pure read for the API
+ * GET /api/advice/session/:id route as well).
+ */
+function loadAdvice(db, sessionId) {
+  const row = queryOne(
+    db,
+    'SELECT llm_advice FROM session_analysis WHERE session_id = ?',
+    [sessionId]
+  );
+  if (!row || !row.llm_advice) return null;
+  try { return JSON.parse(row.llm_advice); }
+  catch { return null; }
+}
+
+/**
+ * Decide whether a cached payload is still usable.  Mirrors judge.js.
+ */
+function isCacheFresh(cache, currentMsgCount) {
+  return !!cache
+      && cache.v === ADVICE_PROMPT_VERSION
+      && cache.msgCount === currentMsgCount;
+}
+
+/**
+ * Persist the payload to session_analysis.llm_advice.
+ *
+ * We DON'T use upsertSessionAnalysis because that would overwrite every
+ * other column; instead we INSERT a new row if missing, otherwise UPDATE
+ * only this single column.  saveDb() flushes the in-memory sql.js DB to
+ * disk so a crash before the next auto-save still keeps the result.
+ */
+function storeAdvice(db, sessionId, payload) {
+  const exists = queryOne(
+    db,
+    'SELECT 1 FROM session_analysis WHERE session_id = ?',
+    [sessionId]
+  );
+  const json = JSON.stringify(payload);
+  if (exists) {
+    db.run(
+      'UPDATE session_analysis SET llm_advice = ? WHERE session_id = ?',
+      [json, sessionId]
+    );
+  } else {
+    db.run(
+      `INSERT INTO session_analysis (session_id, status, llm_advice)
+       VALUES (?, 'pending', ?)`,
+      [sessionId, json]
+    );
+  }
+  // Best-effort flush; ignore failure (auto-save will catch up).
+  try { saveDb(); } catch { /* noop */ }
+}
+
+// ---------------------------------------------------------------------------
+//  Context assembly
+// ---------------------------------------------------------------------------
+
+/**
+ * Build the AdviceContext object that buildAdvicePrompt() expects.
+ *
+ * Pulls:
+ *   - unified_session row     — meta only (model, duration, tokens, cost)
+ *   - unified_message rows    — full transcript (with .text payload —
+ *                                may be NULL on legacy ETL data)
+ *   - tool-call breakdown     — Top-20 by call count
+ *
+ * Deliberately does NOT pull anything from session_analysis: no scores,
+ * no sub-scores, no difficulty.  See advice-prompt.js header for why
+ * — we want the model to evaluate the conversation directly, not to
+ * launder our rule-based numbers through an LLM.
+ */
+function assembleContext(db, sessionId) {
+  const session = getSessionById(db, sessionId);
+  if (!session) return null;
+
+  const messagesRaw = getMessagesBySession(db, sessionId);
+  const messages = messagesRaw.map((m) => ({
+    role: m.role,
+    text: m.text || '',
+  }));
+  const userCount = messages.filter((m) => m.role === 'user').length;
+  const assistantCount = messages.filter((m) => m.role === 'assistant').length;
+
+  // Tool calls: aggregate to Top-20 with counts and error counts.
+  const toolRows = queryAll(
+    db,
+    `SELECT tool_name,
+            COUNT(*)                                                AS count,
+            SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END)        AS error_count,
+            MAX(COALESCE(target_file, ''))                          AS args_preview
+       FROM unified_tool_call
+      WHERE session_id = ?
+   GROUP BY tool_name
+   ORDER BY count DESC
+      LIMIT 20`,
+    [sessionId]
+  );
+  const toolBreakdown = toolRows.map((t) => ({
+    name: t.tool_name,
+    count: t.count,
+    errorCount: t.error_count,
+    avgDurationMs: 0, // not currently tracked by ETL — keep field for spec parity
+    argsPreview: t.args_preview || '',
+  }));
+
+  return {
+    session: {
+      id: session.id,
+      model: session.model,
+      // difficulty intentionally omitted — it's a derived/computed value
+      // that the model could mistake for a score.  Surface only facts.
+      difficulty: null,
+      durationMinutes: session.duration_minutes || 0,
+      cost: session.cost_usd || 0,
+      tokens: {
+        input: session.tokens_input || 0,
+        output: session.tokens_output || 0,
+        reasoning: session.tokens_reasoning || 0,
+        cacheRead: session.tokens_cache_read || 0,
+        cacheWrite: session.tokens_cache_write || 0,
+      },
+      errorCount: session.error_count || 0,
+      toolCallCount: session.tool_call_count || 0,
+      messageCount: session.message_count || messages.length,
+      userCount,
+      assistantCount,
+      reverted: !!session.reverted,
+    },
+    toolBreakdown,
+    messages,
+    truncated: false,
+    omittedMessages: 0,
+  };
+}
+
+// ---------------------------------------------------------------------------
+//  Public: generateAdvice
+// ---------------------------------------------------------------------------
+
+/**
+ * Produce (or return cached) advice for a session.
+ *
+ * @param {Object}  db
+ * @param {string}  sessionId
+ * @param {Object} [opts]
+ * @param {boolean}[opts.force=false] - bypass the cache and always re-judge
+ * @returns {Promise<
+ *   { ok: true,  data: object, fromCache: boolean }
+ * | { ok: false, reason: 'no-session'|'llm-disabled'|'no-cli'|'timeout'|'bad-json'|'spawn-error'|'exit-non-zero'|'internal',
+ *     error?: string }
+ * >}
+ */
+async function generateAdvice(db, sessionId, opts = {}) {
+  const force = opts.force === true;
+  const log = (...a) => {
+    if (process.env.ABOSS_ADVICE_DEBUG === '1') console.error('[advice]', sessionId, ...a);
+  };
+
+  try {
+    // 1. settings gate
+    const settings = getSettings(db);
+    if (!settings.enable_llm_judge) { log('disabled'); return { ok: false, reason: 'llm-disabled' }; }
+
+    // 2. assemble ctx first — we need msgCount for the freshness check
+    const ctxFull = assembleContext(db, sessionId);
+    if (!ctxFull) { log('no-session'); return { ok: false, reason: 'no-session' }; }
+    const msgCount = ctxFull.messages.length;
+
+    if (!force) {
+      const cached = loadAdvice(db, sessionId);
+      if (isCacheFresh(cached, msgCount)) {
+        log('cache hit');
+        return { ok: true, data: cached, fromCache: true };
+      }
+    }
+
+    // 3. CLI detection
+    const cli = await detectAvailableCli();
+    if (!cli) { log('no CLI'); return { ok: false, reason: 'no-cli' }; }
+
+    // 4–5. truncate + build prompt
+    const ctx = truncateContext(ctxFull);
+    const prompt = buildAdvicePrompt(ctx);
+    log('spawning', cli.name, 'prompt bytes=', prompt.length, 'truncated=', ctx.truncated);
+
+    // 6. run under concurrency slot — 90 s is plenty for prompts up to ~80 KB
+    const result = await withSlot(() => runJudge({ prompt, timeoutMs: 90_000 }));
+    log('result ok=', result.ok, 'reason=', result.reason);
+
+    if (!result.ok) {
+      // Map cli-runner reasons 1:1 — caller (api layer) translates to HTTP code.
+      return { ok: false, reason: result.reason, error: result.error };
+    }
+
+    const normalised = normaliseAdvicePayload(result.data, {
+      msgCount,
+      cli: result.cli,
+      truncated: ctx.truncated,
+      omittedMessages: ctx.omittedMessages,
+    });
+
+    // 7. persist
+    storeAdvice(db, sessionId, normalised);
+
+    return { ok: true, data: normalised, fromCache: false };
+  } catch (err) {
+    log('internal error', err && err.message);
+    return { ok: false, reason: 'internal', error: err && err.message };
+  }
+}
+
+// ---------------------------------------------------------------------------
+//  Payload normalisation
+// ---------------------------------------------------------------------------
+
+const ALL_CATEGORIES = ['cost', 'accuracy', 'context', 'skills', 'workflow'];
+const ALL_SEVERITIES = ['high', 'medium', 'low'];
+const ALL_EXECUTORS  = ['opencode', 'claude', 'manual'];
+const ALL_CWD_HINTS  = ['project_root'];
+
+/**
+ * Be lenient about what the model produces:
+ *  - missing summary/rationale → empty strings (UI hides empties)
+ *  - missing category arrays   → empty arrays
+ *  - unknown severities         → 'low'
+ *  - items without evidence     → dropped (spec §8 rule)
+ *  - extra string fields kept   → forward-compat
+ *
+ * v4 (advice prompt VERSION 4) added per-item execution hints:
+ *  - actionable (bool)
+ *  - executor   ('opencode' | 'claude' | 'manual')
+ *  - cwd_hint   ('project_root')
+ * Each defaults / normalises to a safe value if the model omits or
+ * garbles them.  Inconsistent pairs (manual + actionable=true) are
+ * forced to a consistent state so the UI doesn't render an action
+ * button that the server would then reject.
+ *
+ * Also stamps the bookkeeping fields (v, msgCount, cli, cachedAt,
+ * truncated, omittedMessages) so the cache layer can do its job.
+ */
+function normaliseAdvicePayload(raw, meta) {
+  const cats = (raw && typeof raw.categories === 'object' && raw.categories) || {};
+  const categories = {};
+  for (const key of ALL_CATEGORIES) {
+    const arr = Array.isArray(cats[key]) ? cats[key] : [];
+    categories[key] = arr
+      .map(normaliseItem)
+      .filter((it) => it && it.evidence); // drop evidence-less items
+  }
+  return {
+    v: ADVICE_PROMPT_VERSION,
+    msgCount: meta.msgCount,
+    cli: meta.cli,
+    cachedAt: new Date().toISOString(),
+    truncated: meta.truncated || false,
+    omittedMessages: meta.omittedMessages || 0,
+    summary: typeof raw?.summary === 'string' ? raw.summary : '',
+    categories,
+    rationale: typeof raw?.rationale === 'string' ? raw.rationale : '',
+  };
+}
+
+function normaliseItem(it) {
+  if (!it || typeof it !== 'object') return null;
+  const severity = ALL_SEVERITIES.includes(it.severity) ? it.severity : 'low';
+
+  let executor = ALL_EXECUTORS.includes(it.executor) ? it.executor : 'manual';
+  let actionable = it.actionable === true;
+  const cwd_hint = ALL_CWD_HINTS.includes(it.cwd_hint) ? it.cwd_hint : 'project_root';
+
+  // Force consistency: a manual item cannot be actionable, and an
+  // actionable item cannot have executor=manual (would render a button
+  // that the API would then reject as NOT_ACTIONABLE).
+  if (executor === 'manual') actionable = false;
+  if (actionable && executor === 'manual') executor = 'opencode';
+
+  return {
+    severity,
+    title:    typeof it.title === 'string'    ? it.title.trim()    : '',
+    why:      typeof it.why === 'string'      ? it.why.trim()      : '',
+    action:   typeof it.action === 'string'   ? it.action.trim()   : '',
+    evidence: typeof it.evidence === 'string' ? it.evidence.trim() : '',
+    actionable,
+    executor,
+    cwd_hint,
+  };
+}
+
+// ---------------------------------------------------------------------------
+//  Exports
+// ---------------------------------------------------------------------------
+
+module.exports = {
+  generateAdvice,
+  loadAdvice,
+  isCacheFresh,
+  invalidateAdviceSettingsCache,
+  // exported for tests / debugging:
+  assembleContext,
+  normaliseAdvicePayload,
+};

package/server/llm/analysis-prompt.js

@@ -0,0 +1,162 @@
+/**
+ * Unified session-analysis prompt — one LLM call that returns BOTH the
+ * v2.1 capability scores (H1/H2/E1/O1) AND the per-session collaboration
+ * advice, as a single strict-JSON object:
+ *
+ *   { "scores": { H1:{clarity,converge,drift cells}, H2:{...}, E1:{...}, O1:{...} },
+ *     "advice": { summary, categories:{cost,accuracy,context,skills,workflow}, rationale } }
+ *
+ * Replaces the two separate calls (judge-prompts.buildSessionJudgePrompt +
+ * advice-prompt.buildAdvicePrompt).
+ *
+ * CRITICAL — score laundering guard: the model produces the scores AND the
+ * advice in the same context, but the advice section MUST NOT reference any
+ * score / level / dimension key.  Earlier (separate-call) experience showed
+ * the model otherwise echoes the numbers back as fake "evidence".  The
+ * `advice` rules below repeat this prohibition; the two JSON sections are
+ * independent.
+ *
+ * @author Felix
+ */
+
+'use strict';
+
+const { truncateContext } = require('./advice-prompt');
+
+/** First line marker so the resulting CLI session is filtered by the ETL
+ *  (registered in server/etl/judge-filter.js). */
+const ANALYSIS_SENTINEL = '[ABOSS-ANALYZE]';
+
+/** Bump when the JSON output contract changes.
+ *  v2: sub-indicator cells return a granular 0–100 `score` (was `level`). */
+const ANALYSIS_PROMPT_VERSION = 2;
+
+const CATEGORIES = ['cost', 'accuracy', 'context', 'skills', 'workflow'];
+
+// ---------------------------------------------------------------------------
+//  Context formatting (mirrors advice-prompt; kept local so this module is
+//  self-contained and changes here can't break advice generation).
+// ---------------------------------------------------------------------------
+
+function fmtNum(n) {
+  if (n == null || Number.isNaN(n)) return '–';
+  if (typeof n !== 'number') return String(n);
+  if (Number.isInteger(n)) return n.toLocaleString('en-US');
+  return n.toFixed(3);
+}
+
+function fmtToolTable(tools) {
+  if (!Array.isArray(tools) || tools.length === 0) return '(无工具调用)';
+  return tools.slice(0, 20).map((t) =>
+    `  ${(t.name || '?').padEnd(20)} count=${String(t.count ?? 0).padStart(4)} ` +
+    `err=${String(t.errorCount ?? 0).padStart(3)} ` +
+    `args="${(t.argsPreview || '').replace(/\s+/g, ' ').slice(0, 120)}"`
+  ).join('\n');
+}
+
+function fmtMessages(messages) {
+  if (!Array.isArray(messages) || messages.length === 0) return '(无消息)';
+  return messages
+    .filter((m) => m.text != null && m.text !== '')
+    .map((m) => `[${(m.role || '?').toUpperCase().padEnd(9)}] ${m.text}`)
+    .join('\n---\n');
+}
+
+// ---------------------------------------------------------------------------
+//  Rubric (scores) — difficulty-conditioned L1–L4 anchors.
+// ---------------------------------------------------------------------------
+
+const RUBRIC = `
+H1 立意（把模糊需求收敛成可执行的精确问题的功力）
+  clarity 初始指令清晰度 · converge 收敛效率 · drift 方向稳定性
+  L4 开局即给出目标+约束+验收，几乎无需追问；L3 信息基本充分，少量澄清；
+  L2 需多轮补全；L1 一句话甩任务、反复改方向。
+H2 判断（不盲从、敢质疑、敢推翻 vs 橡皮图章式照单全收）
+  challenge 合理质疑 · override 该推翻时推翻 · accept_rate 采纳前是否有判断
+  L4 在该质疑处质疑、在合理处高效采纳，质疑有依据；L3 大体审视偶有盲从；
+  L2 多数直接采纳少量质疑；L1 几乎全程"好的/继续"式盲从。
+E1 知识（AI 对该技术栈的掌握）
+  domain_errors 领域错误（越少越好）· staleness 过时引用（越少越好）· best_practice 最佳实践
+  L4 无错误、无过时、全程最佳实践；L3 偶有小瑕疵；L2 多处问题；L1 频繁错误/过时。
+O1 产出（结果是否真的可用）
+  first_take 一次采纳 · code_style 代码规范 · completeness 边界/异常/测试完备
+  L4 几乎一次过且规范完备；L3 小修即可；L2 需明显返工；L1 大量返工或缺失完备性。
+`;
+
+// ---------------------------------------------------------------------------
+//  Prompt
+// ---------------------------------------------------------------------------
+
+/**
+ * Build the unified analysis prompt.
+ *
+ * @param {object} ctx  advice-style context (assembleContext output) plus a
+ *                       numeric `difficulty` (1-4).  Shape:
+ *   { session:{model,difficulty,durationMinutes,cost,tokens,errorCount,
+ *              toolCallCount,messageCount,userCount,assistantCount,reverted},
+ *     toolBreakdown:[...], messages:[{role,text}], truncated, omittedMessages }
+ * @returns {string}
+ */
+function buildSessionAnalysisPrompt(ctx) {
+  const s = ctx.session || {};
+  const t = s.tokens || {};
+  const difficulty = s.difficulty ?? ctx.difficulty ?? '?';
+  const truncatedNote =
+    ctx.truncated === 'hard' ? '（注意:会话很长，已强力截断。）'
+    : ctx.truncated ? `（注意:中段已省略 ${ctx.omittedMessages} 条消息。）`
+    : '';
+
+  return `${ANALYSIS_SENTINEL}（内部标记，忽略本行）
+你是一名严格、客观的"人机协作"审计员兼协作教练。下面是一段开发者与 AI 编程助手的会话原文与基础统计。
+请一次性完成两件事，返回**单个严格 JSON 对象**，不要任何额外文字 / markdown：
+(A) scores —— 按 rubric 给五维能力打分；(B) advice —— 给出可执行的协作改进建议。
+
+================  (A) 评分 rubric（难度越高合格线越宽松，本会话难度 ${difficulty}/4）  ================
+${RUBRIC}
+scores 每个子指标返回三元组：
+- score: 0–100 的数值，**可含一位小数**（如 78.5）；证据不足或对话太短无法判断 → null（不要硬给低分）。
+  档位↔分数区间：L4 = 85–100 · L3 = 65–84 · L2 = 40–64 · L1 = 0–39；
+  请在区间内按实际好坏细分到具体分值，**不要只给 25/55/80/95 这种档位中点或整十的死板分**。
+- confidence: 0.0–1.0。
+- evidence: 引用对话中的具体表现（哪条消息、什么内容），说明为何是这个分值。没有证据写"未发现相关证据"。
+评分客观性：① 不因输出更长/更礼貌而加分；② 证据不足必须 null；③ 难度低不等于做得差。
+
+================  (B) advice —— 只评"如何使用 AI"，不评业务对错  ================
+评估开发者的协作方式（提问清晰度、上下文准备、工具/模型使用、流程节奏、是否该用 skill/subagent、成本）。
+不要评对话里讨论的具体技术/代码是否正确（那属于代码评审）。
+**关键禁令**：advice 部分禁止引用、推测或编造任何"分数/等级/Lx/子分/H1/H2/E1/O1"字眼；
+每条 evidence 必须是对话事实（如"第 3 条用户消息只说'改一下'，未给文件路径"），不得换算成分数。
+5 个类别：cost 省钱 · accuracy 协作层提准确（暴露假设/要求自检/加验证，非业务对错）· context 上下文准备 ·
+skills 推荐 opencode skill/subagent · workflow 流程节奏。
+AdviceItem: { severity: high|medium|low, title:≤20字, why:1句, action:下次怎么做(协作动作), evidence:对话事实,
+  actionable:bool, executor: opencode|claude|manual, cwd_hint: "project_root" }
+硬规则：5 个类别键必须存在(无内容给空数组)；AdviceItem 总数 ≤ 6，按 severity 由高到低；
+executor='manual' 时 actionable 必须 false。
+
+================  会话基础（只作事实参考）  ================
+模型 ${s.model || '未知'} · 难度 ${difficulty}/4 · 时长 ${fmtNum(s.durationMinutes)} 分钟 · 已回退 ${s.reverted ? '是' : '否'}
+消息 ${fmtNum(s.messageCount)}（用户 ${fmtNum(s.userCount)} / 助手 ${fmtNum(s.assistantCount)}）· 工具 ${fmtNum(s.toolCallCount)} 次 错误 ${fmtNum(s.errorCount)}
+Token in ${fmtNum(t.input)} / out ${fmtNum(t.output)} / reasoning ${fmtNum(t.reasoning)} / cacheR ${fmtNum(t.cacheRead)} / cacheW ${fmtNum(t.cacheWrite)} · 成本 $${typeof s.cost === 'number' ? s.cost.toFixed(4) : '–'}
+
+工具使用 Top 20:
+${fmtToolTable(ctx.toolBreakdown)}
+
+消息全文 ${truncatedNote}:
+${fmtMessages(ctx.messages)}
+
+================  输出 JSON（仅此对象）  ================
+{"scores":{
+  "H1":{"clarity":{"score":78.5,"confidence":0.8,"evidence":"…"},"converge":{"score":90,"confidence":0.9,"evidence":"…"},"drift":{"score":null,"confidence":0,"evidence":"无法判断"}},
+  "H2":{"challenge":{…},"override":{…},"accept_rate":{…}},
+  "E1":{"domain_errors":{…},"staleness":{…},"best_practice":{…}},
+  "O1":{"first_take":{…},"code_style":{…},"completeness":{…}}},
+ "advice":{"summary":"≤60字只谈协作","categories":{"cost":[],"accuracy":[],"context":[],"skills":[],"workflow":[]},"rationale":"≤80字只谈协作"}}`;
+}
+
+module.exports = {
+  ANALYSIS_SENTINEL,
+  ANALYSIS_PROMPT_VERSION,
+  CATEGORIES,
+  buildSessionAnalysisPrompt,
+  truncateContext,
+};