@bongos/core 1.19.332 → 1.19.334

package/modules/grading/grader-subagent.js

@@ -0,0 +1,328 @@
+// modules/grading/grader-subagent.js — actually running a subagent: the spawn,
+// the read-context bundle it is given, the cached and one-shot wrappers.
+//
+// Carved out of modules/grading/grader.js by task 1003206 (goal 1000079, the R03
+// de-monolith plan) to bring the grader under the 1,500-line ratchet. A STRUCTURAL
+// move: bodies unchanged, grader.js requires every name back, so the grade port
+// surface and the panel's behaviour are exactly as before.
+
+'use strict';
+
+const path = require('node:path');
+const fs = require('node:fs');
+const { spawn } = require('node:child_process');
+const { DEFAULT_GRADER_MODEL, THRESHOLD, VERSION, api } = require('./grader-config.js');
+const { validateAndScore } = require('./grader-score.js');
+
+function runSubagent({ prompt, opts = {} }) {
+  return new Promise((resolve, reject) => {
+    const timeoutMs = opts.timeoutMs ?? 240_000;
+    const cwd = opts.cwd ?? process.cwd();
+    const model = opts.model ?? DEFAULT_GRADER_MODEL;
+    // Env escape hatch (OTB_CLAUDE_BIN / CLAUDE_BIN) for installs where the CLI
+    // isn't named `claude` on PATH — covers any OS, not just Windows.
+    const bin = opts.claudeBin ?? process.env.OTB_CLAUDE_BIN ?? process.env.CLAUDE_BIN ?? 'claude';
+
+    const args = [
+      '-p',
+      '--print',
+      '--model', model,
+      '--output-format', 'json',
+      '--tools', 'Read',
+      '--disable-slash-commands',
+      '--permission-mode', 'bypassPermissions',
+    ];
+
+    const startedAt = Date.now();
+    const child = spawn(bin, args, {
+      cwd,
+      stdio: ['pipe', 'pipe', 'pipe'],
+      // On Windows the `claude` CLI is a `claude.cmd` shim that Node's spawn
+      // can't launch without a shell (ENOENT -> SUBAGENT_SPAWN_FAILED, observed
+      // on #662). Route through cmd.exe on win32 only; Mac/Linux keep
+      // shell:false (the default, and the proven-working path). args carry no
+      // user input (the prompt goes in via stdin), so shell injection is a
+      // non-issue. windowsHide suppresses the transient console window.
+      shell: process.platform === 'win32',
+      windowsHide: true,
+      // MAS Phase 3 (#427): mark this child as a subagent so the Conductor
+      // UserPromptSubmit hook (.claude/hooks/conductor.js) bails instead of
+      // injecting advisory context into the grader worker's prompt. Any
+      // subagent we spawn inherits this flag and is likewise suppressed.
+      env: { ...process.env, OTB_SUBAGENT: '1' },
+    });
+
+    let stdout = '';
+    let stderr = '';
+    let timedOut = false;
+
+    const killTimer = setTimeout(() => {
+      timedOut = true;
+      try { child.kill('SIGTERM'); } catch (_) {}
+      setTimeout(() => { try { child.kill('SIGKILL'); } catch (_) {} }, 5_000);
+    }, timeoutMs);
+
+    child.stdout.on('data', (b) => { stdout += b.toString('utf8'); });
+    child.stderr.on('data', (b) => { stderr += b.toString('utf8'); });
+
+    child.on('error', (err) => {
+      clearTimeout(killTimer);
+      reject(Object.assign(new Error(`subagent spawn failed: ${err.message}`), {
+        code: 'SUBAGENT_SPAWN_FAILED',
+        cause: err,
+      }));
+    });
+
+    child.on('close', (exitCode) => {
+      clearTimeout(killTimer);
+      const runMs = Date.now() - startedAt;
+      if (timedOut) {
+        return reject(Object.assign(new Error(`subagent timed out after ${timeoutMs}ms`), {
+          code: 'SUBAGENT_TIMEOUT',
+          run_ms: runMs,
+          stdout,
+          stderr,
+        }));
+      }
+      if (exitCode !== 0) {
+        return reject(Object.assign(new Error(`subagent exited with code ${exitCode}`), {
+          code: 'SUBAGENT_NONZERO_EXIT',
+          exit_code: exitCode,
+          run_ms: runMs,
+          stdout,
+          stderr,
+        }));
+      }
+      resolve({ stdout, stderr, run_ms: runMs, exit_code: exitCode, cost_usd: extractCostUsd(stdout) });
+    });
+
+    child.stdin.write(prompt, 'utf8');
+    child.stdin.end();
+  });
+}
+
+// ---------- task 1002660: shared context bundle ----------
+//
+// ~90% of panel cost was fixed re-exploration: four subprocesses each spending
+// agentic Read turns re-opening the same changed files. Read them ONCE here
+// and hand every worker the post-change contents inline (fenced, untrusted),
+// so a worker's Read tool is only needed for files beyond the bundle.
+const CONTEXT_BUNDLE_MAX_FILES = 24;
+const CONTEXT_BUNDLE_MAX_TOTAL_CHARS = 120_000;
+const CONTEXT_BUNDLE_MAX_FILE_CHARS = 16_000;
+// Binary-ish extensions never worth inlining (the diff already names them).
+const CONTEXT_BUNDLE_SKIP_RE = /\.(?:png|jpe?g|gif|webp|ico|svgz|woff2?|ttf|eot|zip|gz|tgz|tar|pdf|mp[34]|wav|ogg|bin|exe|dll|node)$/i;
+
+// Added lines per file, from the unified diff. Only lines long enough to be a
+// meaningful fingerprint (a bare `}` or `});` matches anywhere) — used to prove
+// a truncated bundle prefix actually SHOWS the change (task 1002905).
+const BUNDLE_EVIDENCE_MIN_CHARS = 12;
+function diffAddedLinesByFile(diff) {
+  const byFile = new Map();
+  let cur = null;
+  let prevWasOldHeader = false;
+  for (const line of String(diff || '').split('\n')) {
+    // A '+++ ' header only counts right after its '--- ' partner: a content line
+    // reading '++ x' arrives as '+++ x' and must not be mistaken for a header.
+    if (prevWasOldHeader && line.startsWith('+++ ')) {
+      prevWasOldHeader = false;
+      const p = line.slice(4).trim().replace(/^b\//, '');
+      cur = p === '/dev/null' ? null : p;
+      continue;
+    }
+    prevWasOldHeader = line.startsWith('--- ');
+    if (prevWasOldHeader || line.startsWith('diff --git')) continue;
+    if (!cur || !line.startsWith('+')) continue;
+    const t = line.slice(1).trim();
+    if (t.length < BUNDLE_EVIDENCE_MIN_CHARS) continue;
+    if (!byFile.has(cur)) byFile.set(cur, []);
+    byFile.get(cur).push(t);
+  }
+  return byFile;
+}
+
+// Pure-ish (fs reads only, injectable): returns { files: [{file, content,
+// truncated, chars_total}], skipped: [file...], total_chars } for the post-change
+// contents of changedFiles under cwd. Deleted/unreadable files land in skipped —
+// their absence from the bundle is visible, never silent.
+//
+// Security (panel hacker finding, task 1002660): changedFiles is builder-
+// controlled and git commits symlinks, so a naive readFileSync would follow a
+// committed link OUT of the repo (SSH keys, session tokens) and inject the
+// target into worker prompts + the stored grade. lstat-refuse symlinks and
+// non-files, and require the resolved path to stay under cwd (containment) —
+// a refused entry lands in skipped, visible like every other exclusion.
+function buildContextBundle({ changedFiles, cwd, diff, fsImpl = fs } = {}) {
+  const files = [];
+  const skipped = [];
+  let total = 0;
+  const root = path.resolve(cwd || process.cwd());
+  // task 1002905: the fabricated all-blocker FAIL. A file over the per-file cap
+  // is truncated from the HEAD, so on a large file (ship.js: 250k chars) a change
+  // at char 62k lands entirely outside the bundle — and the prompt presents what
+  // IS bundled as "the post-change contents". A worker then searches the prefix,
+  // doesn't find the new code, and reports it as absent (5 blockers, every claim
+  // exactly the pre-change state). A prefix that contains none of the change is
+  // worse than no bundle at all, so refuse it: skipped[] already tells the worker
+  // to Read the file.
+  const addedByFile = diffAddedLinesByFile(diff);
+  for (const rel of (Array.isArray(changedFiles) ? changedFiles : [])) {
+    if (files.length >= CONTEXT_BUNDLE_MAX_FILES || total >= CONTEXT_BUNDLE_MAX_TOTAL_CHARS) {
+      skipped.push(rel);
+      continue;
+    }
+    if (CONTEXT_BUNDLE_SKIP_RE.test(rel)) { skipped.push(rel); continue; }
+    let content;
+    try {
+      const abs = path.resolve(root, rel);
+      if (abs !== root && !abs.startsWith(root + path.sep)) { skipped.push(rel); continue; }
+      const st = fsImpl.lstatSync(abs);
+      if (!st.isFile() || st.isSymbolicLink()) { skipped.push(rel); continue; }
+      content = fsImpl.readFileSync(abs, 'utf8');
+    } catch (_) { skipped.push(rel); continue; } // deleted by the diff, or unreadable
+    const charsTotal = content.length;
+    let truncated = false;
+    if (content.length > CONTEXT_BUNDLE_MAX_FILE_CHARS) {
+      content = content.slice(0, CONTEXT_BUNDLE_MAX_FILE_CHARS);
+      truncated = true;
+    }
+    if (truncated) {
+      const added = addedByFile.get(rel);
+      if (added && added.length && !added.some((a) => content.includes(a))) { skipped.push(rel); continue; }
+    }
+    if (total + content.length > CONTEXT_BUNDLE_MAX_TOTAL_CHARS) { skipped.push(rel); continue; }
+    total += content.length;
+    files.push({ file: rel, content, truncated, chars_total: charsTotal });
+  }
+  return { files, skipped, total_chars: total };
+}
+
+// System 5 (task 1359): an OPT-IN cached path around runSubagent. When the caller
+// supplies a `cache` spec (from llm-cache.makeCacheSpec), a hit replays the stored
+// result at cost_usd:0 (so $0 flows naturally into extractCostUsd → session-cost);
+// a miss runs the untouched runSubagent and stores a clean (exit 0) result. With
+// no cache spec it's exactly runSubagent — the default path is never altered. The
+// llm-cache reaches us through the doorway (BV1.R73: it's shared infra core also
+// uses, so a module can't own it); if it's somehow absent the path degrades to an
+// ordinary uncached run, exactly as the old lazy-require catch did.
+async function runSubagentCached({ prompt, opts = {}, cache = null }) {
+  if (!cache || !cache.key) return runSubagent({ prompt, opts });
+  const llmCache = api.llmCache;
+  if (!llmCache) return runSubagent({ prompt, opts });
+  return llmCache.withCache(cache, () => runSubagent({ prompt, opts }));
+}
+
+// #486 (ADR 0028 path 1): the `claude -p --output-format json` envelope reports
+// the subprocess's own dollar cost in `total_cost_usd` (verified present:
+// {type, ..., total_cost_usd, usage, modelUsage}). We were parsing only
+// `env.result` (extractAssistantResult) and silently discarding that cost — the
+// single biggest reason cost_log's `api` category sat at $0.60 across the whole
+// project (#482 audit F1). Pull it out so the panel's spend can be logged
+// (category=api, source=grader). Returns null on non-JSON / partial output, so
+// a cost signal never fabricates a number it doesn't have.
+function extractCostUsd(rawStdout) {
+  try {
+    const env = JSON.parse(rawStdout);
+    if (env && typeof env.total_cost_usd === 'number' && env.total_cost_usd >= 0) {
+      return env.total_cost_usd;
+    }
+  } catch (_) { /* non-JSON or partial stdout — no cost signal */ }
+  return null;
+}
+
+function extractAssistantResult(rawStdout) {
+  try {
+    const env = JSON.parse(rawStdout);
+    if (env && typeof env.result === 'string') return env.result;
+    if (env && typeof env === 'object' && 'functional_fidelity' in env) return env;
+    return rawStdout;
+  } catch (_) {
+    return rawStdout;
+  }
+}
+
+// Run one shot end-to-end: build the prompt (caller did this), spawn the
+// subagent at the requested model, parse + validate. Returns a per-shot
+// grade-result; subprocess failures become passed=false per-shot results so
+// aggregateShots can still produce a final verdict.
+async function runOneShot({ prompt, model, shotIndex, runOpts }) {
+  try {
+    const { stdout, run_ms, exit_code, cost_usd } = await runSubagent({
+      prompt,
+      opts: { ...runOpts, model },
+    });
+    const assistantResult = extractAssistantResult(stdout);
+    const scored = validateAndScore(assistantResult, {
+      run_ms,
+      exit_code,
+      raw_chars: stdout.length,
+      model,
+      shot_index: shotIndex,
+    });
+    // #486 (ADR 0028 path 1): carry this shot's own subprocess cost so the
+    // trivial-diff path (gradeSubagentV3MultiShot → aggregateShots) can sum it
+    // into panel_cost_usd just like the full panel does.
+    scored.cost_usd = cost_usd ?? null;
+    return scored;
+  } catch (err) {
+    return {
+      score: 0,
+      passed: false,
+      threshold: THRESHOLD,
+      grader_kind: 'subagent',
+      grader_version: VERSION,
+      rubric_scores: {},
+      signals: {
+        grader_kind: 'subagent',
+        grader_version: VERSION,
+        model,
+        threshold: THRESHOLD,
+        shot_index: shotIndex,
+        run_ms: err.run_ms ?? null,
+        exit_code: err.exit_code ?? null,
+        subprocess_error: err.code || 'UNKNOWN',
+        stderr_tail: (err.stderr || '').slice(-500),
+      },
+      notes_md: `Shot ${shotIndex}: subprocess failed (${err.code || 'UNKNOWN'}: ${err.message}).`,
+      issues: [],
+    };
+  }
+}
+
+// ---------- v4: panel aggregation (MAS Phase 2.0) ----------
+
+// Collapses N per-worker WorkerResults into the final v4 grade-result.
+//
+// Gate logic:
+//   - Any worker `fail`               → overall FAIL (a real finding blocks).
+//   - Else any worker errored         → overall FALLTHROUGH (manual-confirm) —
+//                                        a crash is missing signal, not a
+//                                        finding, so it must not hard-block
+//                                        on infra flakiness (#126).
+//   - Else any worker `concerns`      → overall FALLTHROUGH (manual-confirm).
+//   - All workers `pass`              → overall PASS.
+//
+// Server-compat forwarding:
+//   The grade POST shape (ff, cq, economy, confidence, issues) is preserved
+//   so the server's existing validateAndScore re-validation aligns with the
+//   panel's verdict. We achieve this by setting the forwarded confidence:
+//     pass     → median worker confidence (server's gate also passes)
+//     concerns → FALLTHROUGH_CONFIDENCE (below LOW_CONFIDENCE → server fails)
+//     fail     → 0 + a failing axis preserved at < threshold (server fails)
+//
+// rubric_scores:
+//   Each axis is the median across workers that reported it (Narc reports
+//   functional_fidelity; Quality reports code_quality + economy). Preserves
+//   the trend dashboard's one-number-per-task contract.
+//
+// signals.workers[] carries the full per-worker audit trail.
+
+module.exports = {
+  buildContextBundle,
+  diffAddedLinesByFile,
+  extractAssistantResult,
+  extractCostUsd,
+  runOneShot,
+  runSubagent,
+  runSubagentCached,
+};