@bongos/core 1.19.706 → 1.19.707

package/tests/agent_invoke_live.mjs

@@ -0,0 +1,111 @@
+// tests/agent_invoke_live.mjs — the end-to-end proof, reproducible (task 1002491).
+//
+// tests/agent_invoke.mjs proves the DECISIONS with a fake pool, a fake runner and
+// a fake tree. It cannot prove the one thing criterion `agents-historian-exemplar`
+// actually claims: that the real historian, given the real corpus, comes back with
+// an answer whose citations are real. Only a real model call can say that, and a
+// real model call costs real money (~$0.90 a question at the time of writing) and
+// needs an authenticated `claude` CLI — so it must never run in CI or in the unit
+// gate, and it is OPT-IN.
+//
+//   AGENT_INVOKE_LIVE=1 node tests/agent_invoke_live.mjs ["<question>"]
+//
+// Skipped and exit 0 otherwise, so `run-unit-tests.js` sweeps it up harmlessly.
+//
+// The POOL is still faked — a builder box has no database, which is also why the
+// script's every edge is injectable. What is real here is the definition (parsed
+// from the committed .claude/agents/historian.md exactly as agents-sync would land
+// it), the corpus (this checkout), the model, and the citation check.
+
+import { strict as assert } from 'node:assert';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+
+if (process.env.AGENT_INVOKE_LIVE !== '1') {
+  console.log('  skip  agent_invoke_live — set AGENT_INVOKE_LIVE=1 to spend ~$1 proving the chain for real');
+  process.exit(0);
+}
+
+const invoke = require('../scripts/gds/agent-invoke.js');
+const agentsSync = require('../scripts/gds/agents-sync.js');
+const grader = require('../modules/grading/grader.js');
+
+// The definition exactly as the deploy-time reconcile would land it, so what runs
+// is the committed file and not a fixture that drifted from it.
+const candidate = agentsSync.parseAgentFile(
+  fs.readFileSync(path.join(ROOT, '.claude/agents/historian.md'), 'utf8'),
+  { name: 'historian' },
+);
+const definition = {
+  id: 1,
+  ...candidate,
+  trigger_spec: {},
+  scope_modules: [],
+  scope_paths: [],
+  scope_violation: null,
+  source: 'file',
+  provenance: 'built-in',
+  // Armed for the duration of this check. Note that a real deploy does NOT land
+  // it armed — see the header of scripts/gds/agent-invoke.js.
+  author_rank: 'archon',
+  enabled: true,
+};
+
+const ledger = [];
+const pool = {
+  async query(sql, params) {
+    if (/FROM agents_definitions/.test(sql)) return { rows: [definition] };
+    // No database on a builder box; the degrade path is exercised for free.
+    if (/FROM learnings/.test(sql)) throw new Error('no database on this box');
+    if (/^INSERT INTO agents_runs/.test(sql)) { ledger.push(['insert', params]); return { rows: [{ id: 1 }] }; }
+    ledger.push(['update', params]);
+    return { rows: [] };
+  },
+};
+
+const question = process.argv.slice(2).join(' ')
+  || 'Why does this project enforce permissions server-side rather than in the client, and what is the rule about editing local files to grant yourself authority?';
+
+console.log(`  Q: ${question}`);
+const started = Date.now();
+const result = await invoke.invokeAgent(
+  { name: 'historian', question, requestedByBuilderId: 1, requestedByLogin: 'live-check' },
+  {
+    pool,
+    runSubagentCached: (a) => grader.runSubagentCached({ ...a, opts: { ...a.opts, timeoutMs: 600000 } }),
+    spawnDeps: { brandingModels: { subagentDefault: 'claude-opus-5', subagentRoutine: 'claude-haiku-4-5-20251001' } },
+    warn: (m) => console.log(`  warn: ${m}`),
+  },
+);
+const secs = Math.round((Date.now() - started) / 1000);
+
+if (!result.ok) {
+  console.error(`  FAIL  ${result.code} — ${result.reason || ''} (${secs}s)`);
+  process.exit(1);
+}
+
+console.log(`\n${result.answer.answer}\n`);
+console.log('  Citations, every one resolved against this checkout:');
+for (const c of result.answer.citations) console.log(`    - ${c.ref}`);
+console.log(`\n  ${secs}s · ${result.model} · $${result.cost_usd} · confidence ${result.answer.confidence}`);
+
+// The claim the criterion makes, asserted rather than eyeballed.
+assert.ok(result.answer.citations.length > 0, 'a cited result');
+for (const c of result.answer.citations) {
+  const p = invoke.citationPath(c.ref);
+  if (/^learning:/.test(c.ref)) continue;
+  assert.ok(fs.existsSync(path.join(ROOT, p)), `${p} exists in the tree`);
+}
+// ...and the ledger said so: one go row, finished ok, with a real cost on it.
+const [insert] = ledger.filter(([k]) => k === 'insert');
+const [update] = ledger.filter(([k]) => k === 'update');
+assert.ok(insert && insert[1].includes('go'), 'the gate said go');
+assert.ok(update && update[1].includes('ok'), 'the run finished ok');
+assert.ok(Number(result.cost_usd) > 0, 'a real fire costs real money and the ledger carries it');
+
+console.log('\n  ok  the historian returned a cited result end-to-end, and the ledger agrees');