@claude-flow/cli 3.10.4 → 3.10.6

package/dist/src/benchmarks/gaia-e2e-smoke.d.ts

@@ -0,0 +1,27 @@
+/**
+ * GAIA End-to-End Smoke — ADR-133
+ *
+ * Wires gaia-agent.ts + gaia-judge.ts into a single end-to-end pipeline:
+ *
+ *   for each question in SMOKE_FIXTURE:
+ *     1. runGaiaAgent(question)  — Haiku agent loop, ≤8 turns
+ *     2. judgeAnswer(question, result.finalAnswer)  — exact-match fast-path,
+ *        Sonnet LLM-judge only if exact-match misses
+ *
+ * Reports: pass rate, total cost, mean turn count.
+ * Asserts: ≥ 3/5 questions pass (lenient — smoke fixture is not trivial).
+ *
+ * Cost discipline:
+ *   - Agent: claude-haiku-4-5 at $0.25/$1.25 per M tokens
+ *   - Judge: claude-sonnet-4-6 at $3/$15 per M tokens (only when needed)
+ *   - Expected total for 5 questions × ~2 turns × Haiku + 1-2 Sonnet
+ *     judge calls ≈ $0.02
+ *
+ * Usage:
+ *   ANTHROPIC_API_KEY=sk-ant-... npx tsx src/benchmarks/gaia-e2e-smoke.ts
+ *
+ * Refs: ADR-133, #2156
+ */
+declare function runE2ESmoke(): Promise<void>;
+export { runE2ESmoke };
+//# sourceMappingURL=gaia-e2e-smoke.d.ts.map

package/dist/src/benchmarks/gaia-e2e-smoke.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gaia-e2e-smoke.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/gaia-e2e-smoke.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAqDH,iBAAe,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CA0G1C;AAiBD,OAAO,EAAE,WAAW,EAAE,CAAC"}

package/dist/src/benchmarks/gaia-e2e-smoke.js

@@ -0,0 +1,136 @@
+/**
+ * GAIA End-to-End Smoke — ADR-133
+ *
+ * Wires gaia-agent.ts + gaia-judge.ts into a single end-to-end pipeline:
+ *
+ *   for each question in SMOKE_FIXTURE:
+ *     1. runGaiaAgent(question)  — Haiku agent loop, ≤8 turns
+ *     2. judgeAnswer(question, result.finalAnswer)  — exact-match fast-path,
+ *        Sonnet LLM-judge only if exact-match misses
+ *
+ * Reports: pass rate, total cost, mean turn count.
+ * Asserts: ≥ 3/5 questions pass (lenient — smoke fixture is not trivial).
+ *
+ * Cost discipline:
+ *   - Agent: claude-haiku-4-5 at $0.25/$1.25 per M tokens
+ *   - Judge: claude-sonnet-4-6 at $3/$15 per M tokens (only when needed)
+ *   - Expected total for 5 questions × ~2 turns × Haiku + 1-2 Sonnet
+ *     judge calls ≈ $0.02
+ *
+ * Usage:
+ *   ANTHROPIC_API_KEY=sk-ant-... npx tsx src/benchmarks/gaia-e2e-smoke.ts
+ *
+ * Refs: ADR-133, #2156
+ */
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { SMOKE_FIXTURE, } from './gaia-loader.js';
+import { runGaiaAgent, } from './gaia-agent.js';
+import { judgeAnswer, } from './gaia-judge.js';
+// ---------------------------------------------------------------------------
+// Configuration
+// ---------------------------------------------------------------------------
+/** Agent model — Haiku only for cost discipline. */
+const AGENT_MODEL = 'claude-haiku-4-5';
+/** Judge model — Sonnet for semantic judgments (only when exact-match fails). */
+const JUDGE_MODEL = 'claude-sonnet-4-6';
+/** Minimum pass rate for smoke to succeed. */
+const MIN_PASS_RATE = 3 / 5;
+// Haiku pricing ($/M tokens)
+const HAIKU_IN = 0.25;
+const HAIKU_OUT = 1.25;
+// Sonnet pricing ($/M tokens)
+const SONNET_IN = 3.0;
+const SONNET_OUT = 15.0;
+// ---------------------------------------------------------------------------
+// Runner
+// ---------------------------------------------------------------------------
+async function runE2ESmoke() {
+    const hasKey = !!(process.env.ANTHROPIC_API_KEY?.trim());
+    if (!hasKey) {
+        console.error('ANTHROPIC_API_KEY is required for the end-to-end smoke.\n' +
+            'Set it with: export ANTHROPIC_API_KEY=sk-ant-...');
+        process.exit(1);
+    }
+    const cacheDir = path.join(os.homedir(), '.cache', 'ruflo', 'gaia', 'judgments');
+    const judgeOpts = { judgeModel: JUDGE_MODEL, cacheDir };
+    const questions = SMOKE_FIXTURE;
+    const rows = [];
+    console.log(`\n=== GAIA End-to-End Smoke (${questions.length} questions) ===\n`);
+    console.log(`Agent: ${AGENT_MODEL}  |  Judge: ${JUDGE_MODEL}\n` +
+        `Questions: ${questions.length}  |  Min pass rate: ${(MIN_PASS_RATE * 100).toFixed(0)}%\n`);
+    for (const q of questions) {
+        process.stdout.write(`[${q.task_id}] "${q.question.slice(0, 60)}..." `);
+        // Run agent
+        const agentResult = await runGaiaAgent(q, { model: AGENT_MODEL });
+        // Judge
+        const judgeResult = await judgeAnswer({ id: q.task_id, expected: q.final_answer, questionText: q.question }, agentResult.finalAnswer, judgeOpts);
+        rows.push({ question: q, agentResult, judgeResult });
+        const verdict = judgeResult.passed ? '\x1b[32mPASS\x1b[0m' : '\x1b[31mFAIL\x1b[0m';
+        const path_ = judgeResult.scoringPath;
+        console.log(`${verdict}  (turns=${agentResult.turns}, ` +
+            `answer="${agentResult.finalAnswer ?? 'null'}", ` +
+            `expected="${q.final_answer}", path=${path_})`);
+    }
+    // ── Summary ──
+    console.log('\n--- Summary ---\n');
+    const passed = rows.filter((r) => r.judgeResult.passed).length;
+    const total = rows.length;
+    const passRate = passed / total;
+    let totalAgentCostUsd = 0;
+    let totalJudgeCostUsd = 0;
+    let totalTurns = 0;
+    for (const row of rows) {
+        const { agentResult, judgeResult } = row;
+        totalTurns += agentResult.turns;
+        totalAgentCostUsd +=
+            (agentResult.totalInputTokens / 1_000_000) * HAIKU_IN +
+                (agentResult.totalOutputTokens / 1_000_000) * HAIKU_OUT;
+        totalJudgeCostUsd += judgeResult.judgeCostUsd ?? 0;
+    }
+    const totalCostUsd = totalAgentCostUsd + totalJudgeCostUsd;
+    const meanTurns = totalTurns / total;
+    console.log(`Pass rate      : ${passed}/${total}  (${(passRate * 100).toFixed(0)}%)`);
+    console.log(`Mean turns     : ${meanTurns.toFixed(1)}`);
+    console.log(`Agent cost     : $${totalAgentCostUsd.toFixed(5)}  (Haiku)`);
+    console.log(`Judge cost     : $${totalJudgeCostUsd.toFixed(5)}  (Sonnet, only when needed)`);
+    console.log(`Total cost     : $${totalCostUsd.toFixed(5)}`);
+    // ── Per-row detail ──
+    console.log('\n--- Per-question detail ---\n');
+    for (const row of rows) {
+        const { question, agentResult, judgeResult } = row;
+        const verdict = judgeResult.passed ? 'PASS' : 'FAIL';
+        console.log(`  ${verdict}  ${question.task_id}  turns=${agentResult.turns}  ` +
+            `path=${judgeResult.scoringPath}  ` +
+            `answer="${agentResult.finalAnswer ?? 'null'}"  ` +
+            `expected="${question.final_answer}"`);
+        if (judgeResult.judgeReason) {
+            console.log(`        reason: ${judgeResult.judgeReason}`);
+        }
+        if (agentResult.error) {
+            console.log(`        error : ${agentResult.error}`);
+        }
+    }
+    // ── Assertion ──
+    console.log('');
+    if (passRate >= MIN_PASS_RATE) {
+        console.log(`\x1b[32mSmoke PASSED\x1b[0m — ${passed}/${total} ≥ ${(MIN_PASS_RATE * 100).toFixed(0)}% required.\n`);
+    }
+    else {
+        console.error(`\x1b[31mSmoke FAILED\x1b[0m — ${passed}/${total} < ${(MIN_PASS_RATE * 100).toFixed(0)}% required.\n`);
+        process.exit(1);
+    }
+}
+// ---------------------------------------------------------------------------
+// Entry point
+// ---------------------------------------------------------------------------
+const isMain = process.argv[1] && (process.argv[1].endsWith('gaia-e2e-smoke.ts') ||
+    process.argv[1].endsWith('gaia-e2e-smoke.js'));
+if (isMain) {
+    runE2ESmoke().catch((err) => {
+        console.error('E2E smoke failed:', err);
+        process.exit(1);
+    });
+}
+export { runE2ESmoke };
+//# sourceMappingURL=gaia-e2e-smoke.js.map

package/dist/src/benchmarks/gaia-e2e-smoke.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gaia-e2e-smoke.js","sourceRoot":"","sources":["../../../src/benchmarks/gaia-e2e-smoke.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EACL,aAAa,GAEd,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,YAAY,GAEb,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,WAAW,GAGZ,MAAM,iBAAiB,CAAC;AAEzB,8EAA8E;AAC9E,gBAAgB;AAChB,8EAA8E;AAE9E,oDAAoD;AACpD,MAAM,WAAW,GAAG,kBAAkB,CAAC;AAEvC,iFAAiF;AACjF,MAAM,WAAW,GAAG,mBAAmB,CAAC;AAExC,8CAA8C;AAC9C,MAAM,aAAa,GAAG,CAAC,GAAG,CAAC,CAAC;AAE5B,6BAA6B;AAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC;AACtB,MAAM,SAAS,GAAG,IAAI,CAAC;AAEvB,8BAA8B;AAC9B,MAAM,SAAS,GAAG,GAAG,CAAC;AACtB,MAAM,UAAU,GAAG,IAAI,CAAC;AAYxB,8EAA8E;AAC9E,SAAS;AACT,8EAA8E;AAE9E,KAAK,UAAU,WAAW;IACxB,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,CAAC,KAAK,CACX,2DAA2D;YAC3D,kDAAkD,CACnD,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IACjF,MAAM,SAAS,GAAiB,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;IAEtE,MAAM,SAAS,GAAG,aAAa,CAAC;IAChC,MAAM,IAAI,GAAa,EAAE,CAAC;IAE1B,OAAO,CAAC,GAAG,CAAC,gCAAgC,SAAS,CAAC,MAAM,mBAAmB,CAAC,CAAC;IACjF,OAAO,CAAC,GAAG,CACT,UAAU,WAAW,eAAe,WAAW,IAAI;QACnD,cAAc,SAAS,CAAC,MAAM,uBAAuB,CAAC,aAAa,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAC3F,CAAC;IAEF,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,MAAM,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC;QAExE,YAAY;QACZ,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;QAElE,QAAQ;QACR,MAAM,WAAW,GAAG,MAAM,WAAW,CACnC,EAAE,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC,YAAY,EAAE,YAAY,EAAE,CAAC,CAAC,QAAQ,EAAE,EACrE,WAAW,CAAC,WAAW,EACvB,SAAS,CACV,CAAC;QAEF,IAAI,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC,CAAC;QAErD,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,qBAAqB,CAAC;QACnF,MAAM,KAAK,GAAG,WAAW,CAAC,WAAW,CAAC;QACtC,OAAO,CAAC,GAAG,CACT,GAAG,OAAO,YAAY,WAAW,CAAC,KAAK,IAAI;YAC3C,WAAW,WAAW,CAAC,WAAW,IAAI,MAAM,KAAK;YACjD,aAAa,CAAC,CAAC,YAAY,WAAW,KAAK,GAAG,CAC/C,CAAC;IACJ,CAAC;IAED,gBAAgB;IAChB,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;IAEnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC/D,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;IAC1B,MAAM,QAAQ,GAAG,MAAM,GAAG,KAAK,CAAC;IAEhC,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,GAAG,GAAG,CAAC;QACzC,UAAU,IAAI,WAAW,CAAC,KAAK,CAAC;QAChC,iBAAiB;YACf,CAAC,WAAW,CAAC,gBAAgB,GAAG,SAAS,CAAC,GAAG,QAAQ;gBACrD,CAAC,WAAW,CAAC,iBAAiB,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;QAC1D,iBAAiB,IAAI,WAAW,CAAC,YAAY,IAAI,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,YAAY,GAAG,iBAAiB,GAAG,iBAAiB,CAAC;IAC3D,MAAM,SAAS,GAAG,UAAU,GAAG,KAAK,CAAC;IAErC,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,IAAI,KAAK,MAAM,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACtF,OAAO,CAAC,GAAG,CAAC,oBAAoB,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,qBAAqB,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IAC1E,OAAO,CAAC,GAAG,CAAC,qBAAqB,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,8BAA8B,CAAC,CAAC;IAC7F,OAAO,CAAC,GAAG,CAAC,qBAAqB,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAE5D,uBAAuB;IACvB,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;IAC/C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,GAAG,GAAG,CAAC;QACnD,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QACrD,OAAO,CAAC,GAAG,CACT,KAAK,OAAO,KAAK,QAAQ,CAAC,OAAO,WAAW,WAAW,CAAC,KAAK,IAAI;YACjE,QAAQ,WAAW,CAAC,WAAW,IAAI;YACnC,WAAW,WAAW,CAAC,WAAW,IAAI,MAAM,KAAK;YACjD,aAAa,QAAQ,CAAC,YAAY,GAAG,CACtC,CAAC;QACF,IAAI,WAAW,CAAC,WAAW,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,mBAAmB,WAAW,CAAC,WAAW,EAAE,CAAC,CAAC;QAC5D,CAAC;QACD,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,mBAAmB,WAAW,CAAC,KAAK,EAAE,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAED,kBAAkB;IAClB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,IAAI,QAAQ,IAAI,aAAa,EAAE,CAAC;QAC9B,OAAO,CAAC,GAAG,CACT,iCAAiC,MAAM,IAAI,KAAK,MAAM,CAAC,aAAa,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CACtG,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,KAAK,CACX,iCAAiC,MAAM,IAAI,KAAK,MAAM,CAAC,aAAa,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CACtG,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,cAAc;AACd,8EAA8E;AAE9E,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAChC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,mBAAmB,CAAC;IAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAC9C,CAAC;AACF,IAAI,MAAM,EAAE,CAAC;IACX,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QAC1B,OAAO,CAAC,KAAK,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,OAAO,EAAE,WAAW,EAAE,CAAC"}

package/dist/src/benchmarks/gaia-ensemble-pilot.d.ts

@@ -0,0 +1,17 @@
+/**
+ * GAIA Ensemble Pilot — ADR-139 5-question validation
+ *
+ * Runs 5 diverse GAIA L1 questions through the 2-model ensemble
+ * (claude-sonnet-4-6 + gemini-2.5-pro) and reports accuracy, cost, and
+ * projections. OpenRouter is skipped until credits are topped up.
+ *
+ * Usage:
+ *   node dist/src/benchmarks/gaia-ensemble-pilot.js
+ *   node dist/src/benchmarks/gaia-ensemble-pilot.js --models claude-sonnet-4-6,gemini-2.5-pro,openai/gpt-5
+ *
+ * Exit codes:
+ *   0  pilot passed (accuracy ≥ 3/5 AND projected 53Q cost ≤ $40)
+ *   1  pilot failed or cost exceeded
+ */
+export {};
+//# sourceMappingURL=gaia-ensemble-pilot.d.ts.map

package/dist/src/benchmarks/gaia-ensemble-pilot.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gaia-ensemble-pilot.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/gaia-ensemble-pilot.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG"}

package/dist/src/benchmarks/gaia-ensemble-pilot.js

@@ -0,0 +1,106 @@
+/**
+ * GAIA Ensemble Pilot — ADR-139 5-question validation
+ *
+ * Runs 5 diverse GAIA L1 questions through the 2-model ensemble
+ * (claude-sonnet-4-6 + gemini-2.5-pro) and reports accuracy, cost, and
+ * projections. OpenRouter is skipped until credits are topped up.
+ *
+ * Usage:
+ *   node dist/src/benchmarks/gaia-ensemble-pilot.js
+ *   node dist/src/benchmarks/gaia-ensemble-pilot.js --models claude-sonnet-4-6,gemini-2.5-pro,openai/gpt-5
+ *
+ * Exit codes:
+ *   0  pilot passed (accuracy ≥ 3/5 AND projected 53Q cost ≤ $40)
+ *   1  pilot failed or cost exceeded
+ */
+import { runEnsemblePilot } from './gaia-ensemble.js';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as os from 'node:os';
+const GAIA_CACHE = path.join(os.homedir(), '.cache', 'ruflo', 'gaia', 'level1-main.json');
+const PILOT_QUESTION_IDS = [
+    'e1fc63a2-da7a-432f-be78-7c4a95598703', // marathon pace math
+    '8e867cd7-cff9-4e6c-867a-ff5ddc2550be', // discography lookup
+    '2d83110e-a098-4ebb-9987-066c06fa42d0', // reversed text encoding
+    '27d5d136-8563-469e-92bf-fd103c28b57c', // boolean logic formulas
+    'dc28cf18-6431-458b-83ef-64b3ce566c10', // recipe/cooking lookup
+];
+async function main() {
+    // Parse --models flag
+    const modelsArg = process.argv.find((a) => a.startsWith('--models='))?.split('=')[1];
+    const models = modelsArg ? modelsArg.split(',') : ['claude-sonnet-4-6', 'gemini-2.5-pro'];
+    console.error(`[pilot] Models: ${models.join(', ')}`);
+    console.error(`[pilot] Loading GAIA dataset from: ${GAIA_CACHE}`);
+    if (!fs.existsSync(GAIA_CACHE)) {
+        console.error('[pilot] ERROR: GAIA cache not found. Run a full benchmark first to populate cache.');
+        process.exit(1);
+    }
+    const allQuestions = JSON.parse(fs.readFileSync(GAIA_CACHE, 'utf-8'));
+    const pilotQuestions = allQuestions.filter((q) => PILOT_QUESTION_IDS.includes(q.task_id));
+    if (pilotQuestions.length !== PILOT_QUESTION_IDS.length) {
+        console.error(`[pilot] WARNING: Expected ${PILOT_QUESTION_IDS.length} questions, found ${pilotQuestions.length}`);
+    }
+    console.error(`[pilot] Running ${pilotQuestions.length} questions through ${models.length}-model ensemble...`);
+    const result = await runEnsemblePilot(pilotQuestions, {
+        models,
+        maxTurns: 10,
+        maxTokensPerTurn: 4096,
+        perTurnTimeoutMs: 90_000,
+    });
+    // Write JSON output to stdout
+    const output = {
+        pilotConfig: { models, questionCount: pilotQuestions.length },
+        result: {
+            correct: result.correct,
+            total: result.total,
+            accuracy: result.accuracy,
+            totalCostUsd: result.totalCostUsd,
+            projectedCost53Q: result.projectedCost53Q,
+            projectedCost300Q: result.projectedCost300Q,
+            meanWallMs: result.meanWallMs,
+        },
+        costGate: {
+            limit53Q: 40,
+            limit300Q: 250,
+            passes53Q: result.projectedCost53Q <= 40,
+            passes300Q: result.projectedCost300Q <= 250,
+        },
+        recommendation: buildRecommendation(result.correct, result.total, result.projectedCost53Q, result.projectedCost300Q),
+        perQuestion: result.perQuestion,
+    };
+    process.stdout.write(JSON.stringify(output, null, 2) + '\n');
+    // Human-readable summary to stderr
+    console.error('\n=== ENSEMBLE PILOT SUMMARY ===');
+    console.error(`Accuracy: ${result.correct}/${result.total} (${(result.accuracy * 100).toFixed(1)}%)`);
+    console.error(`Total cost: $${result.totalCostUsd.toFixed(4)}`);
+    console.error(`Projected 53-Q cost: $${result.projectedCost53Q.toFixed(2)} (gate: $40)`);
+    console.error(`Projected 300-Q cost: $${result.projectedCost300Q.toFixed(2)} (gate: $250)`);
+    console.error(`Mean wall time: ${(result.meanWallMs / 1000).toFixed(1)}s/Q`);
+    console.error('');
+    for (const q of result.perQuestion) {
+        const status = q.correct ? 'CORRECT' : 'WRONG  ';
+        console.error(`  ${status} ${q.taskId.slice(0, 8)} | got="${q.got ?? 'null'}" exp="${q.expected}" [${q.aggregationMethod}]`);
+    }
+    console.error('\n=== VERDICT ===');
+    console.error(output.recommendation);
+    const passed = result.correct >= 4 && result.projectedCost53Q <= 40;
+    process.exit(passed ? 0 : 1);
+}
+function buildRecommendation(correct, total, cost53Q, cost300Q) {
+    const costOk = cost53Q <= 40;
+    if (correct >= 4 && costOk) {
+        return `PROCEED: ensemble viable. ${correct}/${total} accuracy (≥4/5) and cost $${cost53Q.toFixed(2)}/53Q (≤$40). Recommend iter 64 full 53-Q validation run.`;
+    }
+    if (correct >= 4 && !costOk) {
+        return `COST ISSUE: accuracy ${correct}/${total} good but projected 53-Q cost $${cost53Q.toFixed(2)} exceeds $40 gate. Reduce model count or use cheaper models.`;
+    }
+    if (correct < 4 && costOk) {
+        return `ACCURACY ISSUE: only ${correct}/${total} correct. Consider: (a) more models, (b) higher maxTurns, (c) OpenRouter credit top-up for GPT-5/DeepSeek. Do NOT proceed to full 53-Q.`;
+    }
+    return `BLOCKED: accuracy ${correct}/${total} below threshold AND cost $${cost53Q.toFixed(2)} above gate. Reassess architecture.`;
+}
+main().catch((err) => {
+    console.error('[pilot] FATAL:', err);
+    process.exit(1);
+});
+//# sourceMappingURL=gaia-ensemble-pilot.js.map

package/dist/src/benchmarks/gaia-ensemble-pilot.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gaia-ensemble-pilot.js","sourceRoot":"","sources":["../../../src/benchmarks/gaia-ensemble-pilot.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAE9B,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,kBAAkB,CAAC,CAAC;AAE1F,MAAM,kBAAkB,GAAG;IACzB,sCAAsC,EAAG,qBAAqB;IAC9D,sCAAsC,EAAG,qBAAqB;IAC9D,sCAAsC,EAAG,yBAAyB;IAClE,sCAAsC,EAAG,yBAAyB;IAClE,sCAAsC,EAAG,wBAAwB;CAClE,CAAC;AAEF,KAAK,UAAU,IAAI;IACjB,sBAAsB;IACtB,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACrF,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,mBAAmB,EAAE,gBAAgB,CAAC,CAAC;IAE1F,OAAO,CAAC,KAAK,CAAC,mBAAmB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACtD,OAAO,CAAC,KAAK,CAAC,sCAAsC,UAAU,EAAE,CAAC,CAAC;IAElE,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,KAAK,CAAC,oFAAoF,CAAC,CAAC;QACpG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,YAAY,GAAmB,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;IACtF,MAAM,cAAc,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAE1F,IAAI,cAAc,CAAC,MAAM,KAAK,kBAAkB,CAAC,MAAM,EAAE,CAAC;QACxD,OAAO,CAAC,KAAK,CAAC,6BAA6B,kBAAkB,CAAC,MAAM,qBAAqB,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC;IACpH,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,mBAAmB,cAAc,CAAC,MAAM,sBAAsB,MAAM,CAAC,MAAM,oBAAoB,CAAC,CAAC;IAE/G,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,cAAc,EAAE;QACpD,MAAM;QACN,QAAQ,EAAE,EAAE;QACZ,gBAAgB,EAAE,IAAI;QACtB,gBAAgB,EAAE,MAAM;KACzB,CAAC,CAAC;IAEH,8BAA8B;IAC9B,MAAM,MAAM,GAAG;QACb,WAAW,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,cAAc,CAAC,MAAM,EAAE;QAC7D,MAAM,EAAE;YACN,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,UAAU,EAAE,MAAM,CAAC,UAAU;SAC9B;QACD,QAAQ,EAAE;YACR,QAAQ,EAAE,EAAE;YACZ,SAAS,EAAE,GAAG;YACd,SAAS,EAAE,MAAM,CAAC,gBAAgB,IAAI,EAAE;YACxC,UAAU,EAAE,MAAM,CAAC,iBAAiB,IAAI,GAAG;SAC5C;QACD,cAAc,EAAE,mBAAmB,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,iBAAiB,CAAC;QACpH,WAAW,EAAE,MAAM,CAAC,WAAW;KAChC,CAAC;IAEF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAE7D,mCAAmC;IACnC,OAAO,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAClD,OAAO,CAAC,KAAK,CAAC,aAAa,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACtG,OAAO,CAAC,KAAK,CAAC,gBAAgB,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAChE,OAAO,CAAC,KAAK,CAAC,yBAAyB,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;IACzF,OAAO,CAAC,KAAK,CAAC,0BAA0B,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;IAC5F,OAAO,CAAC,KAAK,CAAC,mBAAmB,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC7E,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAElB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;QACjD,OAAO,CAAC,KAAK,CAAC,KAAK,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,GAAG,IAAI,MAAM,UAAU,CAAC,CAAC,QAAQ,MAAM,CAAC,CAAC,iBAAiB,GAAG,CAAC,CAAC;IAC/H,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;IACnC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IAErC,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,IAAI,MAAM,CAAC,gBAAgB,IAAI,EAAE,CAAC;IACpE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAe,EAAE,KAAa,EAAE,OAAe,EAAE,QAAgB;IAC5F,MAAM,MAAM,GAAG,OAAO,IAAI,EAAE,CAAC;IAC7B,IAAI,OAAO,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC;QAC3B,OAAO,6BAA6B,OAAO,IAAI,KAAK,8BAA8B,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,0DAA0D,CAAC;IACjK,CAAC;IACD,IAAI,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QAC5B,OAAO,wBAAwB,OAAO,IAAI,KAAK,kCAAkC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,8DAA8D,CAAC;IACpK,CAAC;IACD,IAAI,OAAO,GAAG,CAAC,IAAI,MAAM,EAAE,CAAC;QAC1B,OAAO,wBAAwB,OAAO,IAAI,KAAK,yIAAyI,CAAC;IAC3L,CAAC;IACD,OAAO,qBAAqB,OAAO,IAAI,KAAK,8BAA8B,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,qCAAqC,CAAC;AACpI,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC;IACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/src/benchmarks/gaia-ensemble.d.ts

@@ -0,0 +1,103 @@
+/**
+ * GAIA Ensemble Runner — ADR-139
+ *
+ * Runs a GAIA question through N models in parallel, aggregates answers via
+ * majority vote, with a judge-model tiebreak when no consensus is reached.
+ *
+ * Architecture:
+ *   1. Each model runs independently using the full tool harness.
+ *   2. Answers are normalised (via normaliseAnswer from gaia-judge.ts).
+ *   3. Majority vote: if ≥2 models agree on normalised answer → that wins.
+ *   4. Tiebreak: when all answers differ (or N=2 with disagreement), the judge
+ *      model picks the best answer with a brief rationale.
+ *   5. Abstain: if all models return null/timedOut → failed question.
+ *
+ * Supported models:
+ *   - Claude (claude-sonnet-4-6, etc.) via Anthropic API (gaia-agent.ts)
+ *   - Gemini (gemini-2.5-pro, etc.) via Google AI API (gaia-agent-gemini.js compiled)
+ *   - OpenRouter (gpt-5, deepseek-v3.2, kimi-k2, etc.) via OpenAI-compatible API
+ *     NOTE: OpenRouter requires funded account — returns 402 when credits exhausted.
+ *
+ * CLI integration:
+ *   gaia-bench run --mode=ensemble --models=claude-sonnet-4-6,gemini-2.5-pro,openai/gpt-5
+ *
+ * Cost model (per question, typical L1 ~10k input + ~3k output tokens):
+ *   claude-sonnet-4-6:  ~$0.075
+ *   gemini-2.5-pro:     ~$0.043
+ *   openai/gpt-5 (OR):  ~$0.043
+ *   3-model total:      ~$0.161  (53-Q: ~$8.54, 300-Q: ~$48.30)
+ *
+ * Refs: ADR-139, ADR-133, ADR-135, #2156
+ */
+import { GaiaQuestion } from './gaia-loader.js';
+export interface ModelRunResult {
+    model: string;
+    finalAnswer: string | null;
+    normalisedAnswer: string;
+    turns: number;
+    totalInputTokens: number;
+    totalOutputTokens: number;
+    wallMs: number;
+    estimatedCostUsd: number;
+    timedOut?: boolean;
+    error?: string;
+}
+export type AggregationMethod = 'majority' | 'judge-tiebreak' | 'abstain';
+export interface EnsembleResult {
+    questionId: string;
+    finalAnswer: string | null;
+    aggregationMethod: AggregationMethod;
+    /** Rationale from the judge tiebreak (only set when method is 'judge-tiebreak'). */
+    judgeRationale?: string;
+    models: ModelRunResult[];
+    totalInputTokens: number;
+    totalOutputTokens: number;
+    estimatedCostUsd: number;
+    wallMs: number;
+}
+export interface EnsembleOptions {
+    /** Models to use (provider inferred from model ID). */
+    models?: string[];
+    /** Judge model for tiebreak (default: claude-sonnet-4-6). */
+    judgeModel?: string;
+    /** Anthropic API key (resolved from env/gcloud if not supplied). */
+    anthropicApiKey?: string;
+    /** Google AI API key (resolved from env/gcloud if not supplied). */
+    geminiApiKey?: string;
+    /** OpenRouter API key (resolved from env/gcloud if not supplied). */
+    openrouterApiKey?: string;
+    /** Per-model max turns (default: 8). */
+    maxTurns?: number;
+    /** Per-model max tokens per turn (default: 2048). */
+    maxTokensPerTurn?: number;
+    /** Per-turn timeout in ms (default: 60 000). */
+    perTurnTimeoutMs?: number;
+}
+export declare function runEnsembleQuestion(question: GaiaQuestion, options?: EnsembleOptions): Promise<EnsembleResult>;
+export interface EnsemblePilotResult {
+    correct: number;
+    total: number;
+    accuracy: number;
+    perQuestion: Array<{
+        taskId: string;
+        question: string;
+        expected: string;
+        got: string | null;
+        correct: boolean;
+        aggregationMethod: AggregationMethod;
+        judgeRationale?: string;
+        costUsd: number;
+        wallMs: number;
+        perModel: Array<{
+            model: string;
+            answer: string | null;
+            costUsd: number;
+        }>;
+    }>;
+    totalCostUsd: number;
+    projectedCost53Q: number;
+    projectedCost300Q: number;
+    meanWallMs: number;
+}
+export declare function runEnsemblePilot(questions: GaiaQuestion[], options?: EnsembleOptions): Promise<EnsemblePilotResult>;
+//# sourceMappingURL=gaia-ensemble.d.ts.map

package/dist/src/benchmarks/gaia-ensemble.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gaia-ensemble.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/gaia-ensemble.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAmDhD,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,MAAM,iBAAiB,GAAG,UAAU,GAAG,gBAAgB,GAAG,SAAS,CAAC;AAE1E,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,oFAAoF;IACpF,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,cAAc,EAAE,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,uDAAuD;IACvD,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,6DAA6D;IAC7D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,oEAAoE;IACpE,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oEAAoE;IACpE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qEAAqE;IACrE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,wCAAwC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,qDAAqD;IACrD,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,gDAAgD;IAChD,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAmVD,wBAAsB,mBAAmB,CACvC,QAAQ,EAAE,YAAY,EACtB,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,cAAc,CAAC,CAiGzB;AAkBD,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,KAAK,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;QACnB,OAAO,EAAE,OAAO,CAAC;QACjB,iBAAiB,EAAE,iBAAiB,CAAC;QACrC,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,KAAK,CAAC;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KAC5E,CAAC,CAAC;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,wBAAsB,gBAAgB,CACpC,SAAS,EAAE,YAAY,EAAE,EACzB,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,mBAAmB,CAAC,CAiD9B"}