@claude-flow/cli 3.10.46 → 3.11.0

package/dist/src/benchmarks/gaia-codeagent.smoke.js

@@ -0,0 +1,241 @@
+/**
+ * GAIA CodeAgent Smoke Tests — ADR-138 iter 54 (FINAL design)
+ *
+ * 5 tests validating the CodeAgent harness end-to-end.
+ * Tests 1-4 are unit-level (no API calls). Test 5 uses 1 API call.
+ *
+ * Tests:
+ *   T1: Code block extraction — parser correctness (no subprocess)
+ *   T2: Python runner — simple math (subprocess, no API)
+ *   T3: Python runner — file read via ATTACHMENT_PATH
+ *   T4: Python runner — code error recovery (traceback returned as obs)
+ *   T5: End-to-end — simple factual question (1 API call, ~$0.01)
+ *
+ * Pass criteria: 5/5. Gate before iter 55 5Q run.
+ *
+ * Run: npx tsx gaia-codeagent.smoke.ts
+ *      or: node dist/src/benchmarks/gaia-codeagent.smoke.js
+ *
+ * Refs: ADR-138, #2156, iter 54
+ */
+// ---------------------------------------------------------------------------
+// Imports
+// ---------------------------------------------------------------------------
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { extractCodeBlock, runCodeAgentStep, runGaiaCodeAgent, } from './gaia-codeagent.js';
+import { resolveAnthropicApiKey } from './gaia-agent.js';
+function makeQ(overrides = {}) {
+    return {
+        task_id: 'smoke-test',
+        level: 1,
+        question: 'What is 2 + 2?',
+        final_answer: '4',
+        file_name: null,
+        file_path: null,
+        ...overrides,
+    };
+}
+// ---------------------------------------------------------------------------
+// T1: Code block extraction (no subprocess, no API)
+// ---------------------------------------------------------------------------
+function testCodeBlockExtraction() {
+    const name = 'T1: Code block extraction';
+    const start = Date.now();
+    const cases = [
+        { input: '```python\nprint(2+2)\n```', expected: 'print(2+2)' },
+        { input: '```py\nresult = 42\n```', expected: 'result = 42' },
+        { input: '```\nfinal_answer("test")\n```', expected: 'final_answer("test")' },
+        { input: 'No code block here', expected: null },
+        {
+            input: 'Reasoning...\n```python\nweb_search("q")\nprint(r)\n```\nMore',
+            expected: 'web_search("q")\nprint(r)',
+        },
+    ];
+    for (const { input, expected } of cases) {
+        const result = extractCodeBlock(input);
+        if (result !== expected) {
+            return {
+                name,
+                passed: false,
+                message: `Case "${input.slice(0, 40)}": got "${result}", expected "${expected}"`,
+                durationMs: Date.now() - start,
+            };
+        }
+    }
+    return { name, passed: true, message: `${cases.length}/5 cases passed`, durationMs: Date.now() - start };
+}
+// ---------------------------------------------------------------------------
+// T2: Python runner — simple math (subprocess, no API)
+// ---------------------------------------------------------------------------
+function testPythonRunnerMath() {
+    const name = 'T2: Python runner — simple math';
+    const start = Date.now();
+    const apiKey = process.env.ANTHROPIC_API_KEY ?? 'test-key';
+    const code = `result = 2 + 2\nprint(result)\nfinal_answer(str(result))`;
+    const stepResult = runCodeAgentStep(code, null, 15_000, apiKey);
+    if (stepResult.finalAnswer !== '4') {
+        return {
+            name,
+            passed: false,
+            message: `Expected finalAnswer "4", got "${stepResult.finalAnswer ?? 'null'}". Obs: ${stepResult.observation.slice(0, 200)}`,
+            durationMs: Date.now() - start,
+        };
+    }
+    return { name, passed: true, message: `finalAnswer="4" as expected`, durationMs: Date.now() - start };
+}
+// ---------------------------------------------------------------------------
+// T3: Python runner — file read via ATTACHMENT_PATH
+// ---------------------------------------------------------------------------
+function testPythonRunnerFileRead() {
+    const name = 'T3: Python runner — file read';
+    const start = Date.now();
+    const tmpFile = path.join(os.tmpdir(), `gaia-smoke-${Date.now()}.txt`);
+    fs.writeFileSync(tmpFile, 'The magic number is forty-two (42).', 'utf-8');
+    try {
+        const apiKey = process.env.ANTHROPIC_API_KEY ?? 'test-key';
+        const code = `content = read_file(ATTACHMENT_PATH)\nprint("Read:", content[:80])\nfinal_answer("42")`;
+        const stepResult = runCodeAgentStep(code, tmpFile, 15_000, apiKey);
+        if (stepResult.finalAnswer !== '42') {
+            return {
+                name, passed: false,
+                message: `Expected finalAnswer "42", got "${stepResult.finalAnswer ?? 'null'}". Obs: ${stepResult.observation.slice(0, 200)}`,
+                durationMs: Date.now() - start,
+            };
+        }
+        if (!stepResult.observation.includes('forty-two') && !stepResult.observation.includes('42')) {
+            return {
+                name, passed: false,
+                message: `File content not in observation. Obs: ${stepResult.observation.slice(0, 200)}`,
+                durationMs: Date.now() - start,
+            };
+        }
+        return { name, passed: true, message: `File read OK, finalAnswer="42"`, durationMs: Date.now() - start };
+    }
+    finally {
+        try {
+            fs.unlinkSync(tmpFile);
+        }
+        catch { /* ignore */ }
+    }
+}
+// ---------------------------------------------------------------------------
+// T4: Python runner — code error recovery
+// ---------------------------------------------------------------------------
+function testPythonRunnerErrorRecovery() {
+    const name = 'T4: Python runner — error recovery';
+    const start = Date.now();
+    const apiKey = process.env.ANTHROPIC_API_KEY ?? 'test-key';
+    const badCode = `result = undefined_variable_xyz + 1\nprint(result)`;
+    const stepResult = runCodeAgentStep(badCode, null, 15_000, apiKey);
+    if (stepResult.finalAnswer !== null) {
+        return {
+            name, passed: false,
+            message: `Expected no finalAnswer from errored code, got "${stepResult.finalAnswer}"`,
+            durationMs: Date.now() - start,
+        };
+    }
+    const obs = stepResult.observation.toLowerCase();
+    if (!obs.includes('error') && !obs.includes('nameerror')) {
+        return {
+            name, passed: false,
+            message: `Observation should contain error info, got: ${stepResult.observation.slice(0, 200)}`,
+            durationMs: Date.now() - start,
+        };
+    }
+    return {
+        name, passed: true,
+        message: `Error returned as observation: "${stepResult.observation.slice(0, 60)}..."`,
+        durationMs: Date.now() - start,
+    };
+}
+// ---------------------------------------------------------------------------
+// T5: End-to-end — simple factual question (1 API call)
+// ---------------------------------------------------------------------------
+async function testEndToEndSimple() {
+    const name = 'T5: End-to-end — simple factual (1 API call)';
+    const start = Date.now();
+    let apiKey;
+    try {
+        apiKey = resolveAnthropicApiKey();
+    }
+    catch (err) {
+        return {
+            name, passed: false,
+            message: `No API key: ${String(err)}`,
+            durationMs: Date.now() - start,
+        };
+    }
+    const q = makeQ({
+        task_id: 'smoke-e2e-01',
+        question: 'What is 6 multiplied by 7? Answer as a number.',
+        final_answer: '42',
+    });
+    try {
+        const result = await runGaiaCodeAgent(q, {
+            model: 'claude-sonnet-4-6',
+            maxTurns: 5,
+            apiKey,
+        });
+        if (result.error && !result.finalAnswer) {
+            return { name, passed: false, message: `Agent error: ${result.error}`, durationMs: Date.now() - start };
+        }
+        if (result.finalAnswer === null) {
+            return { name, passed: false, message: `No final answer (timedOut=${result.timedOut})`, durationMs: Date.now() - start };
+        }
+        const num = parseFloat(result.finalAnswer.replace(/,/g, ''));
+        if (Math.abs(num - 42) > 0.01) {
+            return {
+                name, passed: false,
+                message: `Expected "42", got "${result.finalAnswer}" (turns=${result.turns})`,
+                durationMs: Date.now() - start,
+            };
+        }
+        const estCost = (result.totalInputTokens / 1e6) * 3.0 + (result.totalOutputTokens / 1e6) * 15.0;
+        return {
+            name, passed: true,
+            message: `finalAnswer="${result.finalAnswer}" turns=${result.turns} cost~=$${estCost.toFixed(4)}`,
+            durationMs: Date.now() - start,
+        };
+    }
+    catch (err) {
+        return { name, passed: false, message: `Exception: ${String(err)}`, durationMs: Date.now() - start };
+    }
+}
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+async function main() {
+    console.log('\n=== GAIA CodeAgent Smoke Tests (ADR-138 iter 54) ===\n');
+    const results = [
+        testCodeBlockExtraction(),
+        testPythonRunnerMath(),
+        testPythonRunnerFileRead(),
+        testPythonRunnerErrorRecovery(),
+        await testEndToEndSimple(),
+    ];
+    let passed = 0;
+    for (const r of results) {
+        const status = r.passed ? 'PASS' : 'FAIL';
+        console.log(`[${status}] ${r.name} (${r.durationMs}ms)`);
+        console.log(`       ${r.message}`);
+        if (r.passed)
+            passed++;
+    }
+    console.log(`\n=== ${passed}/${results.length} passed ===`);
+    if (passed === results.length) {
+        console.log('All smoke tests passed. CodeAgent harness ready for 1Q sanity check.\n');
+        process.exit(0);
+    }
+    else {
+        console.log(`SMOKE FAILED — ${results.length - passed} test(s) failed.\n`);
+        process.exit(1);
+    }
+}
+main().catch((err) => {
+    console.error('Smoke test crashed:', err);
+    process.exit(2);
+});
+export { testCodeBlockExtraction, testPythonRunnerMath, testPythonRunnerFileRead, testPythonRunnerErrorRecovery, testEndToEndSimple, };
+//# sourceMappingURL=gaia-codeagent.smoke.js.map

package/dist/src/benchmarks/gaia-codeagent.smoke.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gaia-codeagent.smoke.js","sourceRoot":"","sources":["../../../src/benchmarks/gaia-codeagent.smoke.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,GAEjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AAazD,SAAS,KAAK,CAAC,YAAmC,EAAE;IAClD,OAAO;QACL,OAAO,EAAE,YAAY;QACrB,KAAK,EAAE,CAAU;QACjB,QAAQ,EAAE,gBAAgB;QAC1B,YAAY,EAAE,GAAG;QACjB,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,IAAI;QACf,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,oDAAoD;AACpD,8EAA8E;AAE9E,SAAS,uBAAuB;IAC9B,MAAM,IAAI,GAAG,2BAA2B,CAAC;IACzC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEzB,MAAM,KAAK,GAAsD;QAC/D,EAAE,KAAK,EAAE,4BAA4B,EAAE,QAAQ,EAAE,YAAY,EAAE;QAC/D,EAAE,KAAK,EAAE,yBAAyB,EAAE,QAAQ,EAAE,aAAa,EAAE;QAC7D,EAAE,KAAK,EAAE,gCAAgC,EAAE,QAAQ,EAAE,sBAAsB,EAAE;QAC7E,EAAE,KAAK,EAAE,oBAAoB,EAAE,QAAQ,EAAE,IAAI,EAAE;QAC/C;YACE,KAAK,EAAE,+DAA+D;YACtE,QAAQ,EAAE,2BAA2B;SACtC;KACF,CAAC;IAEF,KAAK,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,KAAK,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;YACxB,OAAO;gBACL,IAAI;gBACJ,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,SAAS,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,MAAM,gBAAgB,QAAQ,GAAG;gBAChF,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;aAC/B,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC,MAAM,iBAAiB,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;AAC3G,CAAC;AAED,8EAA8E;AAC9E,uDAAuD;AACvD,8EAA8E;AAE9E,SAAS,oBAAoB;IAC3B,MAAM,IAAI,GAAG,iCAAiC,CAAC;IAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEzB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,UAAU,CAAC;IAC3D,MAAM,IAAI,GAAG,0DAA0D,CAAC;IACxE,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAEhE,IAAI,UAAU,CAAC,WAAW,KAAK,GAAG,EAAE,CAAC;QACnC,OAAO;YACL,IAAI;YACJ,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,kCAAkC,UAAU,CAAC,WAAW,IAAI,MAAM,WAAW,UAAU,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;YAC5H,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;SAC/B,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,6BAA6B,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;AACxG,CAAC;AAED,8EAA8E;AAC9E,oDAAoD;AACpD,8EAA8E;AAE9E,SAAS,wBAAwB;IAC/B,MAAM,IAAI,GAAG,+BAA+B,CAAC;IAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,cAAc,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACvE,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,qCAAqC,EAAE,OAAO,CAAC,CAAC;IAE1E,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,UAAU,CAAC;QAC3D,MAAM,IAAI,GAAG,wFAAwF,CAAC;QACtG,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAEnE,IAAI,UAAU,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;YACpC,OAAO;gBACL,IAAI,EAAE,MAAM,EAAE,KAAK;gBACnB,OAAO,EAAE,mCAAmC,UAAU,CAAC,WAAW,IAAI,MAAM,WAAW,UAAU,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;gBAC7H,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;aAC/B,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5F,OAAO;gBACL,IAAI,EAAE,MAAM,EAAE,KAAK;gBACnB,OAAO,EAAE,yCAAyC,UAAU,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;gBACxF,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;aAC/B,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,gCAAgC,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;IAC3G,CAAC;YAAS,CAAC;QACT,IAAI,CAAC;YAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IACxD,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,0CAA0C;AAC1C,8EAA8E;AAE9E,SAAS,6BAA6B;IACpC,MAAM,IAAI,GAAG,oCAAoC,CAAC;IAClD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEzB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,UAAU,CAAC;IAC3D,MAAM,OAAO,GAAG,oDAAoD,CAAC;IACrE,MAAM,UAAU,GAAG,gBAAgB,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAEnE,IAAI,UAAU,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;QACpC,OAAO;YACL,IAAI,EAAE,MAAM,EAAE,KAAK;YACnB,OAAO,EAAE,mDAAmD,UAAU,CAAC,WAAW,GAAG;YACrF,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;SAC/B,CAAC;IACJ,CAAC;IACD,MAAM,GAAG,GAAG,UAAU,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;IACjD,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QACzD,OAAO;YACL,IAAI,EAAE,MAAM,EAAE,KAAK;YACnB,OAAO,EAAE,+CAA+C,UAAU,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;YAC9F,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;SAC/B,CAAC;IACJ,CAAC;IACD,OAAO;QACL,IAAI,EAAE,MAAM,EAAE,IAAI;QAClB,OAAO,EAAE,mCAAmC,UAAU,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM;QACrF,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;KAC/B,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,wDAAwD;AACxD,8EAA8E;AAE9E,KAAK,UAAU,kBAAkB;IAC/B,MAAM,IAAI,GAAG,8CAA8C,CAAC;IAC5D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEzB,IAAI,MAAc,CAAC;IACnB,IAAI,CAAC;QACH,MAAM,GAAG,sBAAsB,EAAE,CAAC;IACpC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,IAAI,EAAE,MAAM,EAAE,KAAK;YACnB,OAAO,EAAE,eAAe,MAAM,CAAC,GAAG,CAAC,EAAE;YACrC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;SAC/B,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,GAAG,KAAK,CAAC;QACd,OAAO,EAAE,cAAc;QACvB,QAAQ,EAAE,gDAAgD;QAC1D,YAAY,EAAE,IAAI;KACnB,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,CAAC,EAAE;YACvC,KAAK,EAAE,mBAAmB;YAC1B,QAAQ,EAAE,CAAC;YACX,MAAM;SACP,CAAC,CAAC;QAEH,IAAI,MAAM,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YACxC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,gBAAgB,MAAM,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;QAC1G,CAAC;QACD,IAAI,MAAM,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;YAChC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,6BAA6B,MAAM,CAAC,QAAQ,GAAG,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;QAC3H,CAAC;QACD,MAAM,GAAG,GAAG,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;QAC7D,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC;YAC9B,OAAO;gBACL,IAAI,EAAE,MAAM,EAAE,KAAK;gBACnB,OAAO,EAAE,uBAAuB,MAAM,CAAC,WAAW,YAAY,MAAM,CAAC,KAAK,GAAG;gBAC7E,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;aAC/B,CAAC;QACJ,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,gBAAgB,GAAG,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,iBAAiB,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC;QAChG,OAAO;YACL,IAAI,EAAE,MAAM,EAAE,IAAI;YAClB,OAAO,EAAE,gBAAgB,MAAM,CAAC,WAAW,WAAW,MAAM,CAAC,KAAK,WAAW,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;YACjG,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;SAC/B,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;IACvG,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,OAAO;AACP,8EAA8E;AAE9E,KAAK,UAAU,IAAI;IACjB,OAAO,CAAC,GAAG,CAAC,0DAA0D,CAAC,CAAC;IAExE,MAAM,OAAO,GAAiB;QAC5B,uBAAuB,EAAE;QACzB,oBAAoB,EAAE;QACtB,wBAAwB,EAAE;QAC1B,6BAA6B,EAAE;QAC/B,MAAM,kBAAkB,EAAE;KAC3B,CAAC;IAEF,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,MAAM,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC;QACzD,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QACnC,IAAI,CAAC,CAAC,MAAM;YAAE,MAAM,EAAE,CAAC;IACzB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,IAAI,OAAO,CAAC,MAAM,aAAa,CAAC,CAAC;IAC5D,IAAI,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,wEAAwE,CAAC,CAAC;QACtF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,kBAAkB,OAAO,CAAC,MAAM,GAAG,MAAM,oBAAoB,CAAC,CAAC;QAC3E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,GAAG,CAAC,CAAC;IAC1C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC;AAEH,OAAO,EACL,uBAAuB,EACvB,oBAAoB,EACpB,wBAAwB,EACxB,6BAA6B,EAC7B,kBAAkB,GACnB,CAAC"}

package/dist/src/benchmarks/gaia-critic.js

@@ -70,25 +70,25 @@ function resolveApiKey(override) {
  * Instructs Sonnet to act as an adversarial reviewer and respond in JSON.
  */
 function buildCriticSystemPrompt() {
-    return `You are an adversarial reviewer of agent answers on the GAIA benchmark. \
-Your job is to find flaws in candidate answers before they are submitted.
-
-Given a question, a candidate answer, and a summary of the agent's reasoning \
-trajectory, you must decide whether the candidate answer is correct.
-
-Evaluation criteria:
-1. Does the answer directly address what the question asks?
-2. Is there evidence in the trajectory that supports the answer?
-3. Are there obvious flaws (wrong unit, wrong format, missed constraint, \
-fabricated source, off-by-one error, truncation)?
-4. For numeric answers: is the magnitude and unit plausible?
-5. For list answers: are all required items present and correctly ordered?
-
-Respond ONLY with a JSON object — no markdown, no prose outside the JSON:
-{"verdict":"pass"|"fail"|"uncertain","reasoning":"<1-3 sentences>","suggestedRevision":"<corrected answer or empty string>"}
-
-Use "uncertain" only when you genuinely cannot determine correctness from the \
-available information. Never use "uncertain" to avoid making a decision when \
+    return `You are an adversarial reviewer of agent answers on the GAIA benchmark. \
+Your job is to find flaws in candidate answers before they are submitted.
+
+Given a question, a candidate answer, and a summary of the agent's reasoning \
+trajectory, you must decide whether the candidate answer is correct.
+
+Evaluation criteria:
+1. Does the answer directly address what the question asks?
+2. Is there evidence in the trajectory that supports the answer?
+3. Are there obvious flaws (wrong unit, wrong format, missed constraint, \
+fabricated source, off-by-one error, truncation)?
+4. For numeric answers: is the magnitude and unit plausible?
+5. For list answers: are all required items present and correctly ordered?
+
+Respond ONLY with a JSON object — no markdown, no prose outside the JSON:
+{"verdict":"pass"|"fail"|"uncertain","reasoning":"<1-3 sentences>","suggestedRevision":"<corrected answer or empty string>"}
+
+Use "uncertain" only when you genuinely cannot determine correctness from the \
+available information. Never use "uncertain" to avoid making a decision when \
 evidence is available.`;
 }
 /**
@@ -108,11 +108,11 @@ function buildCriticUserMessage(question, candidateAnswer, trajectory) {
             const result = s.result ? ` → ${s.result.slice(0, 200)}` : '';
             return `${label}${result}`;
         }).join('\n');
-    return `QUESTION: ${question.question}
-
-CANDIDATE ANSWER: ${candidateAnswer}
-
-AGENT TRAJECTORY (${trajectory.turns} turns, summarised):
+    return `QUESTION: ${question.question}
+
+CANDIDATE ANSWER: ${candidateAnswer}
+
+AGENT TRAJECTORY (${trajectory.turns} turns, summarised):
 ${trajectoryText}`;
 }
 /**

package/dist/src/benchmarks/gaia-dag-pilot.d.ts

@@ -0,0 +1,22 @@
+/**
+ * GAIA DAG 5-Question Pilot Runner — iter 64
+ *
+ * Runs 5 specific GAIA L1 questions through the Co-Sight DAG harness
+ * and compares results against single-Sonnet baseline (from iter 63b).
+ *
+ * Pilot question mix (selected from iter 63b failures):
+ *   1. 5d0080cb — calculation (fish bag volume from academic paper, multi-hop)
+ *   2. cffe0e32 — reasoning puzzle (Secret Santa assignment chain)
+ *   3. ec09fa32 — riddle (game show, requires careful reasoning)
+ *   4. 46719c30 — retrieval (paper authors -> their other publications)
+ *   5. b816bfce — retrieval+reasoning (journal name from Norse mythology)
+ *
+ * Cost cap: $2.00
+ *
+ * Usage:
+ *   node dist/src/benchmarks/gaia-dag-pilot.js
+ *
+ * Refs: ADR-139, iter 64, #2156
+ */
+export {};
+//# sourceMappingURL=gaia-dag-pilot.d.ts.map

package/dist/src/benchmarks/gaia-dag-pilot.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gaia-dag-pilot.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/gaia-dag-pilot.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG"}

package/dist/src/benchmarks/gaia-dag-pilot.js

@@ -0,0 +1,158 @@
+/**
+ * GAIA DAG 5-Question Pilot Runner — iter 64
+ *
+ * Runs 5 specific GAIA L1 questions through the Co-Sight DAG harness
+ * and compares results against single-Sonnet baseline (from iter 63b).
+ *
+ * Pilot question mix (selected from iter 63b failures):
+ *   1. 5d0080cb — calculation (fish bag volume from academic paper, multi-hop)
+ *   2. cffe0e32 — reasoning puzzle (Secret Santa assignment chain)
+ *   3. ec09fa32 — riddle (game show, requires careful reasoning)
+ *   4. 46719c30 — retrieval (paper authors -> their other publications)
+ *   5. b816bfce — retrieval+reasoning (journal name from Norse mythology)
+ *
+ * Cost cap: $2.00
+ *
+ * Usage:
+ *   node dist/src/benchmarks/gaia-dag-pilot.js
+ *
+ * Refs: ADR-139, iter 64, #2156
+ */
+import { loadGaia } from './gaia-loader.js';
+import { runDagPilot } from './gaia-dag.js';
+// ---------------------------------------------------------------------------
+// Pilot question IDs (5 questions from iter 63b failures)
+// ---------------------------------------------------------------------------
+const PILOT_TASK_IDS = [
+    '5d0080cb-90d7-4712-bc33-848150e917d3', // calculation: fish bag volume
+    'cffe0e32-c9a6-4c52-9877-78ceb4aaa9fb', // reasoning: Secret Santa
+    'ec09fa32-d03f-4bf8-84b0-1f16922c3ae4', // riddle: game show bees
+    '46719c30-f4c3-4cad-be07-d5cb21eee6bb', // retrieval: paper authors + their work
+    'b816bfce-3d80-4913-a07d-69b752ce6377', // retrieval: journal from Norse mythology
+];
+// Baseline: single-Sonnet results from iter 63b
+const BASELINE = {
+    '5d0080cb-90d7-4712-bc33-848150e917d3': { correct: false, answer: null },
+    'cffe0e32-c9a6-4c52-9877-78ceb4aaa9fb': { correct: false, answer: null },
+    'ec09fa32-d03f-4bf8-84b0-1f16922c3ae4': { correct: false, answer: null },
+    '46719c30-f4c3-4cad-be07-d5cb21eee6bb': { correct: false, answer: null },
+    'b816bfce-3d80-4913-a07d-69b752ce6377': { correct: false, answer: 'cuddly' }, // close but wrong
+};
+const COST_CAP_USD = 2.00;
+async function main() {
+    console.log('\n=== GAIA DAG 5-Question Pilot (iter 64) ===\n');
+    console.log('Questions: 5 (from iter 63b failures)');
+    console.log(`Planner: ${process.env['PLAN_MODEL'] ?? 'claude-sonnet-4-6'}`);
+    console.log(`Actor:   ${process.env['ACT_MODEL'] ?? 'gemini-2.5-pro'}`);
+    console.log(`Cost cap: $${COST_CAP_USD}\n`);
+    // Load full GAIA L1 dataset and filter to pilot questions
+    console.log('Loading GAIA L1 dataset...');
+    let allQuestions;
+    try {
+        allQuestions = await loadGaia({ level: 1 });
+    }
+    catch (err) {
+        console.error('Failed to load GAIA dataset:', err instanceof Error ? err.message : String(err));
+        process.exit(1);
+    }
+    const pilotQuestions = PILOT_TASK_IDS.map((id) => {
+        const q = allQuestions.find((q) => q.task_id === id);
+        if (!q) {
+            console.warn(`WARNING: Question ${id} not found in dataset`);
+        }
+        return q;
+    }).filter(Boolean);
+    if (pilotQuestions.length === 0) {
+        console.error('No pilot questions found. Check HF token and dataset availability.');
+        process.exit(1);
+    }
+    console.log(`Loaded ${pilotQuestions.length} pilot questions\n`);
+    for (const q of pilotQuestions) {
+        console.log(`  ${q.task_id.slice(0, 8)}: ${q.question.slice(0, 70)}...`);
+        console.log(`           expected: "${q.final_answer}"`);
+        if (q.file_path)
+            console.log(`           file: ${q.file_path}`);
+    }
+    console.log('');
+    // Run DAG pilot
+    const result = await runDagPilot(pilotQuestions, {});
+    // Check cost cap
+    if (result.totalCostUsd > COST_CAP_USD) {
+        console.warn(`\nWARNING: Cost $${result.totalCostUsd.toFixed(4)} exceeded cap $${COST_CAP_USD}\n`);
+    }
+    // Report results
+    console.log('\n=== DAG Pilot Results ===\n');
+    console.log(`Score: ${result.correct}/${result.total} (${(result.accuracy * 100).toFixed(1)}%)`);
+    console.log(`Avg steps/question: ${result.avgStepsPerQuestion.toFixed(1)}`);
+    console.log(`Total cost: $${result.totalCostUsd.toFixed(4)}`);
+    console.log(`Projected 53Q cost: $${result.projectedCost53Q.toFixed(2)}`);
+    console.log(`Mean wall time: ${(result.meanWallMs / 1000).toFixed(1)}s\n`);
+    console.log('Per-question breakdown:');
+    const baselineCorrect = Object.values(BASELINE).filter((b) => b.correct).length;
+    let dagBetter = 0;
+    let dagWorse = 0;
+    for (const q of result.perQuestion) {
+        const baseline = BASELINE[q.taskId];
+        const baselineResult = baseline?.correct ? 'PASS' : 'FAIL';
+        const dagResult = q.correct ? 'PASS' : 'FAIL';
+        const change = !baseline?.correct && q.correct ? ' (+RECOVERED)' :
+            baseline?.correct && !q.correct ? ' (-REGRESSED)' : '';
+        if (!baseline?.correct && q.correct)
+            dagBetter++;
+        if (baseline?.correct && !q.correct)
+            dagWorse++;
+        console.log(`  ${dagResult} [was ${baselineResult}]${change} ${q.taskId.slice(0, 8)}: got="${q.got?.slice(0, 40) ?? 'null'}" expected="${q.expected.slice(0, 40)}"`);
+        console.log(`         steps=${q.steps} (completed=${q.completedSteps}, blocked=${q.blockedSteps}) plannerCycles=${q.plannerCycles} cost=$${q.costUsd.toFixed(4)} wall=${(q.wallMs / 1000).toFixed(1)}s`);
+    }
+    console.log(`\nComparison vs single-Sonnet (iter 63b):`);
+    console.log(`  Baseline: ${baselineCorrect}/5 (${(baselineCorrect * 20).toFixed(0)}%)`);
+    console.log(`  DAG:      ${result.correct}/5 (${(result.accuracy * 100).toFixed(1)}%)`);
+    console.log(`  Recovered: ${dagBetter} questions (single-Sonnet failed, DAG solved)`);
+    console.log(`  Regressed: ${dagWorse} questions (single-Sonnet passed, DAG failed)`);
+    console.log('\n=== Gate Verdict ===');
+    const multiStepExecuted = result.perQuestion.some((q) => q.steps > 1);
+    const avgSteps = result.avgStepsPerQuestion;
+    if (result.correct >= 4 && multiStepExecuted) {
+        console.log(`PROCEED to iter 65 full 53Q run.`);
+        console.log(`  Reason: DAG ${result.correct}/5, avg ${avgSteps.toFixed(1)} steps/Q, multi-step plans executing.`);
+    }
+    else if (result.correct >= 3 && avgSteps >= 2) {
+        console.log(`PROCEED with CAUTION to iter 65 — diagnose blocking first.`);
+        console.log(`  Reason: DAG ${result.correct}/5, avg ${avgSteps.toFixed(1)} steps/Q.`);
+    }
+    else {
+        console.log(`DIAGNOSE before iter 65.`);
+        console.log(`  Reason: DAG ${result.correct}/5, avg ${avgSteps.toFixed(1)} steps/Q.`);
+        if (!multiStepExecuted) {
+            console.log(`  Issue: Plans collapsed to single step — check planner prompt.`);
+        }
+    }
+    // Write JSON result
+    const jsonOut = {
+        run_type: 'dag-5q-pilot',
+        iter: 64,
+        timestamp: new Date().toISOString(),
+        planModel: process.env['PLAN_MODEL'] ?? 'claude-sonnet-4-6',
+        actModel: process.env['ACT_MODEL'] ?? 'gemini-2.5-pro',
+        summary: {
+            total: result.total,
+            correct: result.correct,
+            accuracy: result.accuracy,
+            avgStepsPerQuestion: result.avgStepsPerQuestion,
+            totalCostUsd: result.totalCostUsd,
+            projectedCost53Q: result.projectedCost53Q,
+            meanWallMs: result.meanWallMs,
+        },
+        baseline: { correct: baselineCorrect, total: 5, source: 'iter-63b' },
+        perQuestion: result.perQuestion,
+    };
+    const outPath = `/Users/cohen/Projects/ruflo/docs/benchmarks/runs/gaia-l1-iter64-dag-5q-pilot.json`;
+    const fs = await import('node:fs');
+    fs.writeFileSync(outPath, JSON.stringify(jsonOut, null, 2));
+    console.log(`\nResults written to: ${outPath}\n`);
+}
+main().catch((err) => {
+    console.error('Pilot failed:', err);
+    process.exit(1);
+});
+//# sourceMappingURL=gaia-dag-pilot.js.map

package/dist/src/benchmarks/gaia-dag-pilot.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gaia-dag-pilot.js","sourceRoot":"","sources":["../../../src/benchmarks/gaia-dag-pilot.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAkB,MAAM,eAAe,CAAC;AAG5D,8EAA8E;AAC9E,0DAA0D;AAC1D,8EAA8E;AAE9E,MAAM,cAAc,GAAG;IACrB,sCAAsC,EAAE,+BAA+B;IACvE,sCAAsC,EAAE,0BAA0B;IAClE,sCAAsC,EAAE,yBAAyB;IACjE,sCAAsC,EAAE,wCAAwC;IAChF,sCAAsC,EAAE,0CAA0C;CACnF,CAAC;AAEF,gDAAgD;AAChD,MAAM,QAAQ,GAAgE;IAC5E,sCAAsC,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE;IACxE,sCAAsC,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE;IACxE,sCAAsC,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE;IACxE,sCAAsC,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE;IACxE,sCAAsC,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,kBAAkB;CACjG,CAAC;AAEF,MAAM,YAAY,GAAG,IAAI,CAAC;AAE1B,KAAK,UAAU,IAAI;IACjB,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC;IAC/D,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,mBAAmB,EAAE,CAAC,CAAC;IAC5E,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,gBAAgB,EAAE,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,cAAc,YAAY,IAAI,CAAC,CAAC;IAE5C,0DAA0D;IAC1D,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;IAC1C,IAAI,YAAY,CAAC;IACjB,IAAI,CAAC;QACH,YAAY,GAAG,MAAM,QAAQ,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAC9C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,8BAA8B,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAChG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAC/C,MAAM,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,qBAAqB,EAAE,uBAAuB,CAAC,CAAC;QAC/D,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAwB,CAAC;IAE1C,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,CAAC,KAAK,CAAC,oEAAoE,CAAC,CAAC;QACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,UAAU,cAAc,CAAC,MAAM,oBAAoB,CAAC,CAAC;IACjE,KAAK,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;QACzE,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC;QACxD,IAAI,CAAC,CAAC,SAAS;YAAE,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEhB,gBAAgB;IAChB,MAAM,MAAM,GAAmB,MAAM,WAAW,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;IAErE,iBAAiB;IACjB,IAAI,MAAM,CAAC,YAAY,GAAG,YAAY,EAAE,CAAC;QACvC,OAAO,CAAC,IAAI,CAAC,oBAAoB,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB,YAAY,IAAI,CAAC,CAAC;IACrG,CAAC;IAED,iBAAiB;IACjB,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;IAC7C,OAAO,CAAC,GAAG,CAAC,UAAU,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACjG,OAAO,CAAC,GAAG,CAAC,uBAAuB,MAAM,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC5E,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC9D,OAAO,CAAC,GAAG,CAAC,wBAAwB,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC1E,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAE3E,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;IACvC,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAChF,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACpC,MAAM,cAAc,GAAG,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAC3D,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAC9C,MAAM,MAAM,GAAG,CAAC,QAAQ,EAAE,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;YACnD,QAAQ,EAAE,OAAO,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC;QACtE,IAAI,CAAC,QAAQ,EAAE,OAAO,IAAI,CAAC,CAAC,OAAO;YAAE,SAAS,EAAE,CAAC;QACjD,IAAI,QAAQ,EAAE,OAAO,IAAI,CAAC,CAAC,CAAC,OAAO;YAAE,QAAQ,EAAE,CAAC;QAEhD,OAAO,CAAC,GAAG,CAAC,KAAK,SAAS,SAAS,cAAc,IAAI,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,eAAe,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;QACrK,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,KAAK,eAAe,CAAC,CAAC,cAAc,aAAa,CAAC,CAAC,YAAY,mBAAmB,CAAC,CAAC,aAAa,UAAU,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC3M,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC;IACzD,OAAO,CAAC,GAAG,CAAC,eAAe,eAAe,OAAO,CAAC,eAAe,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACxF,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,OAAO,OAAO,CAAC,MAAM,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACxF,OAAO,CAAC,GAAG,CAAC,gBAAgB,SAAS,+CAA+C,CAAC,CAAC;IACtF,OAAO,CAAC,GAAG,CAAC,gBAAgB,QAAQ,+CAA+C,CAAC,CAAC;IAErF,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IACtC,MAAM,iBAAiB,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IACtE,MAAM,QAAQ,GAAG,MAAM,CAAC,mBAAmB,CAAC;IAE5C,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,IAAI,iBAAiB,EAAE,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;QAChD,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,OAAO,WAAW,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,uCAAuC,CAAC,CAAC;IACpH,CAAC;SAAM,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;QAChD,OAAO,CAAC,GAAG,CAAC,4DAA4D,CAAC,CAAC;QAC1E,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,OAAO,WAAW,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IACxF,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;QACxC,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,OAAO,WAAW,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QACtF,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,iEAAiE,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IAED,oBAAoB;IACpB,MAAM,OAAO,GAAG;QACd,QAAQ,EAAE,cAAc;QACxB,IAAI,EAAE,EAAE;QACR,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,mBAAmB;QAC3D,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,gBAAgB;QACtD,OAAO,EAAE;YACP,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;YAC/C,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,UAAU,EAAE,MAAM,CAAC,UAAU;SAC9B;QACD,QAAQ,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE;QACpE,WAAW,EAAE,MAAM,CAAC,WAAW;KAChC,CAAC;IAEF,MAAM,OAAO,GAAG,mFAAmF,CAAC;IACpG,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACnC,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5D,OAAO,CAAC,GAAG,CAAC,yBAAyB,OAAO,IAAI,CAAC,CAAC;AACpD,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;IACpC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/src/benchmarks/gaia-dag.d.ts

@@ -0,0 +1,110 @@
+/**
+ * GAIA DAG Harness — Co-Sight Architecture Port (ADR-139 Addendum)
+ *
+ * Ports the ZTE-AICloud/Co-Sight DAG orchestration pattern (Apache 2.0,
+ * arXiv 2510.21557) into the ruflo GAIA harness.
+ *
+ * Architecture:
+ *   1. PLAN  — Claude Sonnet 4.6 reads the question and emits a DAG of 3-7
+ *              steps: {id, description, depends_on: [], suggested_tool}.
+ *              Claude-aware prompt: "3-5 steps max, direct answer when clear."
+ *   2. EXECUTE — Loop while ready steps exist (deps satisfied).
+ *              Run all ready steps in PARALLEL (Promise.all, cap ≤5).
+ *              Each step = a Gemini 2.5 Pro actor with the full tool suite.
+ *              Actor marks step completed/blocked + writes step_notes.
+ *              Blocked steps trigger planner re_plan before next cycle.
+ *   3. FINALIZE — Planner reads all step_notes → produces final answer
+ *              using T2 extraction cascade from gaia-agent.ts.
+ *   4. CAMV   — Async credibility labeling per step (stubbed, iter 65).
+ *
+ * Role assignment (env-configurable):
+ *   PLAN_MODEL  = claude-sonnet-4-6  (default)
+ *   ACT_MODEL   = gemini-2.5-pro     (default)
+ *   VISION_MODEL = gemini-2.5-pro    (default, same as act)
+ *
+ * CLI:
+ *   gaia-bench run --mode=dag --model claude-sonnet-4-6
+ *
+ * Cost: planner ~$0.02/Q + actors ~$0.03/Q = ~$0.05/Q (vs single-Sonnet ~$0.075)
+ *
+ * Refs: ADR-139, github.com/ZTE-AICloud/Co-Sight, arXiv 2510.21557, #2156
+ */
+import { GaiaQuestion } from './gaia-loader.js';
+import { GaiaToolCatalogue } from './gaia-tools/index.js';
+export interface DagStep {
+    id: number;
+    description: string;
+    depends_on: number[];
+    suggested_tool?: string;
+    status: 'not_started' | 'in_progress' | 'completed' | 'blocked';
+    step_notes: string;
+}
+export interface DagPlan {
+    title: string;
+    question: string;
+    steps: DagStep[];
+}
+/**
+ * Get all steps whose dependencies are fully satisfied (completed).
+ * Mirrors Co-Sight's Plan.get_ready_steps().
+ */
+export declare function getReadySteps(plan: DagPlan): DagStep[];
+export interface DagResult {
+    questionId: string;
+    finalAnswer: string | null;
+    normalisedAnswer: string;
+    plan: DagPlan;
+    totalSteps: number;
+    completedSteps: number;
+    blockedSteps: number;
+    plannerCycles: number;
+    totalInputTokens: number;
+    totalOutputTokens: number;
+    estimatedCostUsd: number;
+    wallMs: number;
+    timedOut?: boolean;
+    error?: string;
+}
+export interface DagOptions {
+    planModel?: string;
+    actModel?: string;
+    anthropicApiKey?: string;
+    geminiApiKey?: string;
+    catalogue?: GaiaToolCatalogue;
+    /** Disable CAMV stub (default: false — stub is always disabled in this iter). */
+    enableCamv?: boolean;
+}
+/**
+ * Run a GAIA question through the Co-Sight DAG harness.
+ *
+ * Steps:
+ *   1. Planner (Claude Sonnet) creates a DAG plan.
+ *   2. Execute loop: parallel actors (Gemini 2.5 Pro) run ready steps.
+ *   3. Blocked steps trigger replan (up to MAX_REPLAN_CYCLES).
+ *   4. Finalizer (Claude Sonnet) reads all step notes → final answer.
+ */
+export declare function runGaiaDAG(question: GaiaQuestion, options?: DagOptions): Promise<DagResult>;
+export interface DagPilotResult {
+    correct: number;
+    total: number;
+    accuracy: number;
+    avgStepsPerQuestion: number;
+    perQuestion: Array<{
+        taskId: string;
+        question: string;
+        expected: string;
+        got: string | null;
+        correct: boolean;
+        steps: number;
+        completedSteps: number;
+        blockedSteps: number;
+        plannerCycles: number;
+        costUsd: number;
+        wallMs: number;
+    }>;
+    totalCostUsd: number;
+    projectedCost53Q: number;
+    meanWallMs: number;
+}
+export declare function runDagPilot(questions: GaiaQuestion[], options?: DagOptions): Promise<DagPilotResult>;
+//# sourceMappingURL=gaia-dag.d.ts.map

package/dist/src/benchmarks/gaia-dag.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gaia-dag.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/gaia-dag.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAKH,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAEL,iBAAiB,EAKlB,MAAM,uBAAuB,CAAC;AAiD/B,MAAM,WAAW,OAAO;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,aAAa,GAAG,aAAa,GAAG,WAAW,GAAG,SAAS,CAAC;IAChE,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,OAAO,EAAE,CAAC;CAClB;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,OAAO,GAAG,OAAO,EAAE,CAQtD;AAygBD,MAAM,WAAW,SAAS;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,IAAI,EAAE,OAAO,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,iFAAiF;IACjF,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AA6BD;;;;;;;;GAQG;AACH,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,YAAY,EACtB,OAAO,GAAE,UAAe,GACvB,OAAO,CAAC,SAAS,CAAC,CA6IpB;AAMD,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,WAAW,EAAE,KAAK,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;QACnB,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,wBAAsB,WAAW,CAC/B,SAAS,EAAE,YAAY,EAAE,EACzB,OAAO,GAAE,UAAe,GACvB,OAAO,CAAC,cAAc,CAAC,CA6CzB"}