@claude-flow/cli 3.10.3 → 3.10.5

package/dist/src/benchmarks/gaia-critic.smoke.js

@@ -0,0 +1,327 @@
+/**
+ * GAIA Critic Smoke Tests — ADR-135 Track D
+ *
+ * Tests for the adversarial critic agent in gaia-critic.ts.
+ * All tests use mocked responses — NO live API calls.
+ *
+ * Test coverage:
+ *  1. Critic returns "pass" → no retry, immediately returns candidate
+ *  2. Critic returns "fail" with suggestedRevision → triggers one retry
+ *  3. Critic returns "fail" twice → retries exhausted, returns last candidate
+ *  4. Critic returns "uncertain" → treated as "pass", no retry
+ *  5. API error in critic → graceful fallback, returns candidate as-is
+ *  6. Malformed JSON from critic → fallback parser extracts verdict
+ *
+ * Usage (no API key required):
+ *   npx tsx src/benchmarks/gaia-critic.smoke.ts
+ *
+ * Refs: ADR-135, #2156
+ */
+import { criticReview, runGaiaAgentWithCritic, } from './gaia-critic.js';
+// ---------------------------------------------------------------------------
+// Test helpers
+// ---------------------------------------------------------------------------
+let passed = 0;
+let failed = 0;
+function assert(condition, message) {
+    if (condition) {
+        console.log(`  PASS  ${message}`);
+        passed++;
+    }
+    else {
+        console.error(`  FAIL  ${message}`);
+        failed++;
+    }
+}
+function assertEqual(actual, expected, message) {
+    assert(actual === expected, `${message} (expected ${String(expected)}, got ${String(actual)})`);
+}
+/** A minimal GaiaQuestion fixture. */
+const FIXTURE_QUESTION = {
+    task_id: 'smoke-001',
+    level: 1,
+    question: 'What is the capital of France?',
+    final_answer: 'Paris',
+    file_name: null,
+    file_path: null,
+};
+/** Minimal GaiaAgentResult with a given finalAnswer. */
+function makeAgentResult(finalAnswer, turns = 2) {
+    return {
+        questionId: FIXTURE_QUESTION.task_id,
+        finalAnswer,
+        turns,
+        toolCallsByName: { web_search: 1 },
+        totalInputTokens: 100,
+        totalOutputTokens: 50,
+        wallMs: 1200,
+    };
+}
+function installFetchMock(responses) {
+    let callIndex = 0;
+    const original = globalThis.fetch;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    globalThis.fetch = async (_url, _init) => {
+        const resp = responses[Math.min(callIndex, responses.length - 1)];
+        callIndex++;
+        if (!resp.ok) {
+            return {
+                ok: false,
+                status: resp.status ?? 500,
+                text: async () => resp.text ?? 'Internal error',
+                json: async () => { throw new Error('not ok'); },
+            };
+        }
+        return {
+            ok: true,
+            status: 200,
+            json: async () => resp.body ?? {},
+            text: async () => JSON.stringify(resp.body ?? {}),
+        };
+    };
+    return () => { globalThis.fetch = original; };
+}
+/**
+ * Build a mock Anthropic response body containing a critic verdict JSON.
+ */
+function mockCriticResponse(verdict, reasoning, suggestedRevision = '') {
+    const content = JSON.stringify({ verdict, reasoning, suggestedRevision });
+    return {
+        content: [{ type: 'text', text: content }],
+        usage: { input_tokens: 200, output_tokens: 40 },
+    };
+}
+/**
+ * Mock for runGaiaAgent that immediately returns a fixed result.
+ * We monkey-patch the module-level import by wrapping runGaiaAgentWithCritic
+ * via its options.catalogue approach — but since gaia-agent's runGaiaAgent is
+ * imported directly, we mock at the fetch level instead (the agent also calls
+ * the Anthropic API, so we intercept there).
+ *
+ * For the smoke tests we only exercise criticReview directly in Tests 1-6, and
+ * use a simplified version of runGaiaAgentWithCritic that pre-supplies a
+ * mocked agent result rather than actually calling the API for the agent run.
+ *
+ * This avoids needing to rewrite the import mechanism just for smoke tests.
+ */
+// ---------------------------------------------------------------------------
+// Test 1: critic returns "pass" → no retry
+// ---------------------------------------------------------------------------
+async function test1_criticPass() {
+    console.log('\nTest 1: critic returns "pass" → no retry');
+    const restore = installFetchMock([
+        { ok: true, body: mockCriticResponse('pass', 'Answer is correct.') },
+    ]);
+    try {
+        const verdict = await criticReview(FIXTURE_QUESTION, 'Paris', { steps: [{ tool: 'web_search', result: 'Paris is the capital' }], turns: 2 }, { model: 'claude-sonnet-4-6', apiKey: 'test-key' });
+        assertEqual(verdict.verdict, 'pass', 'verdict is "pass"');
+        assert(verdict.reasoning.length > 0, 'reasoning is non-empty');
+        assert(verdict.costUsd >= 0, 'costUsd is non-negative');
+        assert(!verdict.error, 'no error flag');
+    }
+    finally {
+        restore();
+    }
+}
+// ---------------------------------------------------------------------------
+// Test 2: critic returns "fail" with suggestedRevision
+// ---------------------------------------------------------------------------
+async function test2_criticFail() {
+    console.log('\nTest 2: critic returns "fail" with suggestedRevision');
+    const restore = installFetchMock([
+        {
+            ok: true,
+            body: mockCriticResponse('fail', 'The answer is the wrong city.', 'Paris'),
+        },
+    ]);
+    try {
+        const verdict = await criticReview(FIXTURE_QUESTION, 'Lyon', { steps: [{ tool: 'web_search', result: 'Lyon is in France' }], turns: 1 }, { model: 'claude-sonnet-4-6', apiKey: 'test-key' });
+        assertEqual(verdict.verdict, 'fail', 'verdict is "fail"');
+        assert((verdict.suggestedRevision ?? '').length > 0, 'suggestedRevision is non-empty');
+        assertEqual(verdict.suggestedRevision, 'Paris', 'suggestedRevision is "Paris"');
+    }
+    finally {
+        restore();
+    }
+}
+// ---------------------------------------------------------------------------
+// Test 3: critic returns "fail" twice → retries exhausted
+// ---------------------------------------------------------------------------
+async function test3_retriesExhausted() {
+    console.log('\nTest 3: critic fails twice → retries exhausted, returns last candidate');
+    // We test this at the runGaiaAgentWithCritic level.
+    // We need to mock BOTH the critic fetch calls AND the agent API calls.
+    // Strategy: sequence the mock responses in the order they will be called.
+    //
+    // Call sequence (with enableCritic=true, maxRetries=1):
+    //   1. runGaiaAgent attempt 1   → agent Anthropic API call (returns answer "Lyon")
+    //   2. criticReview attempt 1   → critic API call (returns "fail")
+    //   3. runGaiaAgent attempt 2   → agent Anthropic API call (returns answer "Marseille")
+    //   4. criticReview attempt 2   → critic API call (returns "fail")
+    //
+    // For simplicity we make the agent calls also return valid Anthropic responses
+    // that produce a FINAL_ANSWER (the agent code parses stop_reason=end_turn).
+    const agentResponseLyon = {
+        id: 'msg_01',
+        type: 'message',
+        role: 'assistant',
+        stop_reason: 'end_turn',
+        content: [{ type: 'text', text: 'FINAL_ANSWER: Lyon' }],
+        usage: { input_tokens: 150, output_tokens: 20 },
+        model: 'claude-haiku-4-5',
+    };
+    const agentResponseMarseille = {
+        id: 'msg_02',
+        type: 'message',
+        role: 'assistant',
+        stop_reason: 'end_turn',
+        content: [{ type: 'text', text: 'FINAL_ANSWER: Marseille' }],
+        usage: { input_tokens: 150, output_tokens: 20 },
+        model: 'claude-haiku-4-5',
+    };
+    const restore = installFetchMock([
+        { ok: true, body: agentResponseLyon }, // agent call 1
+        { ok: true, body: mockCriticResponse('fail', 'Wrong city.', 'Paris') }, // critic 1
+        { ok: true, body: agentResponseMarseille }, // agent call 2 (retry)
+        { ok: true, body: mockCriticResponse('fail', 'Still wrong.', 'Paris') }, // critic 2
+    ]);
+    try {
+        const result = await runGaiaAgentWithCritic(FIXTURE_QUESTION, {
+            enableCritic: true,
+            apiKey: 'test-key',
+            criticOptions: { apiKey: 'test-key', maxRetries: 1 },
+        });
+        assertEqual(result.retriesAttempted, 1, 'retriesAttempted is 1');
+        assertEqual(result.criticVerdicts.length, 2, 'two critic verdicts collected');
+        assertEqual(result.criticVerdicts[0].verdict, 'fail', 'first verdict is fail');
+        assertEqual(result.criticVerdicts[1].verdict, 'fail', 'second verdict is fail');
+        // Last candidate is returned regardless.
+        assert(result.finalAnswer !== null, 'finalAnswer is non-null (last candidate returned)');
+    }
+    finally {
+        restore();
+    }
+}
+// ---------------------------------------------------------------------------
+// Test 4: critic returns "uncertain" → treated as "pass", no retry
+// ---------------------------------------------------------------------------
+async function test4_uncertainAsPass() {
+    console.log('\nTest 4: critic returns "uncertain" → treated as pass, no retry');
+    const restore = installFetchMock([
+        {
+            ok: true,
+            body: mockCriticResponse('uncertain', 'Cannot verify without more context.'),
+        },
+    ]);
+    try {
+        const verdict = await criticReview(FIXTURE_QUESTION, 'Paris', { steps: [], turns: 1 }, { model: 'claude-sonnet-4-6', apiKey: 'test-key' });
+        assertEqual(verdict.verdict, 'uncertain', 'verdict is "uncertain"');
+        // Verify orchestrator treats uncertain as pass (no retry fired).
+        // We simulate by checking the logic directly: uncertain !== 'fail', so loop
+        // body is skipped.  We test it via runGaiaAgentWithCritic with a minimal
+        // agent mock that returns a final answer.
+        const agentResponseParis = {
+            id: 'msg_03',
+            type: 'message',
+            role: 'assistant',
+            stop_reason: 'end_turn',
+            content: [{ type: 'text', text: 'FINAL_ANSWER: Paris' }],
+            usage: { input_tokens: 100, output_tokens: 15 },
+            model: 'claude-haiku-4-5',
+        };
+        const restore2 = installFetchMock([
+            { ok: true, body: agentResponseParis }, // agent call
+            { ok: true, body: mockCriticResponse('uncertain', 'Cannot verify.') }, // critic
+        ]);
+        try {
+            const result = await runGaiaAgentWithCritic(FIXTURE_QUESTION, {
+                enableCritic: true,
+                apiKey: 'test-key',
+                criticOptions: { apiKey: 'test-key', maxRetries: 1 },
+            });
+            assertEqual(result.retriesAttempted, 0, 'no retries for uncertain verdict');
+            assertEqual(result.criticVerdicts.length, 1, 'one critic verdict collected');
+            assertEqual(result.criticVerdicts[0].verdict, 'uncertain', 'verdict is uncertain');
+        }
+        finally {
+            restore2();
+        }
+    }
+    finally {
+        restore();
+    }
+}
+// ---------------------------------------------------------------------------
+// Test 5: API error in critic → graceful fallback
+// ---------------------------------------------------------------------------
+async function test5_apiErrorFallback() {
+    console.log('\nTest 5: API error in critic → graceful fallback, original candidate returned');
+    const restore = installFetchMock([
+        { ok: false, status: 529, text: 'Overloaded' },
+    ]);
+    try {
+        const verdict = await criticReview(FIXTURE_QUESTION, 'Paris', { steps: [], turns: 1 }, { model: 'claude-sonnet-4-6', apiKey: 'test-key' });
+        // On API error, critic returns uncertain with error flag — does not throw.
+        assertEqual(verdict.verdict, 'uncertain', 'verdict is "uncertain" on API error');
+        assert(verdict.error === true, 'error flag is set');
+        assert(verdict.costUsd === 0, 'costUsd is 0 on error');
+        assert(verdict.reasoning.includes('failed'), 'reasoning mentions failure');
+    }
+    finally {
+        restore();
+    }
+}
+// ---------------------------------------------------------------------------
+// Test 6: malformed JSON from critic → fallback parser
+// ---------------------------------------------------------------------------
+async function test6_malformedJson() {
+    console.log('\nTest 6: malformed JSON from critic → fallback parser extracts verdict');
+    // Simulate Sonnet returning prose with embedded JSON fragment.
+    const malformedBody = {
+        content: [{
+                type: 'text',
+                text: 'After careful review, I believe the answer is wrong. {"verdict":"fail","reasoning":"Incorrect city","suggestedRevision":"Paris"} The agent should try again.',
+            }],
+        usage: { input_tokens: 180, output_tokens: 60 },
+    };
+    const restore = installFetchMock([
+        { ok: true, body: malformedBody },
+    ]);
+    try {
+        const verdict = await criticReview(FIXTURE_QUESTION, 'Lyon', { steps: [], turns: 1 }, { model: 'claude-sonnet-4-6', apiKey: 'test-key' });
+        // Fallback parser should extract "fail" from the embedded JSON.
+        assertEqual(verdict.verdict, 'fail', 'fallback parser extracts "fail" verdict');
+        assert(!verdict.error, 'no error flag for recoverable parse');
+    }
+    finally {
+        restore();
+    }
+}
+// ---------------------------------------------------------------------------
+// Run all tests
+// ---------------------------------------------------------------------------
+async function main() {
+    console.log('=== GAIA Critic Smoke Tests (ADR-135 Track D) ===');
+    console.log('All tests use mocked responses — no live API calls.\n');
+    try {
+        await test1_criticPass();
+        await test2_criticFail();
+        await test3_retriesExhausted();
+        await test4_uncertainAsPass();
+        await test5_apiErrorFallback();
+        await test6_malformedJson();
+    }
+    catch (err) {
+        console.error('\nUnexpected test runner error:', err);
+        process.exit(1);
+    }
+    console.log(`\n=== Results: ${passed} passed, ${failed} failed ===`);
+    if (failed > 0) {
+        process.exit(1);
+    }
+}
+main().catch(err => {
+    console.error(err);
+    process.exit(1);
+});
+//# sourceMappingURL=gaia-critic.smoke.js.map

package/dist/src/benchmarks/gaia-critic.smoke.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gaia-critic.smoke.js","sourceRoot":"","sources":["../../../src/benchmarks/gaia-critic.smoke.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EACL,YAAY,EACZ,sBAAsB,GAGvB,MAAM,kBAAkB,CAAC;AAI1B,8EAA8E;AAC9E,eAAe;AACf,8EAA8E;AAE9E,IAAI,MAAM,GAAG,CAAC,CAAC;AACf,IAAI,MAAM,GAAG,CAAC,CAAC;AAEf,SAAS,MAAM,CAAC,SAAkB,EAAE,OAAe;IACjD,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,WAAW,OAAO,EAAE,CAAC,CAAC;QAClC,MAAM,EAAE,CAAC;IACX,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,KAAK,CAAC,WAAW,OAAO,EAAE,CAAC,CAAC;QACpC,MAAM,EAAE,CAAC;IACX,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAI,MAAS,EAAE,QAAW,EAAE,OAAe;IAC7D,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,GAAG,OAAO,cAAc,MAAM,CAAC,QAAQ,CAAC,SAAS,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAClG,CAAC;AAED,sCAAsC;AACtC,MAAM,gBAAgB,GAAiB;IACrC,OAAO,EAAE,WAAW;IACpB,KAAK,EAAE,CAAC;IACR,QAAQ,EAAE,gCAAgC;IAC1C,YAAY,EAAE,OAAO;IACrB,SAAS,EAAE,IAAI;IACf,SAAS,EAAE,IAAI;CAChB,CAAC;AAEF,wDAAwD;AACxD,SAAS,eAAe,CAAC,WAA0B,EAAE,KAAK,GAAG,CAAC;IAC5D,OAAO;QACL,UAAU,EAAE,gBAAgB,CAAC,OAAO;QACpC,WAAW;QACX,KAAK;QACL,eAAe,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE;QAClC,gBAAgB,EAAE,GAAG;QACrB,iBAAiB,EAAE,EAAE;QACrB,MAAM,EAAE,IAAI;KACb,CAAC;AACJ,CAAC;AAiBD,SAAS,gBAAgB,CAAC,SAAyB;IACjD,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC;IAElC,8DAA8D;IAC7D,UAAkB,CAAC,KAAK,GAAG,KAAK,EAAE,IAAa,EAAE,KAAmB,EAAqB,EAAE;QAC1F,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QAClE,SAAS,EAAE,CAAC;QAEZ,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,GAAG;gBAC1B,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,IAAI,IAAI,gBAAgB;gBAC/C,IAAI,EAAE,KAAK,IAAI,EAAE,GAAG,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;aAC1B,CAAC;QAC3B,CAAC;QAED,OAAO;YACL,EAAE,EAAE,IAAI;YACR,MAAM,EAAE,GAAG;YACX,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE;YACjC,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;SAC3B,CAAC;IAC3B,CAAC,CAAC;IAEF,OAAO,GAAG,EAAE,GAAG,UAAU,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;AAChD,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CACzB,OAA0B,EAC1B,SAAiB,EACjB,iBAAiB,GAAG,EAAE;IAEtB,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,CAAC,CAAC;IAC1E,OAAO;QACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;QAC1C,KAAK,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,aAAa,EAAE,EAAE,EAAE;KAChD,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;GAYG;AAEH,8EAA8E;AAC9E,2CAA2C;AAC3C,8EAA8E;AAE9E,KAAK,UAAU,gBAAgB;IAC7B,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;IAE1D,MAAM,OAAO,GAAG,gBAAgB,CAAC;QAC/B,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,kBAAkB,CAAC,MAAM,EAAE,oBAAoB,CAAC,EAAE;KACrE,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,OAAO,GAAkB,MAAM,YAAY,CAC/C,gBAAgB,EAChB,OAAO,EACP,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,sBAAsB,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAC7E,EAAE,KAAK,EAAE,mBAAmB,EAAE,MAAM,EAAE,UAAU,EAAE,CACnD,CAAC;QAEF,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,mBAAmB,CAAC,CAAC;QAC1D,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,wBAAwB,CAAC,CAAC;QAC/D,MAAM,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,EAAE,yBAAyB,CAAC,CAAC;QACxD,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC;IAC1C,CAAC;YAAS,CAAC;QACT,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,uDAAuD;AACvD,8EAA8E;AAE9E,KAAK,UAAU,gBAAgB;IAC7B,OAAO,CAAC,GAAG,CAAC,wDAAwD,CAAC,CAAC;IAEtE,MAAM,OAAO,GAAG,gBAAgB,CAAC;QAC/B;YACE,EAAE,EAAE,IAAI;YACR,IAAI,EAAE,kBAAkB,CACtB,MAAM,EACN,+BAA+B,EAC/B,OAAO,CACR;SACF;KACF,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,OAAO,GAAkB,MAAM,YAAY,CAC/C,gBAAgB,EAChB,MAAM,EACN,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAC1E,EAAE,KAAK,EAAE,mBAAmB,EAAE,MAAM,EAAE,UAAU,EAAE,CACnD,CAAC;QAEF,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,mBAAmB,CAAC,CAAC;QAC1D,MAAM,CAAC,CAAC,OAAO,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,gCAAgC,CAAC,CAAC;QACvF,WAAW,CAAC,OAAO,CAAC,iBAAiB,EAAE,OAAO,EAAE,8BAA8B,CAAC,CAAC;IAClF,CAAC;YAAS,CAAC;QACT,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,0DAA0D;AAC1D,8EAA8E;AAE9E,KAAK,UAAU,sBAAsB;IACnC,OAAO,CAAC,GAAG,CAAC,0EAA0E,CAAC,CAAC;IAExF,oDAAoD;IACpD,uEAAuE;IACvE,0EAA0E;IAC1E,EAAE;IACF,wDAAwD;IACxD,mFAAmF;IACnF,mEAAmE;IACnE,wFAAwF;IACxF,mEAAmE;IACnE,EAAE;IACF,+EAA+E;IAC/E,4EAA4E;IAE5E,MAAM,iBAAiB,GAAG;QACxB,EAAE,EAAE,QAAQ;QACZ,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,WAAW;QACjB,WAAW,EAAE,UAAU;QACvB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,oBAAoB,EAAE,CAAC;QACvD,KAAK,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,aAAa,EAAE,EAAE,EAAE;QAC/C,KAAK,EAAE,kBAAkB;KAC1B,CAAC;IACF,MAAM,sBAAsB,GAAG;QAC7B,EAAE,EAAE,QAAQ;QACZ,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,WAAW;QACjB,WAAW,EAAE,UAAU;QACvB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,yBAAyB,EAAE,CAAC;QAC5D,KAAK,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,aAAa,EAAE,EAAE,EAAE;QAC/C,KAAK,EAAE,kBAAkB;KAC1B,CAAC;IAEF,MAAM,OAAO,GAAG,gBAAgB,CAAC;QAC/B,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,iBAAiB,EAAE,EAA2B,eAAe;QAC/E,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,OAAO,CAAC,EAAE,EAAG,WAAW;QACpF,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,sBAAsB,EAAE,EAAsB,uBAAuB;QACvF,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,kBAAkB,CAAC,MAAM,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,EAAE,WAAW;KACrF,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,gBAAgB,EAAE;YAC5D,YAAY,EAAE,IAAI;YAClB,MAAM,EAAE,UAAU;YAClB,aAAa,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,EAAE;SACrD,CAAC,CAAC;QAEH,WAAW,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC,EAAE,uBAAuB,CAAC,CAAC;QACjE,WAAW,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,+BAA+B,CAAC,CAAC;QAC9E,WAAW,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,uBAAuB,CAAC,CAAC;QAC/E,WAAW,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,wBAAwB,CAAC,CAAC;QAChF,yCAAyC;QACzC,MAAM,CAAC,MAAM,CAAC,WAAW,KAAK,IAAI,EAAE,mDAAmD,CAAC,CAAC;IAC3F,CAAC;YAAS,CAAC;QACT,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,mEAAmE;AACnE,8EAA8E;AAE9E,KAAK,UAAU,qBAAqB;IAClC,OAAO,CAAC,GAAG,CAAC,kEAAkE,CAAC,CAAC;IAEhF,MAAM,OAAO,GAAG,gBAAgB,CAAC;QAC/B;YACE,EAAE,EAAE,IAAI;YACR,IAAI,EAAE,kBAAkB,CAAC,WAAW,EAAE,qCAAqC,CAAC;SAC7E;KACF,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,OAAO,GAAkB,MAAM,YAAY,CAC/C,gBAAgB,EAChB,OAAO,EACP,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,EACvB,EAAE,KAAK,EAAE,mBAAmB,EAAE,MAAM,EAAE,UAAU,EAAE,CACnD,CAAC;QAEF,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,WAAW,EAAE,wBAAwB,CAAC,CAAC;QAEpE,iEAAiE;QACjE,4EAA4E;QAC5E,yEAAyE;QACzE,0CAA0C;QAC1C,MAAM,kBAAkB,GAAG;YACzB,EAAE,EAAE,QAAQ;YACZ,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,WAAW;YACjB,WAAW,EAAE,UAAU;YACvB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,qBAAqB,EAAE,CAAC;YACxD,KAAK,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,aAAa,EAAE,EAAE,EAAE;YAC/C,KAAK,EAAE,kBAAkB;SAC1B,CAAC;QAEF,MAAM,QAAQ,GAAG,gBAAgB,CAAC;YAChC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,kBAAkB,EAAE,EAAI,aAAa;YACvD,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,kBAAkB,CAAC,WAAW,EAAE,gBAAgB,CAAC,EAAE,EAAE,SAAS;SACjF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,gBAAgB,EAAE;gBAC5D,YAAY,EAAE,IAAI;gBAClB,MAAM,EAAE,UAAU;gBAClB,aAAa,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,EAAE;aACrD,CAAC,CAAC;YAEH,WAAW,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC,EAAE,kCAAkC,CAAC,CAAC;YAC5E,WAAW,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,8BAA8B,CAAC,CAAC;YAC7E,WAAW,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,WAAW,EAAE,sBAAsB,CAAC,CAAC;QACrF,CAAC;gBAAS,CAAC;YACT,QAAQ,EAAE,CAAC;QACb,CAAC;IACH,CAAC;YAAS,CAAC;QACT,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,kDAAkD;AAClD,8EAA8E;AAE9E,KAAK,UAAU,sBAAsB;IACnC,OAAO,CAAC,GAAG,CAAC,gFAAgF,CAAC,CAAC;IAE9F,MAAM,OAAO,GAAG,gBAAgB,CAAC;QAC/B,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,YAAY,EAAE;KAC/C,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,OAAO,GAAkB,MAAM,YAAY,CAC/C,gBAAgB,EAChB,OAAO,EACP,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,EACvB,EAAE,KAAK,EAAE,mBAAmB,EAAE,MAAM,EAAE,UAAU,EAAE,CACnD,CAAC;QAEF,2EAA2E;QAC3E,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,WAAW,EAAE,qCAAqC,CAAC,CAAC;QACjF,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,IAAI,EAAE,mBAAmB,CAAC,CAAC;QACpD,MAAM,CAAC,OAAO,CAAC,OAAO,KAAK,CAAC,EAAE,uBAAuB,CAAC,CAAC;QACvD,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,4BAA4B,CAAC,CAAC;IAC7E,CAAC;YAAS,CAAC;QACT,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,uDAAuD;AACvD,8EAA8E;AAE9E,KAAK,UAAU,mBAAmB;IAChC,OAAO,CAAC,GAAG,CAAC,yEAAyE,CAAC,CAAC;IAEvF,+DAA+D;IAC/D,MAAM,aAAa,GAAG;QACpB,OAAO,EAAE,CAAC;gBACR,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,8JAA8J;aACrK,CAAC;QACF,KAAK,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,aAAa,EAAE,EAAE,EAAE;KAChD,CAAC;IAEF,MAAM,OAAO,GAAG,gBAAgB,CAAC;QAC/B,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE;KAClC,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,OAAO,GAAkB,MAAM,YAAY,CAC/C,gBAAgB,EAChB,MAAM,EACN,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,EACvB,EAAE,KAAK,EAAE,mBAAmB,EAAE,MAAM,EAAE,UAAU,EAAE,CACnD,CAAC;QAEF,gEAAgE;QAChE,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,yCAAyC,CAAC,CAAC;QAChF,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,qCAAqC,CAAC,CAAC;IAChE,CAAC;YAAS,CAAC;QACT,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,gBAAgB;AAChB,8EAA8E;AAE9E,KAAK,UAAU,IAAI;IACjB,OAAO,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC;IACjE,OAAO,CAAC,GAAG,CAAC,uDAAuD,CAAC,CAAC;IAErE,IAAI,CAAC;QACH,MAAM,gBAAgB,EAAE,CAAC;QACzB,MAAM,gBAAgB,EAAE,CAAC;QACzB,MAAM,sBAAsB,EAAE,CAAC;QAC/B,MAAM,qBAAqB,EAAE,CAAC;QAC9B,MAAM,sBAAsB,EAAE,CAAC;QAC/B,MAAM,mBAAmB,EAAE,CAAC;IAC9B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE,GAAG,CAAC,CAAC;QACtD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,YAAY,MAAM,aAAa,CAAC,CAAC;IAErE,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE;IACjB,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/src/benchmarks/gaia-decomposer.d.ts

@@ -0,0 +1,125 @@
+/**
+ * GAIA Question Decomposer — ADR-135 Track E
+ *
+ * Decomposes complex GAIA benchmark questions into 1-5 ordered sub-questions
+ * that can each be answered with a single tool call, then synthesizes the
+ * sub-answers into a final response.
+ *
+ * Motivation (iter 29 finding): tool quality is the bottleneck on L1 (~20.8%).
+ * Bad tools that fail on complex queries may succeed on focused sub-queries.
+ * This mimics what humans do at 92% on GAIA (decompose-then-solve).
+ * Expected L1 lift: +5-10pp on multi-step questions (~30-40% of L1 set).
+ *
+ * Design:
+ *  - NEW standalone file only; NOT wired into gaia-bench.ts to avoid merge
+ *    conflicts with in-flight iter 29/31/34/35/36 branches.
+ *    Wiring is a small follow-up PR.
+ *  - `decomposeQuestion()` — Haiku-cheap classification + decomposition
+ *    (~$0.0003 per question).
+ *  - `synthesizeFromSubAnswers()` — Sonnet synthesis from sub-answers
+ *    (the sub-answers are the hard work; synthesis is just combination).
+ *  - Atomic questions are returned as-is — no overhead when not needed.
+ *  - Graceful fallback to atomic on API errors or malformed JSON.
+ *
+ * Cost discipline:
+ *  - Decomposition uses claude-haiku-4-5 (~$0.0003/question).
+ *  - Synthesis uses claude-sonnet-4-6 (~$0.002/question).
+ *  - Total overhead per question (when decomposed): ~$0.002-0.003.
+ *
+ * Plugin sync TODO (follow-up PR after gaia-bench wiring):
+ *  - Update plugins/ruflo-workflows/commands/gaia-run.md with --decompose flag.
+ *  - Update plugins/ruflo-workflows/skills/gaia-debugging/SKILL.md: add
+ *    decomposition as a recommended strategy for multi-step failures.
+ *
+ * Refs: ADR-135, ADR-133, iter 29 finding, #2156
+ */
+export interface DecomposedQuestion {
+    /** The original unmodified question text. */
+    originalQuestion: string;
+    /**
+     * Ordered sub-questions in dependency order (1-5).
+     * If decomposed=false, contains exactly one entry equal to originalQuestion.
+     */
+    subQuestions: string[];
+    /**
+     * Brief hint for how to combine sub-answers into the final answer.
+     * Example: "Multiply the two values found in sub-questions 1 and 2."
+     */
+    synthesisHint: string;
+    /**
+     * true if the question was split into multiple sub-questions;
+     * false if it was deemed atomic (single lookup / single computation).
+     */
+    decomposed: boolean;
+    /** USD spent on the decomposition API call (0 if fallback/atomic). */
+    cost: number;
+}
+export interface DecomposerOptions {
+    /**
+     * Model to use for decomposition classification.
+     * Default: 'claude-haiku-4-5' (cheap).
+     */
+    model?: string;
+    /**
+     * Maximum sub-questions to produce when decomposing.
+     * Default: 5.
+     */
+    maxSubQuestions?: number;
+    /** API key override. Falls back to ANTHROPIC_API_KEY env var. */
+    apiKey?: string;
+}
+export interface SynthesizerOptions {
+    /**
+     * Model to use for final-answer synthesis.
+     * Default: 'claude-sonnet-4-6'.
+     */
+    model?: string;
+    /** API key override. Falls back to ANTHROPIC_API_KEY env var. */
+    apiKey?: string;
+}
+export interface SynthesisResult {
+    /** The synthesized final answer string. */
+    finalAnswer: string;
+    /** Brief reasoning that led to the final answer. */
+    reasoning: string;
+    /** USD spent on the synthesis API call. */
+    cost: number;
+}
+/**
+ * Decomposes a complex question into 1-5 sub-questions in dependency order,
+ * OR returns the question as-is if it is already atomic.
+ *
+ * Uses `claude-haiku-4-5` by default for cheap classification + decomposition
+ * (~$0.0003 per question).
+ *
+ * Heuristics the model uses internally for "should decompose":
+ *  - Question contains "and", "then", "after", "if X …"
+ *  - Question contains multiple named entities that must each be looked up
+ *  - Question asks for a derived/computed answer (X of Y where Y must be found)
+ *
+ * Graceful degradation: on API errors or malformed JSON, returns the question
+ * as atomic so the calling agent can still attempt a direct answer.
+ *
+ * @param questionText - The full GAIA question text.
+ * @param options      - Optional overrides (model, maxSubQuestions, apiKey).
+ * @returns            - DecomposedQuestion with subQuestions and synthesisHint.
+ */
+export declare function decomposeQuestion(questionText: string, options?: DecomposerOptions): Promise<DecomposedQuestion>;
+/**
+ * Given a decomposed question and answers to each sub-question, synthesizes
+ * a final concise answer.
+ *
+ * Uses `claude-sonnet-4-6` by default for higher reasoning quality — the
+ * sub-answers contain the hard-won information; synthesis is recombination.
+ *
+ * Graceful degradation: on API errors or malformed JSON, returns the last
+ * sub-answer concatenated with reasoning note.
+ *
+ * @param decomposed  - The DecomposedQuestion from `decomposeQuestion()`.
+ * @param subAnswers  - Array of string answers, one per sub-question.
+ *                      Must be the same length as decomposed.subQuestions.
+ * @param options     - Optional overrides (model, apiKey).
+ * @returns           - SynthesisResult with finalAnswer, reasoning, and cost.
+ */
+export declare function synthesizeFromSubAnswers(decomposed: DecomposedQuestion, subAnswers: string[], options?: SynthesizerOptions): Promise<SynthesisResult>;
+//# sourceMappingURL=gaia-decomposer.d.ts.map

package/dist/src/benchmarks/gaia-decomposer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gaia-decomposer.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/gaia-decomposer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAsCH,MAAM,WAAW,kBAAkB;IACjC,6CAA6C;IAC7C,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB;;;OAGG;IACH,aAAa,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,UAAU,EAAE,OAAO,CAAC;IACpB,sEAAsE;IACtE,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,iBAAiB;IAChC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,kBAAkB;IACjC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,2CAA2C;IAC3C,WAAW,EAAE,MAAM,CAAC;IACpB,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,2CAA2C;IAC3C,IAAI,EAAE,MAAM,CAAC;CACd;AA8LD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,kBAAkB,CAAC,CA+D7B;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,wBAAwB,CAC5C,UAAU,EAAE,kBAAkB,EAC9B,UAAU,EAAE,MAAM,EAAE,EACpB,OAAO,CAAC,EAAE,kBAAkB,GAC3B,OAAO,CAAC,eAAe,CAAC,CAkF1B"}