claude-flow 3.10.4 → 3.10.5

package/v3/@claude-flow/cli/dist/src/benchmarks/gaia-decomposer.smoke.js

@@ -0,0 +1,228 @@
+/**
+ * Smoke tests for gaia-decomposer.ts (ADR-135 Track E)
+ *
+ * All HTTP calls are mocked — no live API calls, no cost.
+ * Covers 7 assertions:
+ *  1. Atomic question → decomposed=false, single sub-question = original
+ *  2. 3-step question → decomposed=true, 3 sub-questions in dependency order
+ *  3. Malformed JSON response → fallback to atomic (graceful degradation)
+ *  4. API error → fallback to atomic (graceful degradation)
+ *  5. synthesizeFromSubAnswers — atomic question returns sub-answer directly (no API call)
+ *  6. synthesizeFromSubAnswers — valid JSON response → finalAnswer + reasoning
+ *  7. synthesizeFromSubAnswers — malformed JSON → fallback to last sub-answer
+ *
+ * Run with:
+ *   npx tsx src/benchmarks/gaia-decomposer.smoke.ts
+ *
+ * Expected cost: $0 (all mocked).
+ */
+import { decomposeQuestion, synthesizeFromSubAnswers } from './gaia-decomposer.js';
+function installFetchMock(responder) {
+    globalThis.fetch = (url, init) => {
+        const r = responder(url, init);
+        return Promise.resolve(r);
+    };
+}
+function mockSuccessResponse(jsonBody) {
+    const bodyStr = JSON.stringify(jsonBody);
+    return {
+        ok: true,
+        status: 200,
+        text: () => Promise.resolve(bodyStr),
+        json: () => Promise.resolve(jsonBody),
+    };
+}
+function mockErrorResponse(status) {
+    return {
+        ok: false,
+        status,
+        text: () => Promise.resolve(`HTTP ${status} error`),
+        json: () => Promise.reject(new Error('error response has no JSON')),
+    };
+}
+/** Build a minimal Anthropic Messages API response shape. */
+function anthropicTextResponse(text, inputTokens = 100, outputTokens = 50) {
+    return {
+        id: 'msg_test',
+        type: 'message',
+        role: 'assistant',
+        content: [{ type: 'text', text }],
+        model: 'claude-haiku-4-5',
+        stop_reason: 'end_turn',
+        usage: { input_tokens: inputTokens, output_tokens: outputTokens },
+    };
+}
+// ---------------------------------------------------------------------------
+// Test runner
+// ---------------------------------------------------------------------------
+const PASS = '\x1b[32mPASS\x1b[0m';
+const FAIL = '\x1b[31mFAIL\x1b[0m';
+let failures = 0;
+let passed = 0;
+function check(label, condition) {
+    if (condition) {
+        console.log(`  ${PASS}  ${label}`);
+        passed++;
+    }
+    else {
+        console.log(`  ${FAIL}  ${label}`);
+        failures++;
+    }
+}
+// ---------------------------------------------------------------------------
+// Smoke tests
+// ---------------------------------------------------------------------------
+async function runSmoke() {
+    console.log('\n=== gaia-decomposer smoke (mocked, $0) ===\n');
+    // Set a fake API key so resolveApiKey() doesn't throw
+    process.env.ANTHROPIC_API_KEY = 'sk-ant-test-key';
+    // ─────────────────────────────────────────────────────────────────────────
+    // Test 1: Atomic question → decomposed=false, single sub-question
+    // ─────────────────────────────────────────────────────────────────────────
+    console.log('-- Test 1: Atomic question --');
+    {
+        const atomicPayload = {
+            decomposed: false,
+            subQuestions: ['What year was the Eiffel Tower built?'],
+            synthesisHint: 'Use directly.',
+        };
+        installFetchMock(() => mockSuccessResponse(anthropicTextResponse(JSON.stringify(atomicPayload))));
+        const result = await decomposeQuestion('What year was the Eiffel Tower built?');
+        check('decomposed=false for atomic question', result.decomposed === false);
+        check('single sub-question equals original', result.subQuestions.length === 1 && result.subQuestions[0] === 'What year was the Eiffel Tower built?');
+        check('cost > 0 (Haiku tokens consumed)', result.cost > 0);
+    }
+    // ─────────────────────────────────────────────────────────────────────────
+    // Test 2: 3-step complex question → decomposed=true, 3 sub-questions in order
+    // ─────────────────────────────────────────────────────────────────────────
+    console.log('\n-- Test 2: 3-step complex question --');
+    {
+        const complexPayload = {
+            decomposed: true,
+            subQuestions: [
+                'What year was the Eiffel Tower built?',
+                'What decade contains the year 1889?',
+                'Who directed the highest-grossing film of the 1880s?',
+            ],
+            synthesisHint: 'The answer to sub-question 3 is the final answer.',
+        };
+        installFetchMock(() => mockSuccessResponse(anthropicTextResponse(JSON.stringify(complexPayload))));
+        const question = 'Who directed the highest-grossing film of the decade the Eiffel Tower was built?';
+        const result = await decomposeQuestion(question);
+        check('decomposed=true for 3-step question', result.decomposed === true);
+        check('3 sub-questions returned', result.subQuestions.length === 3);
+        check('first sub-question asks about Eiffel Tower year', result.subQuestions[0].toLowerCase().includes('eiffel tower'));
+        check('synthesisHint is non-empty', result.synthesisHint.length > 0);
+    }
+    // ─────────────────────────────────────────────────────────────────────────
+    // Test 3: Malformed JSON response → fallback to atomic
+    // ─────────────────────────────────────────────────────────────────────────
+    console.log('\n-- Test 3: Malformed JSON fallback --');
+    {
+        installFetchMock(() => mockSuccessResponse(anthropicTextResponse('This is not JSON at all { broken')));
+        const question = 'What is the capital of France?';
+        const result = await decomposeQuestion(question);
+        check('malformed JSON → decomposed=false', result.decomposed === false);
+        check('malformed JSON → sub-question is original', result.subQuestions[0] === question);
+    }
+    // ─────────────────────────────────────────────────────────────────────────
+    // Test 4: API error → fallback to atomic
+    // ─────────────────────────────────────────────────────────────────────────
+    console.log('\n-- Test 4: API error fallback --');
+    {
+        installFetchMock(() => mockErrorResponse(500));
+        const question = 'How many planets are in the solar system?';
+        const result = await decomposeQuestion(question);
+        check('API error → decomposed=false', result.decomposed === false);
+        check('API error → sub-question is original', result.subQuestions[0] === question);
+        check('API error → cost=0', result.cost === 0);
+    }
+    // ─────────────────────────────────────────────────────────────────────────
+    // Test 5: synthesizeFromSubAnswers — atomic question (decomposed=false)
+    //         returns sub-answer directly WITHOUT an API call
+    // ─────────────────────────────────────────────────────────────────────────
+    console.log('\n-- Test 5: synthesize — atomic passthrough (no API call) --');
+    {
+        // Install a fetch mock that throws if called — should NOT be called
+        let fetchCalled = false;
+        installFetchMock(() => {
+            fetchCalled = true;
+            return mockSuccessResponse({ finalAnswer: 'unexpected', reasoning: 'should not reach here' });
+        });
+        const atomicDecomposed = {
+            originalQuestion: 'What year was the Eiffel Tower built?',
+            subQuestions: ['What year was the Eiffel Tower built?'],
+            synthesisHint: 'Use directly.',
+            decomposed: false,
+            cost: 0,
+        };
+        const result = await synthesizeFromSubAnswers(atomicDecomposed, ['1889']);
+        check('atomic synthesize returns sub-answer directly', result.finalAnswer === '1889');
+        check('atomic synthesize has cost=0', result.cost === 0);
+        check('atomic synthesize did NOT call API', fetchCalled === false);
+    }
+    // ─────────────────────────────────────────────────────────────────────────
+    // Test 6: synthesizeFromSubAnswers — valid JSON → finalAnswer + reasoning
+    // ─────────────────────────────────────────────────────────────────────────
+    console.log('\n-- Test 6: synthesize — valid JSON response --');
+    {
+        const synthPayload = {
+            finalAnswer: 'Georges Méliès',
+            reasoning: 'The highest-grossing film of the 1880s was X. Its director was Georges Méliès.',
+        };
+        installFetchMock(() => mockSuccessResponse(anthropicTextResponse(JSON.stringify(synthPayload))));
+        const decomposed = {
+            originalQuestion: 'Who directed the highest-grossing film of the decade the Eiffel Tower was built?',
+            subQuestions: [
+                'What year was the Eiffel Tower built?',
+                'What decade contains 1889?',
+                'Who directed the highest-grossing film of the 1880s?',
+            ],
+            synthesisHint: 'The director from sub-question 3 is the answer.',
+            decomposed: true,
+            cost: 0.0003,
+        };
+        const result = await synthesizeFromSubAnswers(decomposed, ['1889', '1880s', 'Georges Méliès']);
+        check('synthesize returns correct finalAnswer', result.finalAnswer === 'Georges Méliès');
+        check('synthesize reasoning is non-empty', result.reasoning.length > 0);
+        check('synthesize cost > 0 (Sonnet tokens consumed)', result.cost > 0);
+    }
+    // ─────────────────────────────────────────────────────────────────────────
+    // Test 7: synthesizeFromSubAnswers — malformed JSON → fallback to last sub-answer
+    // ─────────────────────────────────────────────────────────────────────────
+    console.log('\n-- Test 7: synthesize — malformed JSON fallback --');
+    {
+        installFetchMock(() => mockSuccessResponse(anthropicTextResponse('{ bad json here ]]}')));
+        const decomposed = {
+            originalQuestion: 'How many people live in Tokyo?',
+            subQuestions: [
+                'What is the population of Tokyo metropolitan area?',
+                'What is the population of Tokyo proper?',
+            ],
+            synthesisHint: 'Use the Tokyo proper figure.',
+            decomposed: true,
+            cost: 0.0003,
+        };
+        const result = await synthesizeFromSubAnswers(decomposed, ['37.4 million (metropolitan)', '13.96 million (proper)']);
+        check('malformed JSON fallback → returns last sub-answer', result.finalAnswer === '13.96 million (proper)');
+        check('malformed JSON fallback → reasoning mentions parse error', result.reasoning.toLowerCase().includes('parse error'));
+    }
+    // ─────────────────────────────────────────────────────────────────────────
+    // Summary
+    // ─────────────────────────────────────────────────────────────────────────
+    const total = passed + failures;
+    console.log(`\n=== smoke ${failures === 0 ? 'PASSED' : `FAILED (${failures}/${total} assertion(s))`} — ${passed}/${total} passed ===\n`);
+    if (failures > 0)
+        process.exit(1);
+}
+// Run when executed directly
+const isMain = process.argv[1] && (process.argv[1].endsWith('gaia-decomposer.smoke.ts') ||
+    process.argv[1].endsWith('gaia-decomposer.smoke.js'));
+if (isMain) {
+    runSmoke().catch((err) => {
+        console.error('Smoke failed with exception:', err);
+        process.exit(1);
+    });
+}
+export { runSmoke };
+//# sourceMappingURL=gaia-decomposer.smoke.js.map

package/v3/@claude-flow/cli/dist/src/benchmarks/gaia-e2e-smoke.d.ts

@@ -0,0 +1,27 @@
+/**
+ * GAIA End-to-End Smoke — ADR-133
+ *
+ * Wires gaia-agent.ts + gaia-judge.ts into a single end-to-end pipeline:
+ *
+ *   for each question in SMOKE_FIXTURE:
+ *     1. runGaiaAgent(question)  — Haiku agent loop, ≤8 turns
+ *     2. judgeAnswer(question, result.finalAnswer)  — exact-match fast-path,
+ *        Sonnet LLM-judge only if exact-match misses
+ *
+ * Reports: pass rate, total cost, mean turn count.
+ * Asserts: ≥ 3/5 questions pass (lenient — smoke fixture is not trivial).
+ *
+ * Cost discipline:
+ *   - Agent: claude-haiku-4-5 at $0.25/$1.25 per M tokens
+ *   - Judge: claude-sonnet-4-6 at $3/$15 per M tokens (only when needed)
+ *   - Expected total for 5 questions × ~2 turns × Haiku + 1-2 Sonnet
+ *     judge calls ≈ $0.02
+ *
+ * Usage:
+ *   ANTHROPIC_API_KEY=sk-ant-... npx tsx src/benchmarks/gaia-e2e-smoke.ts
+ *
+ * Refs: ADR-133, #2156
+ */
+declare function runE2ESmoke(): Promise<void>;
+export { runE2ESmoke };
+//# sourceMappingURL=gaia-e2e-smoke.d.ts.map

package/v3/@claude-flow/cli/dist/src/benchmarks/gaia-e2e-smoke.js

@@ -0,0 +1,136 @@
+/**
+ * GAIA End-to-End Smoke — ADR-133
+ *
+ * Wires gaia-agent.ts + gaia-judge.ts into a single end-to-end pipeline:
+ *
+ *   for each question in SMOKE_FIXTURE:
+ *     1. runGaiaAgent(question)  — Haiku agent loop, ≤8 turns
+ *     2. judgeAnswer(question, result.finalAnswer)  — exact-match fast-path,
+ *        Sonnet LLM-judge only if exact-match misses
+ *
+ * Reports: pass rate, total cost, mean turn count.
+ * Asserts: ≥ 3/5 questions pass (lenient — smoke fixture is not trivial).
+ *
+ * Cost discipline:
+ *   - Agent: claude-haiku-4-5 at $0.25/$1.25 per M tokens
+ *   - Judge: claude-sonnet-4-6 at $3/$15 per M tokens (only when needed)
+ *   - Expected total for 5 questions × ~2 turns × Haiku + 1-2 Sonnet
+ *     judge calls ≈ $0.02
+ *
+ * Usage:
+ *   ANTHROPIC_API_KEY=sk-ant-... npx tsx src/benchmarks/gaia-e2e-smoke.ts
+ *
+ * Refs: ADR-133, #2156
+ */
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { SMOKE_FIXTURE, } from './gaia-loader.js';
+import { runGaiaAgent, } from './gaia-agent.js';
+import { judgeAnswer, } from './gaia-judge.js';
+// ---------------------------------------------------------------------------
+// Configuration
+// ---------------------------------------------------------------------------
+/** Agent model — Haiku only for cost discipline. */
+const AGENT_MODEL = 'claude-haiku-4-5';
+/** Judge model — Sonnet for semantic judgments (only when exact-match fails). */
+const JUDGE_MODEL = 'claude-sonnet-4-6';
+/** Minimum pass rate for smoke to succeed. */
+const MIN_PASS_RATE = 3 / 5;
+// Haiku pricing ($/M tokens)
+const HAIKU_IN = 0.25;
+const HAIKU_OUT = 1.25;
+// Sonnet pricing ($/M tokens)
+const SONNET_IN = 3.0;
+const SONNET_OUT = 15.0;
+// ---------------------------------------------------------------------------
+// Runner
+// ---------------------------------------------------------------------------
+async function runE2ESmoke() {
+    const hasKey = !!(process.env.ANTHROPIC_API_KEY?.trim());
+    if (!hasKey) {
+        console.error('ANTHROPIC_API_KEY is required for the end-to-end smoke.\n' +
+            'Set it with: export ANTHROPIC_API_KEY=sk-ant-...');
+        process.exit(1);
+    }
+    const cacheDir = path.join(os.homedir(), '.cache', 'ruflo', 'gaia', 'judgments');
+    const judgeOpts = { judgeModel: JUDGE_MODEL, cacheDir };
+    const questions = SMOKE_FIXTURE;
+    const rows = [];
+    console.log(`\n=== GAIA End-to-End Smoke (${questions.length} questions) ===\n`);
+    console.log(`Agent: ${AGENT_MODEL}  |  Judge: ${JUDGE_MODEL}\n` +
+        `Questions: ${questions.length}  |  Min pass rate: ${(MIN_PASS_RATE * 100).toFixed(0)}%\n`);
+    for (const q of questions) {
+        process.stdout.write(`[${q.task_id}] "${q.question.slice(0, 60)}..." `);
+        // Run agent
+        const agentResult = await runGaiaAgent(q, { model: AGENT_MODEL });
+        // Judge
+        const judgeResult = await judgeAnswer({ id: q.task_id, expected: q.final_answer, questionText: q.question }, agentResult.finalAnswer, judgeOpts);
+        rows.push({ question: q, agentResult, judgeResult });
+        const verdict = judgeResult.passed ? '\x1b[32mPASS\x1b[0m' : '\x1b[31mFAIL\x1b[0m';
+        const path_ = judgeResult.scoringPath;
+        console.log(`${verdict}  (turns=${agentResult.turns}, ` +
+            `answer="${agentResult.finalAnswer ?? 'null'}", ` +
+            `expected="${q.final_answer}", path=${path_})`);
+    }
+    // ── Summary ──
+    console.log('\n--- Summary ---\n');
+    const passed = rows.filter((r) => r.judgeResult.passed).length;
+    const total = rows.length;
+    const passRate = passed / total;
+    let totalAgentCostUsd = 0;
+    let totalJudgeCostUsd = 0;
+    let totalTurns = 0;
+    for (const row of rows) {
+        const { agentResult, judgeResult } = row;
+        totalTurns += agentResult.turns;
+        totalAgentCostUsd +=
+            (agentResult.totalInputTokens / 1_000_000) * HAIKU_IN +
+                (agentResult.totalOutputTokens / 1_000_000) * HAIKU_OUT;
+        totalJudgeCostUsd += judgeResult.judgeCostUsd ?? 0;
+    }
+    const totalCostUsd = totalAgentCostUsd + totalJudgeCostUsd;
+    const meanTurns = totalTurns / total;
+    console.log(`Pass rate      : ${passed}/${total}  (${(passRate * 100).toFixed(0)}%)`);
+    console.log(`Mean turns     : ${meanTurns.toFixed(1)}`);
+    console.log(`Agent cost     : $${totalAgentCostUsd.toFixed(5)}  (Haiku)`);
+    console.log(`Judge cost     : $${totalJudgeCostUsd.toFixed(5)}  (Sonnet, only when needed)`);
+    console.log(`Total cost     : $${totalCostUsd.toFixed(5)}`);
+    // ── Per-row detail ──
+    console.log('\n--- Per-question detail ---\n');
+    for (const row of rows) {
+        const { question, agentResult, judgeResult } = row;
+        const verdict = judgeResult.passed ? 'PASS' : 'FAIL';
+        console.log(`  ${verdict}  ${question.task_id}  turns=${agentResult.turns}  ` +
+            `path=${judgeResult.scoringPath}  ` +
+            `answer="${agentResult.finalAnswer ?? 'null'}"  ` +
+            `expected="${question.final_answer}"`);
+        if (judgeResult.judgeReason) {
+            console.log(`        reason: ${judgeResult.judgeReason}`);
+        }
+        if (agentResult.error) {
+            console.log(`        error : ${agentResult.error}`);
+        }
+    }
+    // ── Assertion ──
+    console.log('');
+    if (passRate >= MIN_PASS_RATE) {
+        console.log(`\x1b[32mSmoke PASSED\x1b[0m — ${passed}/${total} ≥ ${(MIN_PASS_RATE * 100).toFixed(0)}% required.\n`);
+    }
+    else {
+        console.error(`\x1b[31mSmoke FAILED\x1b[0m — ${passed}/${total} < ${(MIN_PASS_RATE * 100).toFixed(0)}% required.\n`);
+        process.exit(1);
+    }
+}
+// ---------------------------------------------------------------------------
+// Entry point
+// ---------------------------------------------------------------------------
+const isMain = process.argv[1] && (process.argv[1].endsWith('gaia-e2e-smoke.ts') ||
+    process.argv[1].endsWith('gaia-e2e-smoke.js'));
+if (isMain) {
+    runE2ESmoke().catch((err) => {
+        console.error('E2E smoke failed:', err);
+        process.exit(1);
+    });
+}
+export { runE2ESmoke };
+//# sourceMappingURL=gaia-e2e-smoke.js.map

package/v3/@claude-flow/cli/dist/src/benchmarks/gaia-extract.smoke.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Smoke tests for extractFinalAnswer + buildUserMessage (iter 53a T2 narrowed).
+ *
+ * Iter 53a changes vs iter 52 T2:
+ *   - Stage 2/3 (prose fallback) REMOVED — extractFinalAnswer is Stage 1 only.
+ *   - Commitment prompt no longer has "FINAL_ANSWER: unknown" surrender instruction.
+ *   - Reversed-text preprocessor PRESERVED (not the source of regressions).
+ *
+ * Anti-regression suite (7 cases):
+ *   These are the exact failure modes from iter 52 T2 that must not regress.
+ *   They test that the extraction logic is stable and correct for the 7 questions
+ *   where iter 52 broke but iter 51 was correct.
+ *
+ * Run (after build):
+ *   node dist/benchmarks/gaia-extract.smoke.js
+ *
+ * Exit 0 on all pass, 1 on any failure.
+ *
+ * Refs: ADR-133, ADR-135, iter 52 T2, iter 53a, #2156
+ */
+interface AnthropicResponse {
+    id: string;
+    model: string;
+    stop_reason: string;
+    content: Array<{
+        type: string;
+        text?: string;
+    }>;
+    usage: {
+        input_tokens: number;
+        output_tokens: number;
+    };
+}
+declare let _extractFinalAnswer: (resp: AnthropicResponse) => string | null;
+declare let _buildUserMessage: (question: string) => string;
+declare function makeResp(text: string): AnthropicResponse;
+interface TestCase {
+    label: string;
+    input: string;
+    expected: string | null;
+    isUserMsg?: boolean;
+}
+declare const CASES: TestCase[];
+declare function main(): Promise<void>;
+//# sourceMappingURL=gaia-extract.smoke.d.ts.map