claude-flow 3.10.4 → 3.10.5

package/v3/@claude-flow/cli/dist/src/commands/gaia-bench.js

@@ -0,0 +1,597 @@
+/**
+ * V3 CLI gaia-bench Command — ADR-133-PR8 + ADR-135 Tracks A/B/D/E/Q + ADR-136 Track Q
+ *
+ * Runs GAIA benchmark questions through the claude-flow agent loop and
+ * reports pass-rate, cost, and per-question results.
+ *
+ * Contract (matches gaia-benchmark.yml workflow expectations):
+ *   node bin/cli.js gaia-bench run \
+ *     --level <1|2|3> \
+ *     --limit <N> \
+ *     --models <csv> \
+ *     --output json
+ *
+ * JSON output shape:
+ *   {
+ *     level: number,
+ *     model: string,
+ *     summary: { total, passed, passRate, estCostUsd, hardnessDist? },
+ *     results: [{ task_id, question, model, correct, answer, expected_output, error }]
+ *   }
+ *
+ * Integration (iter 39 — ADR-135):
+ *   Wires standalone track modules into the CLI so they are usable end-to-end.
+ *   - Track A  (--voting-attempts N)   : multi-attempt self-consistency voting
+ *   - Track B  (--planning-interval N) : periodic planning checkpoints in gaia-agent
+ *   - Track D  (--enable-critic)       : adversarial critic review after agent answer
+ *   - Track E  (--decompose)           : question decomposition for multi-step Qs
+ *   - Track Q  (--hardness-routing)    : hardness-based compute allocation
+ *
+ * Precedence when flags combine:
+ *   --hardness-routing overrides --max-turns and --voting-attempts per question.
+ *   --voting-attempts > 1 takes precedence over --enable-critic (cost containment).
+ *   --decompose works independently; sub-question answers feed into voting/critic/plain.
+ *
+ * Refs: ADR-133, ADR-135, ADR-136, #2165, iter 28/34/36/37/39
+ */
+import { output } from '../output.js';
+// ---------------------------------------------------------------------------
+// Pricing constants for cost estimation
+// ---------------------------------------------------------------------------
+const MODEL_PRICING = {
+    'claude-haiku-4-5': { inputPerM: 0.25, outputPerM: 1.25 },
+    'claude-haiku-3': { inputPerM: 0.25, outputPerM: 1.25 },
+    'claude-sonnet-4-5': { inputPerM: 3.0, outputPerM: 15.0 },
+    'claude-sonnet-4-6': { inputPerM: 3.0, outputPerM: 15.0 },
+    'claude-opus-4-5': { inputPerM: 15.0, outputPerM: 75.0 },
+};
+function estimateCost(model, totalInputTokens, totalOutputTokens) {
+    const pricing = MODEL_PRICING[model] ?? { inputPerM: 3.0, outputPerM: 15.0 };
+    return ((totalInputTokens / 1_000_000) * pricing.inputPerM +
+        (totalOutputTokens / 1_000_000) * pricing.outputPerM);
+}
+// ---------------------------------------------------------------------------
+// run subcommand
+// ---------------------------------------------------------------------------
+const runCommand = {
+    name: 'run',
+    description: 'Run GAIA benchmark questions against one or more models',
+    options: [
+        {
+            name: 'level',
+            short: 'l',
+            type: 'number',
+            description: 'GAIA difficulty level: 1 (easiest), 2, or 3',
+            default: '1',
+        },
+        {
+            name: 'limit',
+            short: 'n',
+            type: 'number',
+            description: 'Maximum number of questions to run (default: all)',
+        },
+        {
+            name: 'models',
+            short: 'm',
+            type: 'string',
+            description: 'Comma-separated list of model IDs to test',
+            default: 'claude-haiku-4-5',
+        },
+        {
+            name: 'output',
+            short: 'o',
+            type: 'string',
+            description: 'Output format: text or json',
+            default: 'text',
+        },
+        {
+            name: 'concurrency',
+            short: 'c',
+            type: 'number',
+            description: 'Number of questions to run in parallel (default: 3)',
+            default: '3',
+        },
+        {
+            name: 'smoke-only',
+            type: 'boolean',
+            description: 'Use the 5-question smoke fixture instead of real HF dataset (no HF token required)',
+            default: 'false',
+        },
+        {
+            name: 'max-turns',
+            type: 'number',
+            description: 'Maximum agent turns per question (default: 12). Overridden per-question when --hardness-routing is enabled.',
+            default: '12',
+        },
+        {
+            name: 'judge-model',
+            type: 'string',
+            description: 'Model for LLM-as-judge scoring (default: claude-sonnet-4-6)',
+            default: 'claude-sonnet-4-6',
+        },
+        {
+            name: 'voting-attempts',
+            type: 'number',
+            description: 'Number of parallel attempts for majority-vote self-consistency (default: 1 = no voting). N>1 costs Nx per question. Recommended: 3 (+5-10pp L1 lift per ADR-135 Track A). Overridden per-question when --hardness-routing is enabled.',
+            default: '1',
+        },
+        {
+            name: 'hardness-routing',
+            type: 'boolean',
+            description: 'ADR-136 Track Q: enable hardness-based compute routing. Trains a linear classifier from historical result JSONs and allocates: easy=Haiku/4t/1-attempt, medium=Sonnet/8t/1-attempt, hard=Sonnet/12t/3-vote. Overrides --max-turns and --voting-attempts per question.',
+            default: 'false',
+        },
+        {
+            name: 'hardness-verbose',
+            type: 'boolean',
+            description: 'ADR-136 Track Q: log hardness prediction for each question (requires --hardness-routing).',
+            default: 'false',
+        },
+        {
+            name: 'enable-critic',
+            type: 'boolean',
+            description: 'ADR-135 Track D: enable adversarial critic review after agent answer (+3-5pp L1 lift expected). Skipped when --voting-attempts > 1 (cost containment — voting takes precedence).',
+            default: 'false',
+        },
+        {
+            name: 'decompose',
+            type: 'boolean',
+            description: 'ADR-135 Track E: decompose complex questions into 1-5 sub-questions before solving (+5-10pp on multi-step Qs, ~30-40% of L1 set). Each sub-question runs through voting/critic/plain independently; sub-answers are synthesized before judging.',
+            default: 'false',
+        },
+        {
+            name: 'planning-interval',
+            type: 'number',
+            description: 'ADR-135 Track B: inject a planning checkpoint every N tool_use turns (default: 4, set 0 to disable). Based on smolagents finding — prevents tunnel-vision on bad strategies.',
+            default: '4',
+        },
+        {
+            name: 'enable-convergence',
+            type: 'boolean',
+            description: 'iter 62: enable convergence layer — forces a final commit when max_turns, loop, or token_overflow is detected (default: true). Disabling is for ablation only.',
+            default: 'true',
+        },
+    ],
+    examples: [
+        {
+            command: 'claude-flow gaia-bench run --level 1 --limit 10 --models claude-haiku-4-5 --output json',
+            description: 'Run 10 Level-1 questions with Haiku, JSON output',
+        },
+        {
+            command: 'claude-flow gaia-bench run --level 1 --limit 10 --models claude-haiku-4-5,claude-sonnet-4-6',
+            description: 'Compare Haiku vs Sonnet on 10 Level-1 questions',
+        },
+        {
+            command: 'claude-flow gaia-bench run --smoke-only --output json',
+            description: 'Quick smoke test (5 fixture questions, no HF token needed)',
+        },
+        {
+            command: 'claude-flow gaia-bench run --level 1 --limit 20 --models claude-haiku-4-5 --voting-attempts 3 --output json',
+            description: 'Self-consistency voting: run each question 3x, majority-vote (ADR-135 Track A, +5-10pp expected)',
+        },
+        {
+            command: 'claude-flow gaia-bench run --level 1 --models claude-sonnet-4-6 --hardness-routing --output json',
+            description: 'ADR-136 Track Q: auto-route questions to Haiku/Sonnet based on predicted difficulty',
+        },
+        {
+            command: 'claude-flow gaia-bench run --level 1 --models claude-sonnet-4-6 --enable-critic --output json',
+            description: 'ADR-135 Track D: adversarial critic reviews each answer before submission (+3-5pp expected)',
+        },
+        {
+            command: 'claude-flow gaia-bench run --level 1 --models claude-sonnet-4-6 --decompose --output json',
+            description: 'ADR-135 Track E: decompose complex questions into sub-questions (+5-10pp on multi-step Qs)',
+        },
+        {
+            command: 'claude-flow gaia-bench run --level 1 --models claude-sonnet-4-6 --hardness-routing --enable-critic --planning-interval 4',
+            description: 'Recommended config: hardness routing + critic + planning checkpoints (~$2/run est.)',
+        },
+    ],
+    action: async (ctx) => {
+        const level = parseInt(String(ctx.flags.level ?? '1'), 10);
+        const limit = ctx.flags.limit ? parseInt(String(ctx.flags.limit), 10) : undefined;
+        const modelsRaw = String(ctx.flags.models ?? 'claude-haiku-4-5');
+        const models = modelsRaw.split(',').map((m) => m.trim()).filter(Boolean);
+        const outputFormat = String(ctx.flags.output ?? 'text');
+        const concurrency = parseInt(String(ctx.flags.concurrency ?? '3'), 10);
+        // Parser converts --smoke-only to camelCase "smokeOnly"
+        const smokeOnly = ctx.flags['smokeOnly'] === true || ctx.flags['smokeOnly'] === 'true' ||
+            ctx.flags['smoke-only'] === true || ctx.flags['smoke-only'] === 'true';
+        // Parser converts --max-turns to maxTurns, --judge-model to judgeModel, --voting-attempts to votingAttempts
+        // NOTE: default must match DEFAULT_MAX_TURNS in benchmarks/gaia-agent.ts
+        const maxTurns = parseInt(String(ctx.flags['maxTurns'] ?? ctx.flags['max-turns'] ?? '12'), 10);
+        const judgeModel = String(ctx.flags['judgeModel'] ?? ctx.flags['judge-model'] ?? 'claude-sonnet-4-6');
+        // votingAttempts=1 means no voting (backward-compat default).  N>1 routes through runGaiaAgentWithVoting.
+        const votingAttempts = parseInt(String(ctx.flags['votingAttempts'] ?? ctx.flags['voting-attempts'] ?? '1'), 10);
+        const useVoting = votingAttempts > 1;
+        // ADR-136 Track Q: hardness-based routing.
+        const hardnessRouting = ctx.flags['hardnessRouting'] === true || ctx.flags['hardnessRouting'] === 'true' ||
+            ctx.flags['hardness-routing'] === true || ctx.flags['hardness-routing'] === 'true';
+        const hardnessVerbose = ctx.flags['hardnessVerbose'] === true || ctx.flags['hardnessVerbose'] === 'true' ||
+            ctx.flags['hardness-verbose'] === true || ctx.flags['hardness-verbose'] === 'true';
+        // ADR-135 Track D: adversarial critic.
+        // Voting takes precedence over critic when both are enabled (cost containment).
+        const enableCritic = !useVoting && (ctx.flags['enableCritic'] === true || ctx.flags['enableCritic'] === 'true' ||
+            ctx.flags['enable-critic'] === true || ctx.flags['enable-critic'] === 'true');
+        // ADR-135 Track E: question decomposition.
+        const enableDecompose = ctx.flags['decompose'] === true || ctx.flags['decompose'] === 'true';
+        // ADR-135 Track B: planning interval (passed through to runGaiaAgent via agentOpts).
+        const planningInterval = parseInt(String(ctx.flags['planningInterval'] ?? ctx.flags['planning-interval'] ?? '4'), 10);
+        // iter 62: convergence layer — default ON, disable with --no-enable-convergence.
+        // Note: boolean false is falsy, so we check for explicit false values only.
+        const enableConvergence = !(ctx.flags['enableConvergence'] === false || ctx.flags['enableConvergence'] === 'false' ||
+            ctx.flags['enable-convergence'] === false || ctx.flags['enable-convergence'] === 'false');
+        // Dynamic imports to avoid loading at startup.
+        // NOTE: gaia-*.ts sources are pre-compiled under dist/src/benchmarks/ only --
+        // they are NOT in the src/ include glob so TypeScript cannot resolve them
+        // statically.  We resolve the absolute path from import.meta.url at runtime
+        // and cast to `any` to bypass the static-analysis check.
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const benchmarksBase = new URL('../benchmarks/', import.meta.url).href;
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const { loadGaia } = (await import(benchmarksBase + 'gaia-loader.js'));
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const { runGaiaAgent } = (await import(benchmarksBase + 'gaia-agent.js'));
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const { judgeAnswer } = (await import(benchmarksBase + 'gaia-judge.js'));
+        // ADR-135 Track A: voting wrapper (imported when --voting-attempts > 1 OR hardness routing triggers it).
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const { runGaiaAgentWithVoting } = (useVoting || hardnessRouting)
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            ? (await import(benchmarksBase + 'gaia-voting.js'))
+            : { runGaiaAgentWithVoting: null };
+        // ADR-135 Track D: critic wrapper (only imported when --enable-critic and no voting).
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const { runGaiaAgentWithCritic } = enableCritic
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            ? (await import(benchmarksBase + 'gaia-critic.js'))
+            : { runGaiaAgentWithCritic: null };
+        // ADR-135 Track E: decomposer (only imported when --decompose).
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        let decomposeQuestion = null;
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        let synthesizeFromSubAnswers = null;
+        if (enableDecompose) {
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            const decomposerMod = (await import(benchmarksBase + 'gaia-decomposer.js'));
+            decomposeQuestion = decomposerMod.decomposeQuestion;
+            synthesizeFromSubAnswers = decomposerMod.synthesizeFromSubAnswers;
+        }
+        // ADR-136 Track Q: hardness predictor.
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        let hardnessPredictor = null;
+        if (hardnessRouting) {
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            const { HardnessPredictor } = (await import(benchmarksBase + 'gaia-hardness/predictor.js'));
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            const { loadTrainingData } = (await import(benchmarksBase + 'gaia-hardness/train-data-loader.js'));
+            hardnessPredictor = new HardnessPredictor({ conservativeMode: true });
+            const trainingData = loadTrainingData([], hardnessVerbose);
+            if (trainingData.length >= 10) {
+                hardnessPredictor.train(trainingData);
+            }
+            // If < 10 examples: cold-start (medium for all) -- documented fallback.
+        }
+        // Only print to stderr so stdout stays clean for JSON consumers
+        const log = (msg) => {
+            if (outputFormat !== 'json') {
+                output.writeln(msg);
+            }
+            else {
+                process.stderr.write(msg + '\n');
+            }
+        };
+        log('');
+        log(output.bold(`GAIA Benchmark -- Level ${level}${smokeOnly ? ' [SMOKE]' : ''}`));
+        log(output.dim('-'.repeat(60)));
+        log(`Models  : ${models.join(', ')}`);
+        log(`Limit   : ${limit ?? 'all'}`);
+        log(`Concurrency: ${concurrency}`);
+        if (useVoting && !hardnessRouting) {
+            log(`Voting  : ${votingAttempts}x self-consistency (ADR-135 Track A) -- cost ~${votingAttempts}x per question`);
+        }
+        if (enableCritic) {
+            log(`Critic  : ADR-135 Track D enabled -- adversarial review after each answer`);
+        }
+        if (enableDecompose) {
+            log(`Decompose: ADR-135 Track E enabled -- multi-step questions will be split into sub-questions`);
+        }
+        if (planningInterval > 0) {
+            log(`Planning: ADR-135 Track B -- checkpoint every ${planningInterval} turns`);
+        }
+        if (hardnessRouting) {
+            const trainedStatus = hardnessPredictor?.isTrained
+                ? 'trained (classifier active)'
+                : 'cold-start (no training data -> all medium)';
+            log(`Hardness: ADR-136 Track Q enabled -- ${trainedStatus}`);
+            log('          easy=Haiku/4t/1-attempt  medium=Sonnet/8t/1-attempt  hard=Sonnet/12t/3-vote');
+        }
+        log('');
+        // Load questions
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        let questions;
+        try {
+            questions = await loadGaia({ level, limit, smokeOnly });
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            if (outputFormat === 'json') {
+                process.stdout.write(JSON.stringify({ error: `Failed to load GAIA dataset: ${msg}` }, null, 2) + '\n');
+            }
+            else {
+                output.writeln(output.error(`Failed to load GAIA dataset: ${msg}`));
+            }
+            return { success: false };
+        }
+        log(`Loaded  : ${questions.length} questions`);
+        log('');
+        const allModelOutputs = [];
+        // Resolve the API key once (shared across all per-question agent calls).
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const { resolveAnthropicApiKey } = (await import(benchmarksBase + 'gaia-agent.js'));
+        const apiKey = resolveAnthropicApiKey();
+        for (const model of models) {
+            log(output.bold(`Running model: ${model}`));
+            log(output.dim('-'.repeat(40)));
+            const results = [];
+            let totalInputTokens = 0;
+            let totalOutputTokens = 0;
+            let totalTurns = 0;
+            let totalWallMs = 0;
+            // ADR-136 Track Q: hardness distribution tracking.
+            const hardnessDist = { easy: 0, medium: 0, hard: 0 };
+            // Process questions in batches of `concurrency`
+            for (let i = 0; i < questions.length; i += concurrency) {
+                const batch = questions.slice(i, Math.min(i + concurrency, questions.length));
+                const batchResults = await Promise.all(batch.map(async (q) => {
+                    const qIdx = i + batch.indexOf(q) + 1;
+                    // ADR-136 Track Q: predict hardness and set per-question compute budget.
+                    let effectiveModel = model;
+                    let effectiveMaxTurns = maxTurns;
+                    let effectiveVotingAttempts = votingAttempts;
+                    let predictedDifficulty;
+                    let predictedConfidence;
+                    if (hardnessRouting && hardnessPredictor) {
+                        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                        const prediction = hardnessPredictor.predict(q);
+                        predictedDifficulty = prediction.difficulty;
+                        predictedConfidence = prediction.confidence;
+                        hardnessDist[predictedDifficulty]++;
+                        // Override compute budget from hardness policy.
+                        const budget = prediction.budget;
+                        effectiveModel = budget.model === 'haiku'
+                            ? 'claude-haiku-4-5'
+                            : (model.includes('sonnet') ? model : 'claude-sonnet-4-6');
+                        effectiveMaxTurns = budget.maxTurns;
+                        effectiveVotingAttempts = budget.votingAttempts;
+                        if (hardnessVerbose) {
+                            log(`  [${qIdx}/${questions.length}] ${q.task_id} hardness=${predictedDifficulty}` +
+                                ` conf=${((predictedConfidence ?? 0) * 100).toFixed(0)}%` +
+                                ` -> ${effectiveModel} / ${effectiveMaxTurns}t / ${effectiveVotingAttempts}-attempt`);
+                        }
+                        else {
+                            log(`  [${qIdx}/${questions.length}] ${q.task_id} [${predictedDifficulty}] -- ${String(q.question).slice(0, 50)}...`);
+                        }
+                    }
+                    else {
+                        log(`  [${qIdx}/${questions.length}] ${q.task_id} -- ${String(q.question).slice(0, 60)}...`);
+                    }
+                    const useThisVoting = effectiveVotingAttempts > 1;
+                    // Critic is suppressed when voting is active (same precedence rule as the global flag).
+                    const useThisCritic = enableCritic && !useThisVoting && runGaiaAgentWithCritic;
+                    // Shared agent options (Track B planning interval + iter 62 convergence wired here).
+                    const agentOpts = {
+                        model: effectiveModel,
+                        maxTurns: effectiveMaxTurns,
+                        planningInterval,
+                        apiKey,
+                        enableConvergence,
+                    };
+                    // ADR-135 Track E: decompose the question if enabled.
+                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                    let decomposedResult = null;
+                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                    let questionsToSolve = [q];
+                    if (enableDecompose && decomposeQuestion) {
+                        try {
+                            decomposedResult = await decomposeQuestion(q.question, { apiKey });
+                            if (decomposedResult?.decomposed === true && Array.isArray(decomposedResult.subQuestions) && decomposedResult.subQuestions.length > 1) {
+                                questionsToSolve = decomposedResult.subQuestions.map((sq) => ({ ...q, question: sq }));
+                                log(`    decomposed into ${questionsToSolve.length} sub-questions`);
+                            }
+                        }
+                        catch {
+                            // Graceful fallback: treat as atomic question.
+                            decomposedResult = null;
+                        }
+                    }
+                    // Solve each (sub-)question.
+                    const subAnswers = [];
+                    let lastAgentResult = null; // eslint-disable-line @typescript-eslint/no-explicit-any
+                    let solveError;
+                    for (const sq of questionsToSolve) {
+                        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                        let agentResult;
+                        try {
+                            if (useThisVoting && runGaiaAgentWithVoting) {
+                                // ADR-135 Track A: multi-attempt majority voting.
+                                agentResult = await runGaiaAgentWithVoting(sq, {
+                                    ...agentOpts,
+                                    attempts: effectiveVotingAttempts,
+                                });
+                                const vr = agentResult;
+                                log(`    vote-method=${vr.votingMethod ?? '?'}  agreement=${vr.agreementCount ?? '?'}/${effectiveVotingAttempts}`);
+                            }
+                            else if (useThisCritic) {
+                                // ADR-135 Track D: critic-wrapped agent.
+                                agentResult = await runGaiaAgentWithCritic(sq, {
+                                    ...agentOpts,
+                                    enableCritic: true,
+                                });
+                                const cr = agentResult;
+                                log(`    critic-verdict=${cr.criticVerdict ?? '?'}`);
+                            }
+                            else {
+                                agentResult = await runGaiaAgent(sq, agentOpts);
+                            }
+                        }
+                        catch (err) {
+                            solveError = err instanceof Error ? err.message : String(err);
+                            log(`    ERROR: ${solveError}`);
+                            break;
+                        }
+                        subAnswers.push(agentResult.finalAnswer ?? '');
+                        lastAgentResult = agentResult;
+                    }
+                    if (solveError || !lastAgentResult) {
+                        return {
+                            task_id: q.task_id,
+                            question: q.question,
+                            model: effectiveModel,
+                            correct: false,
+                            answer: null,
+                            expected_output: q.final_answer,
+                            error: solveError ?? 'no result',
+                            hardnessDifficulty: predictedDifficulty,
+                            hardnessConfidence: predictedConfidence,
+                            decomposed: decomposedResult?.decomposed === true,
+                        };
+                    }
+                    // ADR-135 Track E: synthesize sub-answers if decomposed.
+                    let finalAnswer = subAnswers[0] ?? null;
+                    if (decomposedResult?.decomposed === true && questionsToSolve.length > 1 && synthesizeFromSubAnswers) {
+                        try {
+                            const synth = await synthesizeFromSubAnswers(decomposedResult, subAnswers, { apiKey });
+                            finalAnswer = synth.finalAnswer ?? finalAnswer;
+                        }
+                        catch {
+                            // Graceful fallback: use first sub-answer.
+                        }
+                    }
+                    // Judge the answer
+                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                    let judgeResult;
+                    try {
+                        judgeResult = await judgeAnswer({ id: q.task_id, expected: q.final_answer, questionText: q.question }, finalAnswer, { judgeModel });
+                    }
+                    catch (err) {
+                        const errorMsg = err instanceof Error ? err.message : String(err);
+                        judgeResult = {
+                            questionId: q.task_id,
+                            passed: false,
+                            scoringPath: 'exact-match',
+                            candidateAnswer: finalAnswer ?? '',
+                            groundTruth: q.final_answer,
+                            judgeReason: `Judge error: ${errorMsg}`,
+                        };
+                    }
+                    const verdict = judgeResult.passed ? output.success('PASS') : output.error('FAIL');
+                    log(`    ${verdict}  answer="${finalAnswer ?? 'null'}"  expected="${q.final_answer}"` +
+                        `  turns=${lastAgentResult.turns}  ${(lastAgentResult.wallMs / 1000).toFixed(1)}s`);
+                    return {
+                        task_id: q.task_id,
+                        question: q.question,
+                        model: effectiveModel,
+                        correct: judgeResult.passed,
+                        answer: finalAnswer,
+                        expected_output: q.final_answer,
+                        error: lastAgentResult.error,
+                        turns: lastAgentResult.turns,
+                        wallMs: lastAgentResult.wallMs,
+                        inputTokens: lastAgentResult.totalInputTokens,
+                        outputTokens: lastAgentResult.totalOutputTokens,
+                        hardnessDifficulty: predictedDifficulty,
+                        hardnessConfidence: predictedConfidence,
+                        decomposed: decomposedResult?.decomposed === true,
+                    };
+                }));
+                for (const r of batchResults) {
+                    results.push(r);
+                    totalInputTokens += r.inputTokens ?? 0;
+                    totalOutputTokens += r.outputTokens ?? 0;
+                    totalTurns += r.turns ?? 0;
+                    totalWallMs += r.wallMs ?? 0;
+                }
+            }
+            const passed = results.filter((r) => r.correct).length;
+            const total = results.length;
+            const passRate = total > 0 ? passed / total : 0;
+            const estCostUsd = estimateCost(model, totalInputTokens, totalOutputTokens);
+            const modelOutput = {
+                level,
+                model,
+                summary: {
+                    total,
+                    passed,
+                    passRate,
+                    estCostUsd,
+                    meanTurns: total > 0 ? totalTurns / total : 0,
+                    meanWallMs: total > 0 ? totalWallMs / total : 0,
+                    ...(hardnessRouting ? { hardnessDist } : {}),
+                },
+                results,
+            };
+            allModelOutputs.push(modelOutput);
+            log('');
+            log(output.bold(`Results for ${model}:`));
+            log(`  Pass rate : ${passed}/${total} (${(passRate * 100).toFixed(1)}%)`);
+            log(`  Est. cost : $${estCostUsd.toFixed(4)}`);
+            log(`  Mean turns: ${modelOutput.summary.meanTurns.toFixed(1)}`);
+            log(`  Mean time : ${(modelOutput.summary.meanWallMs / 1000).toFixed(1)}s per question`);
+            if (hardnessRouting) {
+                log(`  Hardness  : easy=${hardnessDist.easy} medium=${hardnessDist.medium} hard=${hardnessDist.hard}`);
+            }
+            log('');
+        }
+        // Output results
+        if (outputFormat === 'json') {
+            if (allModelOutputs.length === 1) {
+                // Single model: emit flat object (matches workflow contract)
+                process.stdout.write(JSON.stringify(allModelOutputs[0], null, 2) + '\n');
+            }
+            else {
+                // Multiple models: emit array
+                process.stdout.write(JSON.stringify(allModelOutputs, null, 2) + '\n');
+            }
+        }
+        else {
+            // Print summary table
+            output.writeln(output.bold('Summary'));
+            output.writeln(output.dim('-'.repeat(60)));
+            for (const m of allModelOutputs) {
+                const pct = (m.summary.passRate * 100).toFixed(1);
+                output.writeln(`${m.model.padEnd(28)} ${m.summary.passed}/${m.summary.total} (${pct}%)` +
+                    `  cost=$${m.summary.estCostUsd.toFixed(4)}` +
+                    `  turns=${m.summary.meanTurns.toFixed(1)}`);
+            }
+        }
+        return { success: true };
+    },
+};
+// ---------------------------------------------------------------------------
+// Main gaia-bench command
+// ---------------------------------------------------------------------------
+export const gaiaBenchCommand = {
+    name: 'gaia-bench',
+    description: 'GAIA benchmark harness -- measure agent pass-rate on real GAIA questions',
+    subcommands: [runCommand],
+    examples: [
+        {
+            command: 'claude-flow gaia-bench run --level 1 --limit 10 --models claude-haiku-4-5 --output json',
+            description: 'Mini Level-1 run with Haiku, JSON output',
+        },
+        {
+            command: 'claude-flow gaia-bench run --smoke-only',
+            description: 'Quick smoke test with built-in fixture (no HF token)',
+        },
+        {
+            command: 'claude-flow gaia-bench run --level 1 --models claude-sonnet-4-6 --hardness-routing --output json',
+            description: 'ADR-136 Track Q: tiered compute routing by predicted question difficulty',
+        },
+        {
+            command: 'claude-flow gaia-bench run --level 1 --models claude-sonnet-4-6 --hardness-routing --enable-critic --planning-interval 4',
+            description: 'Recommended config: all tracks active, ~$2/run estimated',
+        },
+    ],
+};
+export default gaiaBenchCommand;
+//# sourceMappingURL=gaia-bench.js.map

package/v3/@claude-flow/cli/dist/src/commands/index.js

@@ -67,6 +67,8 @@ const commandLoaders = {
     'transfer-store': () => import('./transfer-store.js'),
     cleanup: () => import('./cleanup.js'),
     autopilot: () => import('./autopilot.js'),
+    // GAIA Benchmark Harness (ADR-133)
+    'gaia-bench': () => import('./gaia-bench.js'),
 };
 // Cache for loaded commands
 const loadedCommands = new Map();

package/v3/@claude-flow/cli/dist/src/commands/init.js

@@ -129,13 +129,50 @@ async function initCodexAction(ctx, options) {
         return { success: false, exitCode: 1 };
     }
 }
-// Check if project is already initialized
+// Check if project is already initialized with ruflo.
+// #2207: .claude/settings.json alone is NOT a ruflo marker — it's created by
+// Claude Code itself and exists in every Claude Code project. We require a
+// ruflo-specific signal: either a claudeFlow section in settings.json, OR a
+// .mcp.json with a 'claude-flow' or 'ruflo' server key, OR the ruflo-only
+// .claude-flow/config.yaml. Using the bare file-existence check was causing
+// false-positives for new users whose only existing file was Claude Code's own
+// settings.json.
 function isInitialized(cwd) {
-    const claudePath = path.join(cwd, '.claude', 'settings.json');
     const claudeFlowPath = path.join(cwd, '.claude-flow', 'config.yaml');
+    const mcpJsonPath = path.join(cwd, '.mcp.json');
+    const settingsPath = path.join(cwd, '.claude', 'settings.json');
+    // Check .claude-flow/config.yaml — ruflo-specific, always reliable
+    const hasClaudeFlow = fs.existsSync(claudeFlowPath);
+    // Check .claude/settings.json for ruflo-specific content (claudeFlow section)
+    let hasRufloSettings = false;
+    if (fs.existsSync(settingsPath)) {
+        try {
+            const parsed = JSON.parse(fs.readFileSync(settingsPath, 'utf-8'));
+            hasRufloSettings =
+                parsed != null &&
+                    typeof parsed === 'object' &&
+                    'claudeFlow' in parsed;
+        }
+        catch { /* malformed — ignore */ }
+    }
+    // Check .mcp.json for ruflo/claude-flow server key
+    let hasRufloMcp = false;
+    if (fs.existsSync(mcpJsonPath)) {
+        try {
+            const parsed = JSON.parse(fs.readFileSync(mcpJsonPath, 'utf-8'));
+            hasRufloMcp =
+                parsed != null &&
+                    typeof parsed === 'object' &&
+                    parsed.mcpServers != null &&
+                    typeof parsed.mcpServers === 'object' &&
+                    ('claude-flow' in parsed.mcpServers ||
+                        'ruflo' in parsed.mcpServers);
+        }
+        catch { /* malformed — ignore */ }
+    }
     return {
-        claude: fs.existsSync(claudePath),
-        claudeFlow: fs.existsSync(claudeFlowPath),
+        claude: hasRufloSettings || hasRufloMcp,
+        claudeFlow: hasClaudeFlow,
     };
 }
 // Init subcommand (default)