@claude-flow/cli 3.10.46 → 3.11.0

package/dist/src/benchmarks/gaia-codeagent.js

@@ -0,0 +1,590 @@
+/**
+ * GAIA CodeAgent — ADR-138 iter 54 (FINAL design)
+ *
+ * smolagents-style CodeAgent harness for the GAIA L1 benchmark.
+ * NOT the smolagents library — this is the PATTERN implemented natively in ruflo TS.
+ *
+ * Architecture (HAL replication):
+ *   Instead of Anthropic native tool_use blocks (JSON → execute → repeat),
+ *   the agent writes Python code that calls tools as functions.  The code is
+ *   parsed from markdown code blocks, executed in gaia-codeagent-runner.py,
+ *   and the stdout is fed back as the next user turn.  The agent commits its
+ *   answer by calling final_answer("value") in Python.
+ *
+ * Why this beats ToolCallingAgent on GAIA (from HAL-DEEP-STUDY.md):
+ *   - 30% fewer steps for the same task (Python is more expressive than JSON)
+ *   - Variables persist across steps in the agent's mental model
+ *   - Complex control flow (loops, try/except) is native
+ *   - final_answer() is deterministic — no regex extraction fragility
+ *
+ * Loop algorithm:
+ *   1. Build system prompt with tool signatures and GAIA instruction template.
+ *   2. Call Anthropic Messages API (text-in / text-out — NO tools array).
+ *   3. Parse the response for a ```python ... ``` code block.
+ *   4. If code block found: run gaia-codeagent-runner.py subprocess.
+ *      - Runner pre-defines tool functions (web_search, visit_webpage, etc.)
+ *      - If final_answer("X") is called in the code: extract X, return result.
+ *      - Otherwise: feed stdout back as user turn, continue.
+ *   5. If no code block: prompt agent to produce one (max 3 retries).
+ *   6. If maxTurns exceeded: return timedOut=true.
+ *   7. Every planningInterval turns: inject a planning checkpoint.
+ *
+ * Tool routing (via gaia-codeagent-runner.py):
+ *   web_search(query)      → claude -p with WebSearch (best web coverage)
+ *   visit_webpage(url)     → requests + bs4 HTML extraction
+ *   grounded_query(query)  → Gemini with Google Search grounding (ruflo unique)
+ *   read_file(path)        → Python direct (text/csv/json/xlsx) or subprocess
+ *   describe_image(path)   → claude -p with vision
+ *   final_answer(x)        → writes sentinel JSON and exits runner
+ *
+ * Key parameters:
+ *   model:            claude-sonnet-4-6 (default; ADR-138 targets Sonnet 4.5+)
+ *   maxTurns:         20 (HAL uses 200; 20 is cost-controlled for L1)
+ *   planningInterval: 4 (match HAL's planning_interval=4)
+ *   maxTokensPerTurn: 4096 (code generation needs more space than ToolCalling)
+ *
+ * Refs: ADR-138, HAL-DEEP-STUDY.md, smolagents CodeAgent, #2156, iter 54
+ */
+import { spawnSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as os from 'node:os';
+import { SMOKE_FIXTURE, } from './gaia-loader.js';
+import { resolveAnthropicApiKey, isAnswerCorrect } from './gaia-agent.js';
+// ---------------------------------------------------------------------------
+// Re-export for callers
+// ---------------------------------------------------------------------------
+export { isAnswerCorrect };
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+const ANTHROPIC_API_URL = 'https://api.anthropic.com/v1/messages';
+const ANTHROPIC_API_VERSION = '2023-06-01';
+/** Default model: Sonnet 4.6 (ADR-138 targets Sonnet 4.5+; 4.6 is available). */
+const DEFAULT_MODEL = 'claude-sonnet-4-6';
+const DEFAULT_MAX_TURNS = 20;
+const DEFAULT_MAX_TOKENS_PER_TURN = 4096;
+const DEFAULT_PER_TURN_TIMEOUT_MS = 90_000;
+const DEFAULT_PLANNING_INTERVAL = 4;
+const DEFAULT_PER_STEP_TIMEOUT_MS = 30_000;
+const MAX_OBSERVATION_CHARS = 6_000;
+const MAX_NO_CODE_RETRIES = 3;
+/** Pattern for final_answer in text output (GAIA format, fallback). */
+const FINAL_ANSWER_TEXT_RE = /FINAL_ANSWER:\s*(.+)/i;
+/** Path to the Python runner script — co-located with this file. */
+function getRunnerPath() {
+    const RUNNER_NAME = 'gaia-codeagent-runner.py';
+    // Primary: resolve relative to this compiled JS file (dist/src/benchmarks/)
+    try {
+        const thisFile = fileURLToPath(import.meta.url);
+        const candidate = path.join(path.dirname(thisFile), RUNNER_NAME);
+        if (fs.existsSync(candidate))
+            return candidate;
+    }
+    catch { /* ESM URL resolution failed */ }
+    // Secondary: look for the Python runner next to the current file via __dirname-style
+    // heuristic — for Jest/ts-node contexts where import.meta.url is file://...src/...
+    try {
+        const thisFile = fileURLToPath(import.meta.url);
+        // Walk up the dist/ tree to find src/benchmarks/ sibling
+        const distBenchmarks = path.dirname(thisFile);
+        const srcBenchmarks = distBenchmarks.replace(path.join('dist', 'src', 'benchmarks'), path.join('src', 'benchmarks'));
+        const srcCandidate = path.join(srcBenchmarks, RUNNER_NAME);
+        if (fs.existsSync(srcCandidate))
+            return srcCandidate;
+    }
+    catch { /* ignore */ }
+    // Tertiary: CJS fallback (process.argv[1] path)
+    const fallback = path.join(path.dirname(process.argv[1] ?? '.'), RUNNER_NAME);
+    return fallback;
+}
+// ---------------------------------------------------------------------------
+// System prompt
+// ---------------------------------------------------------------------------
+/**
+ * Build the CodeAgent system prompt.
+ *
+ * The prompt explains the coding loop to the model and lists all available
+ * tools with their signatures.  This mirrors HAL's CodeAgent system prompt
+ * format as documented in HAL-DEEP-STUDY.md.
+ */
+function buildCodeAgentSystemPrompt() {
+    const toolDocs = [
+        '  web_search(query: str) -> str',
+        '    Search the web and return snippets. Use for factual lookups.',
+        '',
+        '  visit_webpage(url: str) -> str',
+        '    Fetch full text content of a webpage. Use after web_search to read a page.',
+        '',
+        '  grounded_query(query: str) -> str',
+        '    Ask Gemini with Google grounding — returns synthesized answer with citations.',
+        '    Best for factoid questions. Use this OR web_search (not both for same query).',
+        '',
+        '  read_file(path: str) -> str',
+        '    Read a file — supports txt, csv, json, xlsx, pptx, pdf, images.',
+        '    Use the ATTACHMENT_PATH variable if a file was provided with the question.',
+        '',
+        '  describe_image(path: str) -> str',
+        '    Describe an image using Claude vision.',
+        '',
+        '  final_answer(answer) -> never',
+        '    Submit your final answer. This ENDS the loop immediately.',
+    ].join('\n');
+    return [
+        'You are a Python-capable agent solving GAIA benchmark questions.',
+        'To use tools, write Python code in ```python blocks and call tool functions directly.',
+        'After each code block, you will receive the output. Continue until you have the answer.',
+        '',
+        'TOOL FUNCTIONS AVAILABLE (call these in your Python code):',
+        toolDocs,
+        '',
+        'GLOBAL VARIABLES:',
+        '  ATTACHMENT_PATH: str  (path to attached file, or empty string if no attachment)',
+        '',
+        'PRE-IMPORTED: math, re, json, datetime, collections, itertools, statistics,',
+        '  fractions, decimal, string, unicodedata, urllib.parse, pathlib.Path,',
+        '  pandas as pd (optional), numpy as np (optional), sympy (optional)',
+        '',
+        'CODING RULES:',
+        '1. Write Python code in ```python\\n...\\n``` blocks.',
+        '2. Use print() to see tool outputs and intermediate results.',
+        '3. Call final_answer("your answer here") when you are certain of the answer.',
+        '4. Handle errors with try/except — do not crash on tool failures.',
+        '5. Use ATTACHMENT_PATH to access attached files when provided.',
+        '',
+        'GAIA ANSWER FORMAT (CRITICAL — follow exactly):',
+        'Return only the answer, which should be:',
+        '- A number: no commas, no units unless specified. Do not round unless asked.',
+        '- A string: no articles (a/an/the), no abbreviations. Spell out digits unless specified.',
+        '- A comma-separated list: apply the above per element.',
+        '',
+        'IMPORTANT: Do not include extra explanation in final_answer(). Just the bare answer.',
+        'Example: final_answer("3") or final_answer("Paris") or final_answer("2, 4, 6")',
+    ].join('\n');
+}
+// ---------------------------------------------------------------------------
+// Planning checkpoint
+// ---------------------------------------------------------------------------
+function buildCodeAgentPlanningCheckpoint(turn, maxTurns) {
+    return (`[PLANNING CHECKPOINT — step ${turn}/${maxTurns}]\n` +
+        `You have used ${turn} steps so far. Before continuing:\n` +
+        `1. Briefly summarize what you have learned from tool calls so far.\n` +
+        `2. State explicitly: is your current approach making progress toward the answer?\n` +
+        `3. If NOT making progress: switch strategy. Different tool, different query, different approach.\n` +
+        `4. If you are confident in an answer: call final_answer("your answer") now.\n` +
+        `Continue with your next Python code block.`);
+}
+// ---------------------------------------------------------------------------
+// Code block parser
+// ---------------------------------------------------------------------------
+/**
+ * Extract the first ```python ... ``` code block from assistant output.
+ * Returns null if no code block is found.
+ *
+ * Exported for use in unit tests (gaia-codeagent.smoke.ts T1).
+ */
+export function extractCodeBlock(text) {
+    // Match ```python or ```py (with optional language tag) then newline then content
+    const withLangMatch = /```(?:python|py)\s*\n([\s\S]*?)```/.exec(text);
+    if (withLangMatch)
+        return withLangMatch[1].trim();
+    // Bare ``` with no language tag
+    const lenientMatch = /```\s*\n([\s\S]*?)```/.exec(text);
+    if (lenientMatch)
+        return lenientMatch[1].trim();
+    return null;
+}
+// ---------------------------------------------------------------------------
+// Python execution via gaia-codeagent-runner.py subprocess
+// ---------------------------------------------------------------------------
+/**
+ * Execute one step of agent code via gaia-codeagent-runner.py.
+ *
+ * The runner:
+ * 1. Pre-defines tool functions (web_search, visit_webpage, grounded_query,
+ *    read_file, describe_image, final_answer)
+ * 2. exec()s the agent's code in that context
+ * 3. If final_answer() is called: writes GAIA_RESULT_FILE JSON and exits 0
+ * 4. stdout is captured as the observation for the next turn
+ *
+ * Returns { output, finalAnswer }.
+ */
+function executeAgentCodeStep(code, attachmentPath, stepTimeoutMs, apiKey, toolCallsByName) {
+    const runnerPath = getRunnerPath();
+    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gaia-ca-'));
+    const codeFile = path.join(tmpDir, 'agent_code.py');
+    const resultFile = path.join(tmpDir, 'final_answer.json');
+    try {
+        fs.writeFileSync(codeFile, code, 'utf-8');
+        const env = {
+            ...process.env,
+            PYTHONIOENCODING: 'utf-8',
+            GAIA_CODE_FILE: codeFile,
+            GAIA_RESULT_FILE: resultFile,
+            ANTHROPIC_API_KEY: apiKey,
+        };
+        if (attachmentPath) {
+            env.GAIA_ATTACHMENT_PATH = attachmentPath;
+        }
+        const result = spawnSync('python3', [runnerPath], {
+            encoding: 'utf-8',
+            timeout: stepTimeoutMs,
+            env,
+            maxBuffer: 10 * 1024 * 1024,
+        });
+        // Count tool calls detected in stdout (heuristic from output markers)
+        const stdout = result.stdout ?? '';
+        const toolPatterns = ['web_search', 'visit_webpage', 'grounded_query', 'read_file', 'describe_image'];
+        for (const toolName of toolPatterns) {
+            // Rough heuristic: count function calls in agent code
+            const count = (code.match(new RegExp(`\\b${toolName}\\s*\\(`, 'g')) ?? []).length;
+            if (count > 0) {
+                toolCallsByName[toolName] = (toolCallsByName[toolName] ?? 0) + count;
+            }
+        }
+        // Check for final_answer sentinel
+        let finalAnswer = null;
+        if (fs.existsSync(resultFile)) {
+            try {
+                const sentinel = JSON.parse(fs.readFileSync(resultFile, 'utf-8'));
+                finalAnswer = String(sentinel.answer ?? '').trim();
+            }
+            catch { /* ignore */ }
+        }
+        const stdoutTrimmed = stdout.trim();
+        const stderr = (result.stderr ?? '').trim();
+        if (result.error?.code === 'ETIMEDOUT') {
+            return { output: `[Execution timed out after ${stepTimeoutMs / 1000}s]`, finalAnswer: null };
+        }
+        // Filter out non-critical warnings from stderr
+        const stderrClean = stderr
+            ? stderr.split('\n').filter(l => !l.includes('UserWarning') && !l.includes('DeprecationWarning') && l.trim()).slice(0, 5).join('\n')
+            : '';
+        let observation = stdoutTrimmed;
+        if (stderrClean) {
+            observation = observation
+                ? `${observation}\n[stderr]: ${stderrClean.slice(0, 500)}`
+                : `[stderr]: ${stderrClean.slice(0, 500)}`;
+        }
+        if (!observation && finalAnswer !== null) {
+            observation = `[Code executed — final_answer captured: ${finalAnswer}]`;
+        }
+        if (!observation) {
+            observation = '[Code executed — no output]';
+        }
+        if (observation.length > MAX_OBSERVATION_CHARS) {
+            observation = observation.slice(0, MAX_OBSERVATION_CHARS) + '\n[... truncated ...]';
+        }
+        return { output: observation, finalAnswer };
+    }
+    catch (err) {
+        return { output: `[Execution error: ${String(err)}]`, finalAnswer: null };
+    }
+    finally {
+        try {
+            fs.rmSync(tmpDir, { recursive: true, force: true });
+        }
+        catch { /* ignore */ }
+    }
+}
+// ---------------------------------------------------------------------------
+// Public step runner (exported for unit tests)
+// ---------------------------------------------------------------------------
+/**
+ * Execute a single Python code step via the gaia-codeagent-runner.py subprocess.
+ *
+ * This is a thin public wrapper around `executeAgentCodeStep` that:
+ * - Exposes a clean typed signature for unit tests (gaia-codeagent.smoke.ts T2-T4)
+ * - Renames `output` → `observation` to match the smolagents naming convention
+ *
+ * @param code       Python code to execute
+ * @param attachmentPath  Path to attached file, or null
+ * @param timeoutMs  Subprocess timeout in ms (default: 30s)
+ * @param apiKey     Anthropic API key (passed to runner for claude -p tool calls)
+ */
+export function runCodeAgentStep(code, attachmentPath, timeoutMs = DEFAULT_PER_STEP_TIMEOUT_MS, apiKey = '') {
+    const toolCallsByName = {};
+    const { output, finalAnswer } = executeAgentCodeStep(code, attachmentPath, timeoutMs, apiKey, toolCallsByName);
+    return { observation: output, finalAnswer };
+}
+// ---------------------------------------------------------------------------
+// Anthropic Messages API call (text-only — no tools array)
+// ---------------------------------------------------------------------------
+async function callAnthropicTextOnly(apiKey, model, systemPrompt, messages, maxTokens, timeoutMs) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    let res;
+    try {
+        res = await fetch(ANTHROPIC_API_URL, {
+            method: 'POST',
+            headers: {
+                'x-api-key': apiKey,
+                'anthropic-version': ANTHROPIC_API_VERSION,
+                'content-type': 'application/json',
+            },
+            body: JSON.stringify({
+                model,
+                max_tokens: maxTokens,
+                system: systemPrompt,
+                messages,
+            }),
+            signal: controller.signal,
+        });
+    }
+    finally {
+        clearTimeout(timer);
+    }
+    if (!res.ok) {
+        const errText = await res.text().catch(() => '<unreadable>');
+        throw new Error(`Anthropic API error ${res.status}: ${errText.slice(0, 400)}`);
+    }
+    return (await res.json());
+}
+// ---------------------------------------------------------------------------
+// Main CodeAgent loop
+// ---------------------------------------------------------------------------
+/**
+ * Run a GAIA question through the smolagents-style CodeAgent harness.
+ *
+ * The agent writes Python code, we execute it with tool stubs, and feed
+ * the output back.  The loop continues until final_answer() is called
+ * or maxTurns is exhausted.
+ */
+export async function runGaiaCodeAgent(question, options = {}) {
+    const { model = DEFAULT_MODEL, maxTurns = DEFAULT_MAX_TURNS, maxTokensPerTurn = DEFAULT_MAX_TOKENS_PER_TURN, perTurnTimeoutMs = DEFAULT_PER_TURN_TIMEOUT_MS, perStepTimeoutMs = DEFAULT_PER_STEP_TIMEOUT_MS, planningInterval = DEFAULT_PLANNING_INTERVAL, apiKey: suppliedKey, verbose = false, } = options;
+    const wallStart = Date.now();
+    const apiKey = resolveAnthropicApiKey(suppliedKey);
+    const attachmentPath = question.file_path ?? null;
+    const toolCallsByName = {};
+    let totalInputTokens = 0;
+    let totalOutputTokens = 0;
+    let replanCount = 0;
+    let noCodeRetries = 0;
+    const steps = [];
+    const systemPrompt = buildCodeAgentSystemPrompt();
+    // Build the initial user message following the GAIA instruction template
+    // (exact format from HAL-DEEP-STUDY.md §4)
+    const questionText = buildGaiaQuestionMessage(question);
+    const messages = [
+        { role: 'user', content: questionText },
+    ];
+    let turns = 0;
+    for (let turn = 0; turn < maxTurns; turn++) {
+        turns = turn + 1;
+        // Call the model (text only — no tools array)
+        let resp;
+        try {
+            resp = await callAnthropicTextOnly(apiKey, model, systemPrompt, messages, maxTokensPerTurn, perTurnTimeoutMs);
+        }
+        catch (err) {
+            return {
+                questionId: question.task_id,
+                finalAnswer: null,
+                turns,
+                toolCallsByName,
+                totalInputTokens,
+                totalOutputTokens,
+                wallMs: Date.now() - wallStart,
+                replanCount,
+                error: err instanceof Error ? err.message : String(err),
+                steps: verbose ? steps : undefined,
+            };
+        }
+        totalInputTokens += resp.usage.input_tokens;
+        totalOutputTokens += resp.usage.output_tokens;
+        // Extract text content
+        const assistantText = resp.content
+            .filter((b) => b.type === 'text')
+            .map((b) => b.text)
+            .join('\n')
+            .trim();
+        // Append assistant turn to messages
+        messages.push({ role: 'assistant', content: assistantText });
+        // --- Check for final_answer in TEXT output (before code execution) ---
+        // Sometimes the agent commits in prose without a code block
+        const textFinalMatch = FINAL_ANSWER_TEXT_RE.exec(assistantText);
+        if (textFinalMatch) {
+            const finalAnswer = textFinalMatch[1].trim();
+            return {
+                questionId: question.task_id,
+                finalAnswer,
+                turns,
+                toolCallsByName,
+                totalInputTokens,
+                totalOutputTokens,
+                wallMs: Date.now() - wallStart,
+                replanCount,
+                steps: verbose ? steps : undefined,
+            };
+        }
+        // --- Extract code block ---
+        const codeBlock = extractCodeBlock(assistantText);
+        if (!codeBlock) {
+            noCodeRetries++;
+            if (noCodeRetries >= MAX_NO_CODE_RETRIES) {
+                // Give up on prompting — try to extract answer from prose
+                const textMatch = FINAL_ANSWER_TEXT_RE.exec(assistantText);
+                if (textMatch) {
+                    return {
+                        questionId: question.task_id,
+                        finalAnswer: textMatch[1].trim(),
+                        turns,
+                        toolCallsByName,
+                        totalInputTokens,
+                        totalOutputTokens,
+                        wallMs: Date.now() - wallStart,
+                        replanCount,
+                        steps: verbose ? steps : undefined,
+                    };
+                }
+                break;
+            }
+            // Prompt agent to produce a code block
+            messages.push({
+                role: 'user',
+                content: 'Please write a Python code block. Use ```python\\n...\\n``` format. Call final_answer("your answer") when done.',
+            });
+            continue;
+        }
+        noCodeRetries = 0;
+        // --- Execute the code via the Python runner ---
+        const { output, finalAnswer: execFinalAnswer } = executeAgentCodeStep(codeBlock, attachmentPath, perStepTimeoutMs, apiKey, toolCallsByName);
+        if (verbose) {
+            steps.push({ code: codeBlock, output });
+        }
+        // If execution produced a final_answer, return it
+        if (execFinalAnswer !== null) {
+            return {
+                questionId: question.task_id,
+                finalAnswer: execFinalAnswer,
+                turns,
+                toolCallsByName,
+                totalInputTokens,
+                totalOutputTokens,
+                wallMs: Date.now() - wallStart,
+                replanCount,
+                steps: verbose ? steps : undefined,
+            };
+        }
+        // --- Build next user turn (observation) ---
+        // Every planningInterval turns, inject a planning checkpoint
+        const shouldReplan = planningInterval > 0 && turns > 0 && turns % planningInterval === 0;
+        let observationText = `\`\`\`output\n${output}\n\`\`\``;
+        if (shouldReplan) {
+            replanCount++;
+            observationText += `\n\n${buildCodeAgentPlanningCheckpoint(turns, maxTurns)}`;
+        }
+        messages.push({ role: 'user', content: observationText });
+    }
+    // Exhausted maxTurns
+    return {
+        questionId: question.task_id,
+        finalAnswer: null,
+        turns,
+        toolCallsByName,
+        totalInputTokens,
+        totalOutputTokens,
+        wallMs: Date.now() - wallStart,
+        replanCount,
+        timedOut: true,
+        steps: verbose ? steps : undefined,
+    };
+}
+// ---------------------------------------------------------------------------
+// GAIA question message builder
+// ---------------------------------------------------------------------------
+/**
+ * Build the user message for a GAIA question.
+ * Follows the HAL instruction template exactly (HAL-DEEP-STUDY.md §4).
+ */
+function buildGaiaQuestionMessage(question) {
+    const gaiaPreamble = [
+        'Please answer the question below. You should:',
+        '- Return only your answer, which should be a number, or a short phrase with as few words as possible, or a comma separated list.',
+        '- If you are asked for a number, don\'t use commas to write your number neither use units such as $ or percent sign unless specified otherwise. Don\'t round numbers unless specified.',
+        '- If you are asked for a string, don\'t use articles, neither abbreviations (e.g. for cities), and write the digits in full unless specified otherwise.',
+        '- If you are asked for a comma separated list, apply the above rules depending on whether the element to be put in the list is a number or a string.',
+        '',
+        'To submit your answer, use: final_answer("your answer here")',
+        '',
+    ].join('\n');
+    let questionText = question.question;
+    // Attachment hint
+    if (question.file_path) {
+        const ext = question.file_path.split('.').pop()?.toLowerCase() ?? '';
+        const toolHint = ext === 'pdf'
+            ? `Use pdf_read(path="${question.file_path}") to read the attached PDF.`
+            : ext === 'xlsx' || ext === 'csv'
+                ? `Use file_read(path="${question.file_path}") to read the attached spreadsheet.`
+                : `Use file_read(path="${question.file_path}") to read the attached file.`;
+        questionText += `\n\n[ATTACHMENT: ${question.file_name ?? question.file_path}]\n${toolHint}`;
+    }
+    return gaiaPreamble + questionText;
+}
+// ---------------------------------------------------------------------------
+// Smoke runner
+// ---------------------------------------------------------------------------
+const SONNET_INPUT_COST_PER_M = 3.0;
+const SONNET_OUTPUT_COST_PER_M = 15.0;
+/**
+ * Run all 5 SMOKE_FIXTURE questions through the CodeAgent harness.
+ * Pass criteria: ≥3/5 correct (60%).
+ */
+export async function runCodeAgentSmokeTest(opts = {}) {
+    const { verbose = true, apiKey, model = DEFAULT_MODEL } = opts;
+    if (verbose) {
+        console.log('\n=== GAIA CodeAgent Smoke Test (ADR-138 iter 54) ===');
+        console.log(`Model: ${model}`);
+        console.log(`Questions: ${SMOKE_FIXTURE.length}\n`);
+    }
+    let passed = 0;
+    let totalInputTokens = 0;
+    let totalOutputTokens = 0;
+    for (const question of SMOKE_FIXTURE) {
+        const result = await runGaiaCodeAgent(question, { model, apiKey, verbose });
+        const correct = result.finalAnswer !== null &&
+            isAnswerCorrect(result.finalAnswer, question.final_answer);
+        if (correct)
+            passed++;
+        totalInputTokens += result.totalInputTokens;
+        totalOutputTokens += result.totalOutputTokens;
+        if (verbose) {
+            const status = correct ? 'PASS' : 'FAIL';
+            console.log(`[${status}] ${question.task_id}: ${question.question.slice(0, 60)}`);
+            console.log(`       Expected: "${question.final_answer}" | Got: "${result.finalAnswer ?? 'null'}"`);
+            console.log(`       Turns: ${result.turns} | Replans: ${result.replanCount} ` +
+                `| Tools: ${JSON.stringify(result.toolCallsByName)} | Wall: ${result.wallMs}ms`);
+            if (result.error)
+                console.log(`       Error: ${result.error}`);
+            console.log();
+        }
+    }
+    const passRate = passed / SMOKE_FIXTURE.length;
+    const estCost = (totalInputTokens / 1_000_000) * SONNET_INPUT_COST_PER_M +
+        (totalOutputTokens / 1_000_000) * SONNET_OUTPUT_COST_PER_M;
+    if (verbose) {
+        console.log('=== Summary ===');
+        console.log(`Pass rate: ${passed}/${SMOKE_FIXTURE.length} (${(passRate * 100).toFixed(0)}%)`);
+        console.log(`Status:    ${passed >= 3 ? 'SMOKE PASSED' : 'SMOKE FAILED'}`);
+        console.log(`Tokens in: ${totalInputTokens.toLocaleString()}`);
+        console.log(`Tokens out: ${totalOutputTokens.toLocaleString()}`);
+        console.log(`Est. cost: $${estCost.toFixed(4)} (Sonnet pricing)`);
+        console.log();
+    }
+    return { passRate, passed, total: SMOKE_FIXTURE.length };
+}
+// ---------------------------------------------------------------------------
+// CLI entrypoint
+// ---------------------------------------------------------------------------
+if (process.argv.includes('--smoke')) {
+    runCodeAgentSmokeTest({ verbose: true })
+        .then(({ passed }) => {
+        process.exit(passed >= 3 ? 0 : 1);
+    })
+        .catch((err) => {
+        console.error('CodeAgent smoke test crashed:', err);
+        process.exit(2);
+    });
+}
+//# sourceMappingURL=gaia-codeagent.js.map

package/dist/src/benchmarks/gaia-codeagent.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gaia-codeagent.js","sourceRoot":"","sources":["../../../src/benchmarks/gaia-codeagent.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EAEL,aAAa,GACd,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,sBAAsB,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAE1E,8EAA8E;AAC9E,wBAAwB;AACxB,8EAA8E;AAE9E,OAAO,EAAE,eAAe,EAAE,CAAC;AAE3B,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,MAAM,iBAAiB,GAAG,uCAAuC,CAAC;AAClE,MAAM,qBAAqB,GAAG,YAAY,CAAC;AAE3C,iFAAiF;AACjF,MAAM,aAAa,GAAG,mBAAmB,CAAC;AAC1C,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAC7B,MAAM,2BAA2B,GAAG,IAAI,CAAC;AACzC,MAAM,2BAA2B,GAAG,MAAM,CAAC;AAC3C,MAAM,yBAAyB,GAAG,CAAC,CAAC;AACpC,MAAM,2BAA2B,GAAG,MAAM,CAAC;AAC3C,MAAM,qBAAqB,GAAG,KAAK,CAAC;AACpC,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAE9B,uEAAuE;AACvE,MAAM,oBAAoB,GAAG,uBAAuB,CAAC;AAErD,oEAAoE;AACpE,SAAS,aAAa;IACpB,MAAM,WAAW,GAAG,0BAA0B,CAAC;IAE/C,4EAA4E;IAC5E,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,CAAC;QACjE,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,SAAS,CAAC;IACjD,CAAC;IAAC,MAAM,CAAC,CAAC,+BAA+B,CAAC,CAAC;IAE3C,qFAAqF;IACrF,mFAAmF;IACnF,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChD,yDAAyD;QACzD,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC9C,MAAM,aAAa,GAAG,cAAc,CAAC,OAAO,CAC1C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,EACtC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,CAAC,CAC/B,CAAC;QACF,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;QAC3D,IAAI,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC;YAAE,OAAO,YAAY,CAAC;IACvD,CAAC;IAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IAExB,gDAAgD;IAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;IAC9E,OAAO,QAAQ,CAAC;AAClB,CAAC;AAwDD,8EAA8E;AAC9E,gBAAgB;AAChB,8EAA8E;AAE9E;;;;;;GAMG;AACH,SAAS,0BAA0B;IACjC,MAAM,QAAQ,GAAG;QACf,iCAAiC;QACjC,kEAAkE;QAClE,EAAE;QACF,kCAAkC;QAClC,gFAAgF;QAChF,EAAE;QACF,qCAAqC;QACrC,mFAAmF;QACnF,mFAAmF;QACnF,EAAE;QACF,+BAA+B;QAC/B,qEAAqE;QACrE,gFAAgF;QAChF,EAAE;QACF,oCAAoC;QACpC,4CAA4C;QAC5C,EAAE;QACF,iCAAiC;QACjC,+DAA+D;KAChE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEb,OAAO;QACL,kEAAkE;QAClE,uFAAuF;QACvF,yFAAyF;QACzF,EAAE;QACF,4DAA4D;QAC5D,QAAQ;QACR,EAAE;QACF,mBAAmB;QACnB,mFAAmF;QACnF,EAAE;QACF,6EAA6E;QAC7E,wEAAwE;QACxE,qEAAqE;QACrE,EAAE;QACF,eAAe;QACf,uDAAuD;QACvD,8DAA8D;QAC9D,8EAA8E;QAC9E,mEAAmE;QACnE,gEAAgE;QAChE,EAAE;QACF,iDAAiD;QACjD,0CAA0C;QAC1C,8EAA8E;QAC9E,0FAA0F;QAC1F,wDAAwD;QACxD,EAAE;QACF,sFAAsF;QACtF,gFAAgF;KACjF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E,SAAS,gCAAgC,CAAC,IAAY,EAAE,QAAgB;IACtE,OAAO,CACL,+BAA+B,IAAI,IAAI,QAAQ,KAAK;QACpD,iBAAiB,IAAI,qCAAqC;QAC1D,sEAAsE;QACtE,oFAAoF;QACpF,oGAAoG;QACpG,+EAA+E;QAC/E,4CAA4C,CAC7C,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,kFAAkF;IAClF,MAAM,aAAa,GAAG,oCAAoC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtE,IAAI,aAAa;QAAE,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAElD,gCAAgC;IAChC,MAAM,YAAY,GAAG,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxD,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAEhD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,2DAA2D;AAC3D,8EAA8E;AAE9E;;;;;;;;;;;GAWG;AACH,SAAS,oBAAoB,CAC3B,IAAY,EACZ,cAA6B,EAC7B,aAAqB,EACrB,MAAc,EACd,eAAuC;IAEvC,MAAM,UAAU,GAAG,aAAa,EAAE,CAAC;IACnC,MAAM,MAAM,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,UAAU,CAAC,CAAC,CAAC;IAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IACpD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC;IAE1D,IAAI,CAAC;QACH,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QAE1C,MAAM,GAAG,GAAsB;YAC7B,GAAG,OAAO,CAAC,GAAG;YACd,gBAAgB,EAAE,OAAO;YACzB,cAAc,EAAE,QAAQ;YACxB,gBAAgB,EAAE,UAAU;YAC5B,iBAAiB,EAAE,MAAM;SAC1B,CAAC;QACF,IAAI,cAAc,EAAE,CAAC;YACnB,GAAG,CAAC,oBAAoB,GAAG,cAAc,CAAC;QAC5C,CAAC;QAED,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,EAAE;YAChD,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,aAAa;YACtB,GAAG;YACH,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;SAC5B,CAAC,CAAC;QAEH,sEAAsE;QACtE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;QACnC,MAAM,YAAY,GAAG,CAAC,YAAY,EAAE,eAAe,EAAE,gBAAgB,EAAE,WAAW,EAAE,gBAAgB,CAAC,CAAC;QACtG,KAAK,MAAM,QAAQ,IAAI,YAAY,EAAE,CAAC;YACpC,sDAAsD;YACtD,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,QAAQ,SAAS,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;YAClF,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACd,eAAe,CAAC,QAAQ,CAAC,GAAG,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC;YACvE,CAAC;QACH,CAAC;QAED,kCAAkC;QAClC,IAAI,WAAW,GAAkB,IAAI,CAAC;QACtC,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;gBAClE,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACrD,CAAC;YAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;QAC1B,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAE5C,IAAK,MAAM,CAAC,KAA2C,EAAE,IAAI,KAAK,WAAW,EAAE,CAAC;YAC9E,OAAO,EAAE,MAAM,EAAE,8BAA8B,aAAa,GAAG,IAAI,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;QAC/F,CAAC;QAED,+CAA+C;QAC/C,MAAM,WAAW,GAAG,MAAM;YACxB,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAC5B,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAC5E,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YAC1B,CAAC,CAAC,EAAE,CAAC;QAEP,IAAI,WAAW,GAAG,aAAa,CAAC;QAChC,IAAI,WAAW,EAAE,CAAC;YAChB,WAAW,GAAG,WAAW;gBACvB,CAAC,CAAC,GAAG,WAAW,eAAe,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;gBAC1D,CAAC,CAAC,aAAa,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QAC/C,CAAC;QACD,IAAI,CAAC,WAAW,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;YACzC,WAAW,GAAG,2CAA2C,WAAW,GAAG,CAAC;QAC1E,CAAC;QACD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,WAAW,GAAG,6BAA6B,CAAC;QAC9C,CAAC;QAED,IAAI,WAAW,CAAC,MAAM,GAAG,qBAAqB,EAAE,CAAC;YAC/C,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,qBAAqB,CAAC,GAAG,uBAAuB,CAAC;QACtF,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;IAC9C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,EAAE,MAAM,EAAE,qBAAqB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAC5E,CAAC;YAAS,CAAC;QACT,IAAI,CAAC;YAAC,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IACrF,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,+CAA+C;AAC/C,8EAA8E;AAE9E;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,gBAAgB,CAC9B,IAAY,EACZ,cAA6B,EAC7B,SAAS,GAAG,2BAA2B,EACvC,MAAM,GAAG,EAAE;IAEX,MAAM,eAAe,GAA2B,EAAE,CAAC;IACnD,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,oBAAoB,CAAC,IAAI,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,EAAE,eAAe,CAAC,CAAC;IAC/G,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;AAC9C,CAAC;AAED,8EAA8E;AAC9E,2DAA2D;AAC3D,8EAA8E;AAE9E,KAAK,UAAU,qBAAqB,CAClC,MAAc,EACd,KAAa,EACb,YAAoB,EACpB,QAAwB,EACxB,SAAiB,EACjB,SAAiB;IAEjB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,IAAI,GAAa,CAAC;IAClB,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,KAAK,CAAC,iBAAiB,EAAE;YACnC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,WAAW,EAAE,MAAM;gBACnB,mBAAmB,EAAE,qBAAqB;gBAC1C,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK;gBACL,UAAU,EAAE,SAAS;gBACrB,MAAM,EAAE,YAAY;gBACpB,QAAQ;aACT,CAAC;YACF,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,cAAc,CAAC,CAAC;QAC7D,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,CAAC,MAAM,KAAK,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACjF,CAAC;IAED,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA0B,CAAC;AACrD,CAAC;AAED,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,QAAsB,EACtB,UAA4B,EAAE;IAE9B,MAAM,EACJ,KAAK,GAAG,aAAa,EACrB,QAAQ,GAAG,iBAAiB,EAC5B,gBAAgB,GAAG,2BAA2B,EAC9C,gBAAgB,GAAG,2BAA2B,EAC9C,gBAAgB,GAAG,2BAA2B,EAC9C,gBAAgB,GAAG,yBAAyB,EAC5C,MAAM,EAAE,WAAW,EACnB,OAAO,GAAG,KAAK,GAChB,GAAG,OAAO,CAAC;IAEZ,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAG,sBAAsB,CAAC,WAAW,CAAC,CAAC;IACnD,MAAM,cAAc,GAAG,QAAQ,CAAC,SAAS,IAAI,IAAI,CAAC;IAElD,MAAM,eAAe,GAA2B,EAAE,CAAC;IACnD,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,MAAM,KAAK,GAA4C,EAAE,CAAC;IAE1D,MAAM,YAAY,GAAG,0BAA0B,EAAE,CAAC;IAElD,yEAAyE;IACzE,2CAA2C;IAC3C,MAAM,YAAY,GAAG,wBAAwB,CAAC,QAAQ,CAAC,CAAC;IACxD,MAAM,QAAQ,GAAmB;QAC/B,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE;KACxC,CAAC;IAEF,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC;QAC3C,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC;QAEjB,8CAA8C;QAC9C,IAAI,IAA2B,CAAC;QAChC,IAAI,CAAC;YACH,IAAI,GAAG,MAAM,qBAAqB,CAChC,MAAM,EACN,KAAK,EACL,YAAY,EACZ,QAAQ,EACR,gBAAgB,EAChB,gBAAgB,CACjB,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,UAAU,EAAE,QAAQ,CAAC,OAAO;gBAC5B,WAAW,EAAE,IAAI;gBACjB,KAAK;gBACL,eAAe;gBACf,gBAAgB;gBAChB,iBAAiB;gBACjB,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAC9B,WAAW;gBACX,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;gBACvD,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;aACnC,CAAC;QACJ,CAAC;QAED,gBAAgB,IAAI,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC;QAC5C,iBAAiB,IAAI,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC;QAE9C,uBAAuB;QACvB,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO;aAC/B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;aAChC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;aAClB,IAAI,CAAC,IAAI,CAAC;aACV,IAAI,EAAE,CAAC;QAEV,oCAAoC;QACpC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,CAAC;QAE7D,wEAAwE;QACxE,4DAA4D;QAC5D,MAAM,cAAc,GAAG,oBAAoB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAChE,IAAI,cAAc,EAAE,CAAC;YACnB,MAAM,WAAW,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7C,OAAO;gBACL,UAAU,EAAE,QAAQ,CAAC,OAAO;gBAC5B,WAAW;gBACX,KAAK;gBACL,eAAe;gBACf,gBAAgB;gBAChB,iBAAiB;gBACjB,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAC9B,WAAW;gBACX,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;aACnC,CAAC;QACJ,CAAC;QAED,6BAA6B;QAC7B,MAAM,SAAS,GAAG,gBAAgB,CAAC,aAAa,CAAC,CAAC;QAElD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,aAAa,EAAE,CAAC;YAChB,IAAI,aAAa,IAAI,mBAAmB,EAAE,CAAC;gBACzC,0DAA0D;gBAC1D,MAAM,SAAS,GAAG,oBAAoB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAC3D,IAAI,SAAS,EAAE,CAAC;oBACd,OAAO;wBACL,UAAU,EAAE,QAAQ,CAAC,OAAO;wBAC5B,WAAW,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;wBAChC,KAAK;wBACL,eAAe;wBACf,gBAAgB;wBAChB,iBAAiB;wBACjB,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;wBAC9B,WAAW;wBACX,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;qBACnC,CAAC;gBACJ,CAAC;gBACD,MAAM;YACR,CAAC;YACD,uCAAuC;YACvC,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,iHAAiH;aAC3H,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,aAAa,GAAG,CAAC,CAAC;QAElB,iDAAiD;QACjD,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,eAAe,EAAE,GAAG,oBAAoB,CACnE,SAAS,EACT,cAAc,EACd,gBAAgB,EAChB,MAAM,EACN,eAAe,CAChB,CAAC;QAEF,IAAI,OAAO,EAAE,CAAC;YACZ,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QAC1C,CAAC;QAED,kDAAkD;QAClD,IAAI,eAAe,KAAK,IAAI,EAAE,CAAC;YAC7B,OAAO;gBACL,UAAU,EAAE,QAAQ,CAAC,OAAO;gBAC5B,WAAW,EAAE,eAAe;gBAC5B,KAAK;gBACL,eAAe;gBACf,gBAAgB;gBAChB,iBAAiB;gBACjB,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAC9B,WAAW;gBACX,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;aACnC,CAAC;QACJ,CAAC;QAED,6CAA6C;QAC7C,6DAA6D;QAC7D,MAAM,YAAY,GAChB,gBAAgB,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,gBAAgB,KAAK,CAAC,CAAC;QAEtE,IAAI,eAAe,GAAG,iBAAiB,MAAM,UAAU,CAAC;QACxD,IAAI,YAAY,EAAE,CAAC;YACjB,WAAW,EAAE,CAAC;YACd,eAAe,IAAI,OAAO,gCAAgC,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,CAAC;QAChF,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,qBAAqB;IACrB,OAAO;QACL,UAAU,EAAE,QAAQ,CAAC,OAAO;QAC5B,WAAW,EAAE,IAAI;QACjB,KAAK;QACL,eAAe;QACf,gBAAgB;QAChB,iBAAiB;QACjB,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAC9B,WAAW;QACX,QAAQ,EAAE,IAAI;QACd,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;KACnC,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,gCAAgC;AAChC,8EAA8E;AAE9E;;;GAGG;AACH,SAAS,wBAAwB,CAAC,QAAsB;IACtD,MAAM,YAAY,GAAG;QACnB,+CAA+C;QAC/C,kIAAkI;QAClI,wLAAwL;QACxL,yJAAyJ;QACzJ,sJAAsJ;QACtJ,EAAE;QACF,8DAA8D;QAC9D,EAAE;KACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEb,IAAI,YAAY,GAAG,QAAQ,CAAC,QAAQ,CAAC;IAErC,kBAAkB;IAClB,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;QACrE,MAAM,QAAQ,GAAG,GAAG,KAAK,KAAK;YAC5B,CAAC,CAAC,sBAAsB,QAAQ,CAAC,SAAS,8BAA8B;YACxE,CAAC,CAAC,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,KAAK;gBACjC,CAAC,CAAC,uBAAuB,QAAQ,CAAC,SAAS,sCAAsC;gBACjF,CAAC,CAAC,uBAAuB,QAAQ,CAAC,SAAS,+BAA+B,CAAC;QAE7E,YAAY,IAAI,oBAAoB,QAAQ,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS,MAAM,QAAQ,EAAE,CAAC;IAC/F,CAAC;IAED,OAAO,YAAY,GAAG,YAAY,CAAC;AACrC,CAAC;AAED,8EAA8E;AAC9E,eAAe;AACf,8EAA8E;AAE9E,MAAM,uBAAuB,GAAG,GAAG,CAAC;AACpC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AAEtC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,OAIxC,EAAE;IACJ,MAAM,EAAE,OAAO,GAAG,IAAI,EAAE,MAAM,EAAE,KAAK,GAAG,aAAa,EAAE,GAAG,IAAI,CAAC;IAE/D,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,CAAC,GAAG,CAAC,uDAAuD,CAAC,CAAC;QACrE,OAAO,CAAC,GAAG,CAAC,UAAU,KAAK,EAAE,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,cAAc,aAAa,CAAC,MAAM,IAAI,CAAC,CAAC;IACtD,CAAC;IAED,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAE1B,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QAE5E,MAAM,OAAO,GACX,MAAM,CAAC,WAAW,KAAK,IAAI;YAC3B,eAAe,CAAC,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,YAAY,CAAC,CAAC;QAE7D,IAAI,OAAO;YAAE,MAAM,EAAE,CAAC;QACtB,gBAAgB,IAAI,MAAM,CAAC,gBAAgB,CAAC;QAC5C,iBAAiB,IAAI,MAAM,CAAC,iBAAiB,CAAC;QAE9C,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;YACzC,OAAO,CAAC,GAAG,CAAC,IAAI,MAAM,KAAK,QAAQ,CAAC,OAAO,KAAK,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;YAClF,OAAO,CAAC,GAAG,CACT,qBAAqB,QAAQ,CAAC,YAAY,aAAa,MAAM,CAAC,WAAW,IAAI,MAAM,GAAG,CACvF,CAAC;YACF,OAAO,CAAC,GAAG,CACT,iBAAiB,MAAM,CAAC,KAAK,eAAe,MAAM,CAAC,WAAW,GAAG;gBACjE,YAAY,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,YAAY,MAAM,CAAC,MAAM,IAAI,CAChF,CAAC;YACF,IAAI,MAAM,CAAC,KAAK;gBAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;YAC/D,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;IAC/C,MAAM,OAAO,GACX,CAAC,gBAAgB,GAAG,SAAS,CAAC,GAAG,uBAAuB;QACxD,CAAC,iBAAiB,GAAG,SAAS,CAAC,GAAG,wBAAwB,CAAC;IAE7D,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC9F,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC;QAC3E,OAAO,CAAC,GAAG,CAAC,cAAc,gBAAgB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAC/D,OAAO,CAAC,GAAG,CAAC,eAAe,iBAAiB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QACjE,OAAO,CAAC,GAAG,CAAC,eAAe,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC;QAClE,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,CAAC,MAAM,EAAE,CAAC;AAC3D,CAAC;AAED,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;IACrC,qBAAqB,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;SACrC,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;QACnB,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACb,OAAO,CAAC,KAAK,CAAC,+BAA+B,EAAE,GAAG,CAAC,CAAC;QACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/src/benchmarks/gaia-codeagent.smoke.d.ts

@@ -0,0 +1,33 @@
+/**
+ * GAIA CodeAgent Smoke Tests — ADR-138 iter 54 (FINAL design)
+ *
+ * 5 tests validating the CodeAgent harness end-to-end.
+ * Tests 1-4 are unit-level (no API calls). Test 5 uses 1 API call.
+ *
+ * Tests:
+ *   T1: Code block extraction — parser correctness (no subprocess)
+ *   T2: Python runner — simple math (subprocess, no API)
+ *   T3: Python runner — file read via ATTACHMENT_PATH
+ *   T4: Python runner — code error recovery (traceback returned as obs)
+ *   T5: End-to-end — simple factual question (1 API call, ~$0.01)
+ *
+ * Pass criteria: 5/5. Gate before iter 55 5Q run.
+ *
+ * Run: npx tsx gaia-codeagent.smoke.ts
+ *      or: node dist/src/benchmarks/gaia-codeagent.smoke.js
+ *
+ * Refs: ADR-138, #2156, iter 54
+ */
+interface TestResult {
+    name: string;
+    passed: boolean;
+    message: string;
+    durationMs: number;
+}
+declare function testCodeBlockExtraction(): TestResult;
+declare function testPythonRunnerMath(): TestResult;
+declare function testPythonRunnerFileRead(): TestResult;
+declare function testPythonRunnerErrorRecovery(): TestResult;
+declare function testEndToEndSimple(): Promise<TestResult>;
+export { testCodeBlockExtraction, testPythonRunnerMath, testPythonRunnerFileRead, testPythonRunnerErrorRecovery, testEndToEndSimple, };
+//# sourceMappingURL=gaia-codeagent.smoke.d.ts.map

package/dist/src/benchmarks/gaia-codeagent.smoke.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gaia-codeagent.smoke.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/gaia-codeagent.smoke.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAsBH,UAAU,UAAU;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB;AAkBD,iBAAS,uBAAuB,IAAI,UAAU,CA2B7C;AAMD,iBAAS,oBAAoB,IAAI,UAAU,CAiB1C;AAMD,iBAAS,wBAAwB,IAAI,UAAU,CA6B9C;AAMD,iBAAS,6BAA6B,IAAI,UAAU,CA4BnD;AAMD,iBAAe,kBAAkB,IAAI,OAAO,CAAC,UAAU,CAAC,CAmDvD;AAwCD,OAAO,EACL,uBAAuB,EACvB,oBAAoB,EACpB,wBAAwB,EACxB,6BAA6B,EAC7B,kBAAkB,GACnB,CAAC"}