@claude-flow/cli 3.11.0 → 3.12.0

package/dist/src/benchmarks/gaia-claude-p.js

@@ -1,269 +0,0 @@
-/**
- * GAIA Claude-p Wrapper — iter 54 (#2156)
- *
- * Delegates each GAIA question to `claude -p` (Claude Code headless mode),
- * which gives us WebSearch, WebFetch, Read (multimodal incl. PDF/DOCX/images),
- * and Bash (Python execution) for free — the same tools HAL uses.
- *
- * Why this approach over a native TS CodeAgent:
- *   - HAL gaps vs ruflo were: visit_webpage, file reading (PDF/DOCX/XLSX/images),
- *     Python execution.  Claude Code's built-in tools solve ALL of these.
- *   - No wheel-reinvention: battle-tested tool infra, native multimodal, proper
- *     tool-budget management, Anthropic WebSearch API.
- *   - Baseline: 24/53 (45.3%).  Target: ≥45/53 to surpass HAL's 82.07%.
- *
- * SECURITY NOTE on --dangerously-skip-permissions:
- *   This flag is ONLY used inside the GAIA harness context, which is a sandboxed
- *   benchmark evaluation environment.  GAIA questions have no real-world
- *   side effects — they are read-only research questions.  The flag lets Claude Code
- *   use its tools (WebSearch, WebFetch, Read, Bash) without per-tool permission
- *   prompts, which is required for unattended benchmark execution.  It MUST NOT
- *   be used in production workflows where Claude Code could affect real systems.
- *
- * JSON output format from `claude -p --output-format json`:
- *   {
- *     type: "result",
- *     subtype: "success" | "error_max_budget_usd" | ...,
- *     is_error: boolean,
- *     result: string,          // final assistant message text
- *     total_cost_usd: number,
- *     duration_ms: number,
- *     num_turns: number,
- *     ...
- *   }
- *
- * Refs: ADR-138 (reference, NOT implemented), iter 54, #2156
- */
-import { spawn } from 'node:child_process';
-import * as path from 'node:path';
-// ---------------------------------------------------------------------------
-// Constants
-// ---------------------------------------------------------------------------
-/** Default model for claude -p GAIA runs. Sonnet for quality parity with HAL. */
-export const CLAUDE_P_DEFAULT_MODEL = 'claude-sonnet-4-6';
-/** Per-question budget cap (USD). HAL uses Sonnet 4.5 so $0.30 headroom is safe. */
-export const CLAUDE_P_PER_QUESTION_BUDGET_USD = 0.30;
-/** Subprocess timeout: 5 minutes per question. */
-export const CLAUDE_P_TIMEOUT_MS = 5 * 60 * 1000;
-/** FINAL_ANSWER extraction pattern — same as gaia-agent.ts. */
-const FINAL_ANSWER_RE = /FINAL_ANSWER:\s*(.+?)(?:\n|$)/i;
-// ---------------------------------------------------------------------------
-// Prompt builder
-// ---------------------------------------------------------------------------
-/**
- * Build the prompt sent to claude -p for a GAIA question.
- *
- * Includes the question text, optional attachment path, and precise instructions
- * for using available tools and producing FINAL_ANSWER: in the expected format.
- */
-export function buildClaudePPrompt(question) {
-    const attachmentBlock = question.file_path
-        ? [
-            '',
-            `Attachment file path: ${path.resolve(question.file_path)}`,
-            'You can use the Read tool to view this file directly.',
-            'For images, audio, PDF, spreadsheets — use Read (it handles multimodal formats natively).',
-        ].join('\n')
-        : '';
-    return [
-        'You are answering a question from the GAIA benchmark.',
-        '',
-        `Question: ${question.question}`,
-        attachmentBlock,
-        '',
-        'Instructions:',
-        '1. Find the answer using your available tools:',
-        '   - WebSearch: for current facts, people, events, statistics',
-        '   - WebFetch: for full page content when a snippet is insufficient',
-        '   - Read: for file attachments (PDF, DOCX, XLSX, images, audio transcripts)',
-        '   - Bash: for computation, data processing, Python scripts',
-        '2. Be precise. If a number is asked, give just the number.',
-        '   If a name is asked, give just the name.',
-        '3. Output your final answer on the last line as exactly:',
-        '   FINAL_ANSWER: <your-answer>',
-        '4. Do NOT use commas in numbers (write 50000 not 50,000) unless explicitly required.',
-        '5. Do NOT include units (write "5" not "5 km") unless the question explicitly asks for them.',
-        '6. If you genuinely cannot find the answer after exhausting your tools, write:',
-        '   FINAL_ANSWER: unknown',
-        '',
-        'Find the answer and commit it with FINAL_ANSWER: on the last line.',
-    ].join('\n');
-}
-// ---------------------------------------------------------------------------
-// Core runner
-// ---------------------------------------------------------------------------
-/**
- * Run a single GAIA question via `claude -p` headless mode.
- *
- * Spawns a subprocess, captures JSON output, extracts the final answer.
- */
-export async function runGaiaQuestionViaClaudeP(question, options = {}) {
-    const model = options.model ?? CLAUDE_P_DEFAULT_MODEL;
-    const budgetUsd = options.budgetUsd ?? CLAUDE_P_PER_QUESTION_BUDGET_USD;
-    const timeoutMs = options.timeoutMs ?? CLAUDE_P_TIMEOUT_MS;
-    const claudeBin = options.claudeBin ?? '/Users/cohen/.local/bin/claude';
-    const prompt = buildClaudePPrompt(question);
-    // Build claude -p arguments.
-    // --dangerously-skip-permissions: acceptable here because GAIA is a read-only
-    // sandboxed benchmark context with no real-world side effects.  See module-level
-    // security note above.
-    const args = [
-        '-p',
-        prompt,
-        '--model', model,
-        '--max-budget-usd', String(budgetUsd),
-        '--output-format', 'json',
-        '--dangerously-skip-permissions',
-    ];
-    const startMs = Date.now();
-    return new Promise((resolve) => {
-        let stdout = '';
-        let stderr = '';
-        let settled = false;
-        const child = spawn(claudeBin, args, {
-            env: { ...process.env },
-            stdio: ['ignore', 'pipe', 'pipe'],
-        });
-        child.stdout.on('data', (chunk) => {
-            stdout += chunk.toString('utf-8');
-        });
-        child.stderr.on('data', (chunk) => {
-            stderr += chunk.toString('utf-8');
-        });
-        const timer = setTimeout(() => {
-            if (!settled) {
-                settled = true;
-                child.kill('SIGTERM');
-                resolve({
-                    finalAnswer: null,
-                    rawResult: '',
-                    isError: true,
-                    errorMessage: `Timed out after ${timeoutMs}ms`,
-                    costUsd: 0,
-                    wallMs: Date.now() - startMs,
-                    numTurns: 0,
-                    stopReason: 'timeout',
-                });
-            }
-        }, timeoutMs);
-        child.on('close', (code) => {
-            if (settled)
-                return;
-            settled = true;
-            clearTimeout(timer);
-            const wallMs = Date.now() - startMs;
-            // Parse JSON output from claude -p
-            const parsed = parseClaudePOutput(stdout, stderr, code ?? 1);
-            resolve({ ...parsed, wallMs });
-        });
-        child.on('error', (err) => {
-            if (settled)
-                return;
-            settled = true;
-            clearTimeout(timer);
-            resolve({
-                finalAnswer: null,
-                rawResult: '',
-                isError: true,
-                errorMessage: `Failed to spawn claude: ${err.message}`,
-                costUsd: 0,
-                wallMs: Date.now() - startMs,
-                numTurns: 0,
-            });
-        });
-    });
-}
-function parseClaudePOutput(stdout, stderr, exitCode) {
-    const raw = stdout.trim();
-    // Attempt JSON parse
-    let parsed = null;
-    try {
-        parsed = JSON.parse(raw);
-    }
-    catch {
-        // JSON parse failed — treat as error
-        return {
-            finalAnswer: null,
-            rawResult: raw || stderr.trim(),
-            isError: true,
-            errorMessage: `JSON parse failed (exitCode=${exitCode}): ${(raw || stderr).slice(0, 200)}`,
-            costUsd: 0,
-            numTurns: 0,
-        };
-    }
-    const isError = parsed.is_error === true || exitCode !== 0;
-    const resultText = parsed.result ?? '';
-    const costUsd = parsed.total_cost_usd ?? 0;
-    const numTurns = parsed.num_turns ?? 0;
-    const stopReason = parsed.subtype ?? parsed.stop_reason ?? undefined;
-    if (isError) {
-        const errMsg = (parsed.errors ?? []).join('; ') || `subtype=${parsed.subtype}`;
-        return {
-            finalAnswer: null,
-            rawResult: resultText,
-            isError: true,
-            errorMessage: errMsg,
-            costUsd,
-            numTurns,
-            stopReason,
-        };
-    }
-    // Extract FINAL_ANSWER from result text
-    const finalAnswer = extractFinalAnswer(resultText);
-    return {
-        finalAnswer,
-        rawResult: resultText,
-        isError: false,
-        costUsd,
-        numTurns,
-        stopReason,
-    };
-}
-// ---------------------------------------------------------------------------
-// Answer extraction
-// ---------------------------------------------------------------------------
-/**
- * Extract the FINAL_ANSWER value from claude -p's result text.
- *
- * Primary: regex match on `FINAL_ANSWER: <value>`
- * Fallback: last non-empty line if no FINAL_ANSWER marker found.
- */
-export function extractFinalAnswer(text) {
-    if (!text || !text.trim())
-        return null;
-    // Primary: FINAL_ANSWER: pattern
-    const match = FINAL_ANSWER_RE.exec(text);
-    if (match && match[1]) {
-        return match[1].trim();
-    }
-    // Fallback: last meaningful line (heuristic)
-    const lines = text.split('\n').map((l) => l.trim()).filter(Boolean);
-    const lastLine = lines[lines.length - 1];
-    if (lastLine && lastLine.length < 200) {
-        return lastLine;
-    }
-    return null;
-}
-/**
- * Run a batch of GAIA questions through the claude -p wrapper.
- *
- * Concurrency is limited (default 2) because each claude -p subprocess
- * is heavyweight — it starts a full Claude Code session with LSP etc.
- */
-export async function runGaiaQuestionsBatchViaClaudeP(questions, options = {}) {
-    const concurrency = options.concurrency ?? 2;
-    const results = new Array(questions.length);
-    for (let i = 0; i < questions.length; i += concurrency) {
-        const batch = questions.slice(i, Math.min(i + concurrency, questions.length));
-        const batchResults = await Promise.all(batch.map((q, batchIdx) => runGaiaQuestionViaClaudeP(q, options).then((r) => {
-            const globalIdx = i + batchIdx;
-            options.onProgress?.(globalIdx + 1, questions.length, q.task_id, r.finalAnswer, r.costUsd);
-            return r;
-        })));
-        for (let j = 0; j < batchResults.length; j++) {
-            results[i + j] = batchResults[j];
-        }
-    }
-    return results;
-}
-//# sourceMappingURL=gaia-claude-p.js.map

package/dist/src/benchmarks/gaia-claude-p.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"gaia-claude-p.js","sourceRoot":"","sources":["../../../src/benchmarks/gaia-claude-p.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAGlC,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,iFAAiF;AACjF,MAAM,CAAC,MAAM,sBAAsB,GAAG,mBAAmB,CAAC;AAE1D,oFAAoF;AACpF,MAAM,CAAC,MAAM,gCAAgC,GAAG,IAAI,CAAC;AAErD,kDAAkD;AAClD,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AAEjD,+DAA+D;AAC/D,MAAM,eAAe,GAAG,gCAAgC,CAAC;AAoCzD,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,QAAsB;IACvD,MAAM,eAAe,GAAG,QAAQ,CAAC,SAAS;QACxC,CAAC,CAAC;YACE,EAAE;YACF,yBAAyB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC3D,uDAAuD;YACvD,2FAA2F;SAC5F,CAAC,IAAI,CAAC,IAAI,CAAC;QACd,CAAC,CAAC,EAAE,CAAC;IAEP,OAAO;QACL,uDAAuD;QACvD,EAAE;QACF,aAAa,QAAQ,CAAC,QAAQ,EAAE;QAChC,eAAe;QACf,EAAE;QACF,eAAe;QACf,gDAAgD;QAChD,+DAA+D;QAC/D,qEAAqE;QACrE,8EAA8E;QAC9E,6DAA6D;QAC7D,4DAA4D;QAC5D,4CAA4C;QAC5C,0DAA0D;QAC1D,gCAAgC;QAChC,sFAAsF;QACtF,8FAA8F;QAC9F,gFAAgF;QAChF,0BAA0B;QAC1B,EAAE;QACF,oEAAoE;KACrE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,8EAA8E;AAC9E,cAAc;AACd,8EAA8E;AAE9E;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,QAAsB,EACtB,UAA0B,EAAE;IAE5B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,sBAAsB,CAAC;IACtD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,gCAAgC,CAAC;IACxE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,mBAAmB,CAAC;IAC3D,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,gCAAgC,CAAC;IAExE,MAAM,MAAM,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAE5C,6BAA6B;IAC7B,8EAA8E;IAC9E,iFAAiF;IACjF,uBAAuB;IACvB,MAAM,IAAI,GAAa;QACrB,IAAI;QACJ,MAAM;QACN,SAAS,EAAE,KAAK;QAChB,kBAAkB,EAAE,MAAM,CAAC,SAAS,CAAC;QACrC,iBAAiB,EAAE,MAAM;QACzB,gCAAgC;KACjC,CAAC;IAEF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE3B,OAAO,IAAI,OAAO,CAAgB,CAAC,OAAO,EAAE,EAAE;QAC5C,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,OAAO,GAAG,KAAK,CAAC;QAEpB,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,EAAE,IAAI,EAAE;YACnC,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE;YACvB,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;SAClC,CAAC,CAAC;QAEH,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YACxC,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YACxC,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,GAAG,IAAI,CAAC;gBACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACtB,OAAO,CAAC;oBACN,WAAW,EAAE,IAAI;oBACjB,SAAS,EAAE,EAAE;oBACb,OAAO,EAAE,IAAI;oBACb,YAAY,EAAE,mBAAmB,SAAS,IAAI;oBAC9C,OAAO,EAAE,CAAC;oBACV,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;oBAC5B,QAAQ,EAAE,CAAC;oBACX,UAAU,EAAE,SAAS;iBACtB,CAAC,CAAC;YACL,CAAC;QACH,CAAC,EAAE,SAAS,CAAC,CAAC;QAEd,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YACzB,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,YAAY,CAAC,KAAK,CAAC,CAAC;YAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;YAEpC,mCAAmC;YACnC,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC;YAE7D,OAAO,CAAC,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACxB,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,CAAC;gBACN,WAAW,EAAE,IAAI;gBACjB,SAAS,EAAE,EAAE;gBACb,OAAO,EAAE,IAAI;gBACb,YAAY,EAAE,2BAA2B,GAAG,CAAC,OAAO,EAAE;gBACtD,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;gBAC5B,QAAQ,EAAE,CAAC;aACZ,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAiBD,SAAS,kBAAkB,CACzB,MAAc,EACd,MAAc,EACd,QAAgB;IAEhB,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;IAE1B,qBAAqB;IACrB,IAAI,MAAM,GAA6B,IAAI,CAAC;IAC5C,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAsB,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,qCAAqC;QACrC,OAAO;YACL,WAAW,EAAE,IAAI;YACjB,SAAS,EAAE,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE;YAC/B,OAAO,EAAE,IAAI;YACb,YAAY,EAAE,+BAA+B,QAAQ,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;YAC1F,OAAO,EAAE,CAAC;YACV,QAAQ,EAAE,CAAC;SACZ,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,KAAK,IAAI,IAAI,QAAQ,KAAK,CAAC,CAAC;IAC3D,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;IACvC,MAAM,OAAO,GAAG,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;IACvC,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,WAAW,IAAI,SAAS,CAAC;IAErE,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,WAAW,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/E,OAAO;YACL,WAAW,EAAE,IAAI;YACjB,SAAS,EAAE,UAAU;YACrB,OAAO,EAAE,IAAI;YACb,YAAY,EAAE,MAAM;YACpB,OAAO;YACP,QAAQ;YACR,UAAU;SACX,CAAC;IACJ,CAAC;IAED,wCAAwC;IACxC,MAAM,WAAW,GAAG,kBAAkB,CAAC,UAAU,CAAC,CAAC;IAEnD,OAAO;QACL,WAAW;QACX,SAAS,EAAE,UAAU;QACrB,OAAO,EAAE,KAAK;QACd,OAAO;QACP,QAAQ;QACR,UAAU;KACX,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QAAE,OAAO,IAAI,CAAC;IAEvC,iCAAiC;IACjC,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzC,IAAI,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QACtB,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,6CAA6C;IAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACpE,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACzC,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACtC,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAaD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,+BAA+B,CACnD,SAAyB,EACzB,UAA+B,EAAE;IAEjC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAoB,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAE7D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;QACvD,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,WAAW,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;QAC9E,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,CACxB,yBAAyB,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;YAC/C,MAAM,SAAS,GAAG,CAAC,GAAG,QAAQ,CAAC;YAC/B,OAAO,CAAC,UAAU,EAAE,CAAC,SAAS,GAAG,CAAC,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3F,OAAO,CAAC,CAAC;QACX,CAAC,CAAC,CACH,CACF,CAAC;QACF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/src/benchmarks/gaia-claude-p.smoke.d.ts

@@ -1,20 +0,0 @@
-/**
- * Smoke tests for gaia-claude-p.ts — iter 54 (#2156)
- *
- * Three minimal cases to verify the claude -p wrapper end-to-end:
- *   1. Simple arithmetic: "what is 2+2" → expect "4"
- *   2. Current fact: population of Tokyo → expect a large number
- *   3. (Optional) Attachment path test — only runs when HF cache has a file
- *
- * Cost cap: ~$0.25 × 3 = $0.75 worst case at Sonnet rates, but using Haiku
- * for smoke to keep it under $0.30 total.
- *
- * Run:
- *   npx ts-node src/benchmarks/gaia-claude-p.smoke.ts
- *   # or after build:
- *   node dist/src/benchmarks/gaia-claude-p.smoke.js
- *
- * Refs: iter 54, #2156
- */
-export {};
-//# sourceMappingURL=gaia-claude-p.smoke.d.ts.map

package/dist/src/benchmarks/gaia-claude-p.smoke.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"gaia-claude-p.smoke.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/gaia-claude-p.smoke.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG"}

package/dist/src/benchmarks/gaia-claude-p.smoke.js

@@ -1,168 +0,0 @@
-/**
- * Smoke tests for gaia-claude-p.ts — iter 54 (#2156)
- *
- * Three minimal cases to verify the claude -p wrapper end-to-end:
- *   1. Simple arithmetic: "what is 2+2" → expect "4"
- *   2. Current fact: population of Tokyo → expect a large number
- *   3. (Optional) Attachment path test — only runs when HF cache has a file
- *
- * Cost cap: ~$0.25 × 3 = $0.75 worst case at Sonnet rates, but using Haiku
- * for smoke to keep it under $0.30 total.
- *
- * Run:
- *   npx ts-node src/benchmarks/gaia-claude-p.smoke.ts
- *   # or after build:
- *   node dist/src/benchmarks/gaia-claude-p.smoke.js
- *
- * Refs: iter 54, #2156
- */
-import { runGaiaQuestionViaClaudeP, extractFinalAnswer, buildClaudePPrompt, CLAUDE_P_DEFAULT_MODEL, } from './gaia-claude-p.js';
-// ---------------------------------------------------------------------------
-// Test helpers
-// ---------------------------------------------------------------------------
-let passed = 0;
-let failed = 0;
-function assert(condition, label, detail) {
-    if (condition) {
-        console.log(`  PASS  ${label}`);
-        passed++;
-    }
-    else {
-        console.error(`  FAIL  ${label}${detail ? ': ' + detail : ''}`);
-        failed++;
-    }
-}
-// ---------------------------------------------------------------------------
-// Unit tests (no claude -p invocation)
-// ---------------------------------------------------------------------------
-function testExtractFinalAnswer() {
-    console.log('\n-- Unit: extractFinalAnswer --');
-    assert(extractFinalAnswer('FINAL_ANSWER: 4') === '4', 'extracts plain number');
-    assert(extractFinalAnswer('Some reasoning...\nFINAL_ANSWER: Paris') === 'Paris', 'extracts after reasoning text');
-    assert(extractFinalAnswer('final_answer: 42') === '42', 'case-insensitive extraction');
-    assert(extractFinalAnswer('') === null, 'returns null for empty string');
-    assert(extractFinalAnswer('No answer marker here') !== null, 'fallback to last line when no marker');
-}
-function testBuildClaudePPrompt() {
-    console.log('\n-- Unit: buildClaudePPrompt --');
-    const q = {
-        task_id: 'test-001',
-        level: 1,
-        question: 'What is 2+2?',
-        final_answer: '4',
-        file_name: null,
-        file_path: null,
-    };
-    const prompt = buildClaudePPrompt(q);
-    assert(prompt.includes('What is 2+2?'), 'prompt contains question text');
-    assert(prompt.includes('FINAL_ANSWER:'), 'prompt instructs FINAL_ANSWER format');
-    assert(!prompt.includes('Attachment'), 'no attachment section for null file_path');
-    const qWithFile = { ...q, file_path: '/tmp/test.pdf' };
-    const promptWithFile = buildClaudePPrompt(qWithFile);
-    assert(promptWithFile.includes('/tmp/test.pdf'), 'attachment path included when present');
-    assert(promptWithFile.includes('Read tool'), 'mentions Read tool for attachments');
-}
-// ---------------------------------------------------------------------------
-// Integration tests (invoke claude -p)
-// ---------------------------------------------------------------------------
-async function testArithmetic() {
-    console.log('\n-- Integration: arithmetic (2+2) --');
-    const q = {
-        task_id: 'smoke-arithmetic',
-        level: 1,
-        question: 'What is 2 + 2? Provide only the numeric answer.',
-        final_answer: '4',
-        file_name: null,
-        file_path: null,
-    };
-    console.log('  Spawning claude -p ...');
-    const result = await runGaiaQuestionViaClaudeP(q, {
-        model: 'claude-haiku-4-5',
-        budgetUsd: 0.25,
-        timeoutMs: 120_000,
-    });
-    console.log(`  raw result: ${result.rawResult.slice(0, 100)}`);
-    console.log(`  finalAnswer: ${result.finalAnswer}`);
-    console.log(`  costUsd: $${result.costUsd.toFixed(4)}`);
-    console.log(`  numTurns: ${result.numTurns}`);
-    console.log(`  isError: ${result.isError}`);
-    assert(!result.isError, 'no error', result.errorMessage);
-    assert(result.finalAnswer !== null, 'finalAnswer is not null');
-    assert(result.finalAnswer === '4', `answer is "4"`, `got: "${result.finalAnswer}"`);
-    assert(result.costUsd > 0, 'cost is positive');
-    assert(result.wallMs > 0, 'wallMs is positive');
-}
-async function testCurrentFact() {
-    console.log('\n-- Integration: current fact (Tokyo population) --');
-    const q = {
-        task_id: 'smoke-tokyo-pop',
-        level: 1,
-        question: 'What is the approximate population of Tokyo (the city proper)? Give only a number in millions, rounded to the nearest million.',
-        final_answer: '14',
-        file_name: null,
-        file_path: null,
-    };
-    console.log('  Spawning claude -p ...');
-    const result = await runGaiaQuestionViaClaudeP(q, {
-        model: 'claude-haiku-4-5',
-        budgetUsd: 0.25,
-        timeoutMs: 120_000,
-    });
-    console.log(`  raw result excerpt: ${result.rawResult.slice(0, 150)}`);
-    console.log(`  finalAnswer: ${result.finalAnswer}`);
-    console.log(`  costUsd: $${result.costUsd.toFixed(4)}`);
-    assert(!result.isError, 'no error', result.errorMessage);
-    assert(result.finalAnswer !== null, 'finalAnswer is not null');
-    // Tokyo metro area is ~37M, city proper is ~13-14M — any multi-digit number is reasonable
-    const numAnswer = result.finalAnswer ? parseFloat(result.finalAnswer.replace(/[^0-9.]/g, '')) : NaN;
-    assert(!isNaN(numAnswer) && numAnswer > 0, 'answer is a non-empty number', `got: "${result.finalAnswer}"`);
-}
-async function testAnswerFormat() {
-    console.log('\n-- Integration: answer format discipline --');
-    const q = {
-        task_id: 'smoke-format',
-        level: 1,
-        question: 'What is the capital of France? Give only the city name, nothing else.',
-        final_answer: 'Paris',
-        file_name: null,
-        file_path: null,
-    };
-    console.log('  Spawning claude -p ...');
-    const result = await runGaiaQuestionViaClaudeP(q, {
-        model: 'claude-haiku-4-5',
-        budgetUsd: 0.25,
-        timeoutMs: 120_000,
-    });
-    console.log(`  raw result excerpt: ${result.rawResult.slice(0, 150)}`);
-    console.log(`  finalAnswer: ${result.finalAnswer}`);
-    console.log(`  costUsd: $${result.costUsd.toFixed(4)}`);
-    assert(!result.isError, 'no error', result.errorMessage);
-    assert(result.finalAnswer !== null, 'finalAnswer is not null');
-    assert(result.finalAnswer !== null && result.finalAnswer.toLowerCase().includes('paris'), 'answer contains "paris"', `got: "${result.finalAnswer}"`);
-}
-// ---------------------------------------------------------------------------
-// Main
-// ---------------------------------------------------------------------------
-async function main() {
-    console.log('=== gaia-claude-p smoke tests ===');
-    console.log(`Model: claude-haiku-4-5 (smoke uses Haiku to minimize cost)`);
-    console.log(`Default production model: ${CLAUDE_P_DEFAULT_MODEL}`);
-    // Unit tests (no API calls)
-    testExtractFinalAnswer();
-    testBuildClaudePPrompt();
-    // Integration tests (invoke claude -p)
-    await testArithmetic();
-    await testCurrentFact();
-    await testAnswerFormat();
-    // Summary
-    const total = passed + failed;
-    console.log(`\n=== Results: ${passed}/${total} passed ===`);
-    if (failed > 0) {
-        process.exit(1);
-    }
-}
-main().catch((err) => {
-    console.error('Smoke test runner error:', err);
-    process.exit(1);
-});
-//# sourceMappingURL=gaia-claude-p.smoke.js.map

package/dist/src/benchmarks/gaia-claude-p.smoke.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"gaia-claude-p.smoke.js","sourceRoot":"","sources":["../../../src/benchmarks/gaia-claude-p.smoke.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EACL,yBAAyB,EACzB,kBAAkB,EAClB,kBAAkB,EAClB,sBAAsB,GACvB,MAAM,oBAAoB,CAAC;AAG5B,8EAA8E;AAC9E,eAAe;AACf,8EAA8E;AAE9E,IAAI,MAAM,GAAG,CAAC,CAAC;AACf,IAAI,MAAM,GAAG,CAAC,CAAC;AAEf,SAAS,MAAM,CAAC,SAAkB,EAAE,KAAa,EAAE,MAAe;IAChE,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,WAAW,KAAK,EAAE,CAAC,CAAC;QAChC,MAAM,EAAE,CAAC;IACX,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,KAAK,CAAC,WAAW,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAChE,MAAM,EAAE,CAAC;IACX,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,uCAAuC;AACvC,8EAA8E;AAE9E,SAAS,sBAAsB;IAC7B,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;IAEhD,MAAM,CACJ,kBAAkB,CAAC,iBAAiB,CAAC,KAAK,GAAG,EAC7C,uBAAuB,CACxB,CAAC;IACF,MAAM,CACJ,kBAAkB,CAAC,wCAAwC,CAAC,KAAK,OAAO,EACxE,+BAA+B,CAChC,CAAC;IACF,MAAM,CACJ,kBAAkB,CAAC,kBAAkB,CAAC,KAAK,IAAI,EAC/C,6BAA6B,CAC9B,CAAC;IACF,MAAM,CACJ,kBAAkB,CAAC,EAAE,CAAC,KAAK,IAAI,EAC/B,+BAA+B,CAChC,CAAC;IACF,MAAM,CACJ,kBAAkB,CAAC,uBAAuB,CAAC,KAAK,IAAI,EACpD,sCAAsC,CACvC,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB;IAC7B,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;IAEhD,MAAM,CAAC,GAAiB;QACtB,OAAO,EAAE,UAAU;QACnB,KAAK,EAAE,CAAC;QACR,QAAQ,EAAE,cAAc;QACxB,YAAY,EAAE,GAAG;QACjB,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,IAAI;KAChB,CAAC;IAEF,MAAM,MAAM,GAAG,kBAAkB,CAAC,CAAC,CAAC,CAAC;IACrC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,+BAA+B,CAAC,CAAC;IACzE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,sCAAsC,CAAC,CAAC;IACjF,MAAM,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,0CAA0C,CAAC,CAAC;IAEnF,MAAM,SAAS,GAAiB,EAAE,GAAG,CAAC,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC;IACrE,MAAM,cAAc,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IACrD,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,uCAAuC,CAAC,CAAC;IAC1F,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,oCAAoC,CAAC,CAAC;AACrF,CAAC;AAED,8EAA8E;AAC9E,uCAAuC;AACvC,8EAA8E;AAE9E,KAAK,UAAU,cAAc;IAC3B,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;IAErD,MAAM,CAAC,GAAiB;QACtB,OAAO,EAAE,kBAAkB;QAC3B,KAAK,EAAE,CAAC;QACR,QAAQ,EAAE,iDAAiD;QAC3D,YAAY,EAAE,GAAG;QACjB,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,IAAI;KAChB,CAAC;IAEF,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,yBAAyB,CAAC,CAAC,EAAE;QAChD,KAAK,EAAE,kBAAkB;QACzB,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,OAAO;KACnB,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/D,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;IACpD,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC9C,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IAE5C,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;IACzD,MAAM,CAAC,MAAM,CAAC,WAAW,KAAK,IAAI,EAAE,yBAAyB,CAAC,CAAC;IAC/D,MAAM,CAAC,MAAM,CAAC,WAAW,KAAK,GAAG,EAAE,eAAe,EAAE,SAAS,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC;IACpF,MAAM,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,EAAE,kBAAkB,CAAC,CAAC;IAC/C,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,oBAAoB,CAAC,CAAC;AAClD,CAAC;AAED,KAAK,UAAU,eAAe;IAC5B,OAAO,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC;IAEpE,MAAM,CAAC,GAAiB;QACtB,OAAO,EAAE,iBAAiB;QAC1B,KAAK,EAAE,CAAC;QACR,QAAQ,EAAE,gIAAgI;QAC1I,YAAY,EAAE,IAAI;QAClB,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,IAAI;KAChB,CAAC;IAEF,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,yBAAyB,CAAC,CAAC,EAAE;QAChD,KAAK,EAAE,kBAAkB;QACzB,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,OAAO;KACnB,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,yBAAyB,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;IACpD,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAExD,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;IACzD,MAAM,CAAC,MAAM,CAAC,WAAW,KAAK,IAAI,EAAE,yBAAyB,CAAC,CAAC;IAC/D,0FAA0F;IAC1F,MAAM,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACpG,MAAM,CACJ,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,SAAS,GAAG,CAAC,EAClC,8BAA8B,EAC9B,SAAS,MAAM,CAAC,WAAW,GAAG,CAC/B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,gBAAgB;IAC7B,OAAO,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;IAE7D,MAAM,CAAC,GAAiB;QACtB,OAAO,EAAE,cAAc;QACvB,KAAK,EAAE,CAAC;QACR,QAAQ,EAAE,uEAAuE;QACjF,YAAY,EAAE,OAAO;QACrB,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,IAAI;KAChB,CAAC;IAEF,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,yBAAyB,CAAC,CAAC,EAAE;QAChD,KAAK,EAAE,kBAAkB;QACzB,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,OAAO;KACnB,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,yBAAyB,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;IACpD,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAExD,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;IACzD,MAAM,CAAC,MAAM,CAAC,WAAW,KAAK,IAAI,EAAE,yBAAyB,CAAC,CAAC;IAC/D,MAAM,CACJ,MAAM,CAAC,WAAW,KAAK,IAAI,IAAI,MAAM,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,EACjF,yBAAyB,EACzB,SAAS,MAAM,CAAC,WAAW,GAAG,CAC/B,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,OAAO;AACP,8EAA8E;AAE9E,KAAK,UAAU,IAAI;IACjB,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,6DAA6D,CAAC,CAAC;IAC3E,OAAO,CAAC,GAAG,CAAC,6BAA6B,sBAAsB,EAAE,CAAC,CAAC;IAEnE,4BAA4B;IAC5B,sBAAsB,EAAE,CAAC;IACzB,sBAAsB,EAAE,CAAC;IAEzB,uCAAuC;IACvC,MAAM,cAAc,EAAE,CAAC;IACvB,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,gBAAgB,EAAE,CAAC;IAEzB,UAAU;IACV,MAAM,KAAK,GAAG,MAAM,GAAG,MAAM,CAAC;IAC9B,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,IAAI,KAAK,aAAa,CAAC,CAAC;IAE5D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAE,GAAG,CAAC,CAAC;IAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}