@claude-flow/cli 3.11.0 → 3.12.0

package/dist/src/benchmarks/gaia-agent-gemini.js

@@ -1,352 +0,0 @@
-/**
- * GAIA Agent — Gemini 2.5 Pro Thinking adapter (Phase 2 pilot)
- *
- * Adapts the existing GAIA tool catalogue to Google's generateContent API,
- * translating Anthropic tool_use ↔ Gemini functionCall/functionResponse.
- *
- * Loop algorithm mirrors gaia-agent.ts:
- *   1. Build initial contents array with system instruction + first user turn.
- *   2. Call Gemini generateContent with functionDeclarations.
- *   3. On functionCall parts: execute tools, append functionResponse, repeat.
- *   4. On text-only response: scan for FINAL_ANSWER pattern.
- *   5. On maxTurns: return timedOut result.
- *
- * Gemini 2.5 Pro pricing (2026-05-28, per million tokens):
- *   Input:   $1.25 (≤200k tokens), $2.50 (>200k)
- *   Output:  $10.00 (≤200k tokens), $15.00 (>200k)
- *   Thinking tokens are billed as output tokens.
- *
- * Cost gate for pilot: ≤$0.12 per question average.
- * Expected per-question cost at ~10k input + ~2k output: ~$0.032 — well within gate.
- *
- * Refs: ADR-133, ADR-135, #2156 Phase 2
- */
-import { execSync } from 'node:child_process';
-import * as fs from 'node:fs';
-import * as path from 'node:path';
-import { createDefaultToolCatalogue, } from './gaia-tools/index.js';
-// ---------------------------------------------------------------------------
-// Constants
-// ---------------------------------------------------------------------------
-const GEMINI_API_BASE = 'https://generativelanguage.googleapis.com/v1beta/models';
-export const DEFAULT_GEMINI_MODEL = 'gemini-2.5-pro';
-const DEFAULT_MAX_TURNS = 12;
-const DEFAULT_MAX_TOKENS_PER_TURN = 8192;
-const DEFAULT_PER_TURN_TIMEOUT_MS = 120_000;
-/** Gemini 2.5 Pro pricing (USD per million tokens, ≤200k bracket). */
-const GEMINI_INPUT_COST_PER_M = 1.25;
-const GEMINI_OUTPUT_COST_PER_M = 10.00;
-const FINAL_ANSWER_RE = /FINAL_ANSWER:\s*(.+)/i;
-// ---------------------------------------------------------------------------
-// API key resolution
-// ---------------------------------------------------------------------------
-export function resolveGeminiApiKey(apiKey) {
-    if (apiKey && apiKey.trim())
-        return apiKey.trim();
-    const envKey = process.env['GOOGLE_AI_API_KEY'];
-    if (envKey && envKey.trim())
-        return envKey.trim();
-    try {
-        const out = execSync('gcloud secrets versions access latest --secret=GOOGLE_AI_API_KEY --project=ruv-dev 2>/dev/null', { encoding: 'utf-8', timeout: 10_000 }).trim();
-        if (out)
-            return out;
-    }
-    catch {
-        /* fall through */
-    }
-    throw new Error('GOOGLE_AI_API_KEY not found. Set the env var or store in GCP Secret Manager.');
-}
-function toGeminiType(anthropicType) {
-    const map = {
-        string: 'STRING',
-        number: 'NUMBER',
-        integer: 'INTEGER',
-        boolean: 'BOOLEAN',
-        array: 'ARRAY',
-        object: 'OBJECT',
-    };
-    return map[anthropicType.toLowerCase()] ?? 'STRING';
-}
-function translateToolDef(def) {
-    const props = {};
-    for (const [key, val] of Object.entries(def.input_schema.properties)) {
-        props[key] = { type: toGeminiType(val.type), description: val.description };
-    }
-    return {
-        name: def.name,
-        description: def.description,
-        parameters: {
-            type: 'OBJECT',
-            properties: props,
-            required: def.input_schema.required ?? [],
-        },
-    };
-}
-// ---------------------------------------------------------------------------
-// Single Gemini API call
-// ---------------------------------------------------------------------------
-async function callGemini(apiKey, model, contents, systemInstruction, functionDeclarations, maxTokens, timeoutMs) {
-    const url = `${GEMINI_API_BASE}/${model}:generateContent?key=${encodeURIComponent(apiKey)}`;
-    const requestBody = {
-        contents,
-        systemInstruction: { parts: [{ text: systemInstruction }] },
-        generationConfig: {
-            maxOutputTokens: maxTokens,
-            thinkingConfig: { thinkingBudget: 8192 },
-        },
-    };
-    if (functionDeclarations.length > 0) {
-        requestBody['tools'] = [{ functionDeclarations }];
-        requestBody['toolConfig'] = { functionCallingConfig: { mode: 'AUTO' } };
-    }
-    const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(), timeoutMs);
-    let res;
-    try {
-        res = await fetch(url, {
-            method: 'POST',
-            headers: { 'Content-Type': 'application/json' },
-            body: JSON.stringify(requestBody),
-            signal: controller.signal,
-        });
-    }
-    finally {
-        clearTimeout(timer);
-    }
-    if (!res.ok) {
-        const errText = await res.text().catch(() => '<unreadable>');
-        throw new Error(`Gemini API ${res.status}: ${errText.slice(0, 400)}`);
-    }
-    return (await res.json());
-}
-// ---------------------------------------------------------------------------
-// Build initial contents (handles image attachments inline)
-// ---------------------------------------------------------------------------
-function buildInitialContents(question) {
-    const questionText = question.question;
-    const parts = [];
-    if (question.file_path) {
-        const ext = path.extname(question.file_path).toLowerCase();
-        const imageExts = ['.png', '.jpg', '.jpeg', '.gif', '.webp'];
-        if (imageExts.includes(ext)) {
-            try {
-                const buf = fs.readFileSync(question.file_path);
-                const mediaTypeMap = {
-                    '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
-                    '.gif': 'image/gif', '.webp': 'image/webp',
-                };
-                parts.push({ text: questionText });
-                // Gemini inline image part
-                parts.push({
-                    inlineData: {
-                        mimeType: mediaTypeMap[ext] ?? 'image/png',
-                        data: buf.toString('base64'),
-                    },
-                });
-            }
-            catch {
-                parts.push({
-                    text: questionText + `\n\nAttached file: ${question.file_path}. Call file_read to read it.`,
-                });
-            }
-        }
-        else {
-            parts.push({
-                text: questionText + `\n\nThis question has an attached file. Call file_read with path="${question.file_path}" to read it, then answer.`,
-            });
-        }
-    }
-    else {
-        parts.push({ text: questionText });
-    }
-    return [{ role: 'user', parts }];
-}
-// ---------------------------------------------------------------------------
-// Extract FINAL_ANSWER from model response text
-// ---------------------------------------------------------------------------
-function extractFinalAnswer(candidate) {
-    if (!candidate?.content?.parts)
-        return null;
-    for (const part of candidate.content.parts) {
-        if (part.thought)
-            continue;
-        if (part.text) {
-            const match = FINAL_ANSWER_RE.exec(part.text);
-            if (match?.[1])
-                return match[1].trim();
-        }
-    }
-    return null;
-}
-// ---------------------------------------------------------------------------
-// Check if response has any functionCall parts
-// ---------------------------------------------------------------------------
-function getFunctionCalls(candidate) {
-    if (!candidate?.content?.parts)
-        return [];
-    return candidate.content.parts
-        .filter((p) => !!p.functionCall)
-        .map((p) => p.functionCall);
-}
-// ---------------------------------------------------------------------------
-// Main agent loop
-// ---------------------------------------------------------------------------
-export async function runGeminiAgent(question, options = {}) {
-    const { model = DEFAULT_GEMINI_MODEL, maxTurns = DEFAULT_MAX_TURNS, maxTokensPerTurn = DEFAULT_MAX_TOKENS_PER_TURN, perTurnTimeoutMs = DEFAULT_PER_TURN_TIMEOUT_MS, apiKey: suppliedKey, catalogue: suppliedCatalogue, } = options;
-    const wallStart = Date.now();
-    const apiKey = resolveGeminiApiKey(suppliedKey);
-    const catalogue = suppliedCatalogue ?? createDefaultToolCatalogue();
-    const functionDeclarations = catalogue.map((t) => translateToolDef(t.definition));
-    const systemInstruction = [
-        'You are a precise question-answering agent. Answer the user\'s question using the tools available.',
-        '',
-        'RULES:',
-        '1. Use tools when you need information you do not have with certainty.',
-        '2. When you have a final answer, output it on its own line in this EXACT format:',
-        '   FINAL_ANSWER: <your answer here>',
-        '3. Keep answers concise. For numbers, give just the number. For names, give just the name.',
-        '4. Do not include units unless the question specifically asks for them.',
-        '5. MANDATORY: You MUST ALWAYS end your final response with a FINAL_ANSWER line.',
-        '6. If the question text appears garbled or reversed, try to interpret it before concluding you cannot answer.',
-    ].join('\n');
-    const toolCallsByName = {};
-    let totalInputTokens = 0;
-    let totalOutputTokens = 0;
-    let totalThinkingTokens = 0;
-    const contents = buildInitialContents(question);
-    let turns = 0;
-    for (let turn = 0; turn < maxTurns; turn++) {
-        turns = turn + 1;
-        let resp;
-        try {
-            resp = await callGemini(apiKey, model, contents, systemInstruction, functionDeclarations, maxTokensPerTurn, perTurnTimeoutMs);
-        }
-        catch (err) {
-            return {
-                questionId: question.task_id,
-                finalAnswer: null,
-                turns,
-                toolCallsByName,
-                totalInputTokens,
-                totalOutputTokens,
-                totalThinkingTokens,
-                wallMs: Date.now() - wallStart,
-                estimatedCostUsd: estimateCost(totalInputTokens, totalOutputTokens),
-                error: err instanceof Error ? err.message : String(err),
-            };
-        }
-        if (resp.error) {
-            return {
-                questionId: question.task_id,
-                finalAnswer: null,
-                turns,
-                toolCallsByName,
-                totalInputTokens,
-                totalOutputTokens,
-                totalThinkingTokens,
-                wallMs: Date.now() - wallStart,
-                estimatedCostUsd: estimateCost(totalInputTokens, totalOutputTokens),
-                error: `Gemini error ${resp.error.code}: ${resp.error.message}`,
-            };
-        }
-        // Accumulate token counts
-        const usage = resp.usageMetadata ?? {};
-        totalInputTokens += usage.promptTokenCount ?? 0;
-        totalOutputTokens += usage.candidatesTokenCount ?? 0;
-        totalThinkingTokens += usage.thoughtsTokenCount ?? 0;
-        const candidate = resp.candidates?.[0];
-        const functionCalls = getFunctionCalls(candidate);
-        if (functionCalls.length > 0) {
-            // Append model turn (with functionCall parts)
-            if (candidate?.content) {
-                contents.push({ role: 'model', parts: candidate.content.parts });
-            }
-            // Execute all function calls and build functionResponse parts
-            const responseParts = await Promise.all(functionCalls.map(async (call) => {
-                toolCallsByName[call.name] = (toolCallsByName[call.name] ?? 0) + 1;
-                const tool = catalogue.find((t) => t.name === call.name);
-                if (!tool) {
-                    return {
-                        functionResponse: {
-                            name: call.name,
-                            response: { error: `Unknown tool "${call.name}"` },
-                        },
-                    };
-                }
-                try {
-                    const output = await tool.execute(call.args);
-                    return {
-                        functionResponse: {
-                            name: call.name,
-                            response: { output },
-                        },
-                    };
-                }
-                catch (err) {
-                    return {
-                        functionResponse: {
-                            name: call.name,
-                            response: { error: err instanceof Error ? err.message : String(err) },
-                        },
-                    };
-                }
-            }));
-            contents.push({ role: 'user', parts: responseParts });
-            continue;
-        }
-        // No function calls — extract final answer
-        const finalAnswer = extractFinalAnswer(candidate);
-        return {
-            questionId: question.task_id,
-            finalAnswer,
-            turns,
-            toolCallsByName,
-            totalInputTokens,
-            totalOutputTokens,
-            totalThinkingTokens,
-            wallMs: Date.now() - wallStart,
-            estimatedCostUsd: estimateCost(totalInputTokens, totalOutputTokens + totalThinkingTokens),
-        };
-    }
-    // Exhausted maxTurns
-    return {
-        questionId: question.task_id,
-        finalAnswer: null,
-        turns,
-        toolCallsByName,
-        totalInputTokens,
-        totalOutputTokens,
-        totalThinkingTokens,
-        wallMs: Date.now() - wallStart,
-        estimatedCostUsd: estimateCost(totalInputTokens, totalOutputTokens + totalThinkingTokens),
-        timedOut: true,
-    };
-}
-function estimateCost(inputTokens, outputTokens) {
-    return ((inputTokens / 1_000_000) * GEMINI_INPUT_COST_PER_M +
-        (outputTokens / 1_000_000) * GEMINI_OUTPUT_COST_PER_M);
-}
-// ---------------------------------------------------------------------------
-// Answer matching (same as gaia-agent.ts)
-// ---------------------------------------------------------------------------
-export function isGeminiAnswerCorrect(modelAnswer, expected) {
-    if (!modelAnswer)
-        return false;
-    const norm = (s) => s.trim().toLowerCase();
-    const normModel = norm(modelAnswer);
-    const normExpected = norm(expected);
-    if (normModel === normExpected)
-        return true;
-    if (normModel.includes(normExpected))
-        return true;
-    if (normExpected.includes(normModel))
-        return true;
-    const numModel = parseFloat(normModel.replace(/[^0-9.\-]/g, ''));
-    const numExpected = parseFloat(normExpected.replace(/[^0-9.\-]/g, ''));
-    if (!Number.isNaN(numModel) && !Number.isNaN(numExpected) &&
-        numExpected !== 0 &&
-        Math.abs((numModel - numExpected) / numExpected) < 0.01)
-        return true;
-    return false;
-}
-//# sourceMappingURL=gaia-agent-gemini.js.map

package/dist/src/benchmarks/gaia-agent-gemini.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"gaia-agent-gemini.js","sourceRoot":"","sources":["../../../src/benchmarks/gaia-agent-gemini.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAIlC,OAAO,EACL,0BAA0B,GAG3B,MAAM,uBAAuB,CAAC;AAE/B,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,MAAM,eAAe,GAAG,yDAAyD,CAAC;AAClF,MAAM,CAAC,MAAM,oBAAoB,GAAG,gBAAgB,CAAC;AACrD,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAC7B,MAAM,2BAA2B,GAAG,IAAI,CAAC;AACzC,MAAM,2BAA2B,GAAG,OAAO,CAAC;AAE5C,sEAAsE;AACtE,MAAM,uBAAuB,GAAG,IAAI,CAAC;AACrC,MAAM,wBAAwB,GAAG,KAAK,CAAC;AAEvC,MAAM,eAAe,GAAG,uBAAuB,CAAC;AA6BhD,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,MAAM,UAAU,mBAAmB,CAAC,MAAe;IACjD,IAAI,MAAM,IAAI,MAAM,CAAC,IAAI,EAAE;QAAE,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IAElD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IAChD,IAAI,MAAM,IAAI,MAAM,CAAC,IAAI,EAAE;QAAE,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IAElD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,QAAQ,CAClB,gGAAgG,EAChG,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CACvC,CAAC,IAAI,EAAE,CAAC;QACT,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,kBAAkB;IACpB,CAAC;IAED,MAAM,IAAI,KAAK,CACb,8EAA8E,CAC/E,CAAC;AACJ,CAAC;AAgBD,SAAS,YAAY,CAAC,aAAqB;IACzC,MAAM,GAAG,GAA2B;QAClC,MAAM,EAAE,QAAQ;QAChB,MAAM,EAAE,QAAQ;QAChB,OAAO,EAAE,SAAS;QAClB,OAAO,EAAE,SAAS;QAClB,KAAK,EAAE,OAAO;QACd,MAAM,EAAE,QAAQ;KACjB,CAAC;IACF,OAAO,GAAG,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC,IAAI,QAAQ,CAAC;AACtD,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAmB;IAC3C,MAAM,KAAK,GAA2D,EAAE,CAAC;IACzE,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC;QACrE,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE,CAAC;IAC9E,CAAC;IACD,OAAO;QACL,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,UAAU,EAAE;YACV,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,KAAK;YACjB,QAAQ,EAAE,GAAG,CAAC,YAAY,CAAC,QAAQ,IAAI,EAAE;SAC1C;KACF,CAAC;AACJ,CAAC;AAoCD,8EAA8E;AAC9E,yBAAyB;AACzB,8EAA8E;AAE9E,KAAK,UAAU,UAAU,CACvB,MAAc,EACd,KAAa,EACb,QAAyB,EACzB,iBAAyB,EACzB,oBAAiD,EACjD,SAAiB,EACjB,SAAiB;IAEjB,MAAM,GAAG,GAAG,GAAG,eAAe,IAAI,KAAK,wBAAwB,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;IAE5F,MAAM,WAAW,GAA4B;QAC3C,QAAQ;QACR,iBAAiB,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,EAAE;QAC3D,gBAAgB,EAAE;YAChB,eAAe,EAAE,SAAS;YAC1B,cAAc,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE;SACzC;KACF,CAAC;IAEF,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,oBAAoB,EAAE,CAAC,CAAC;QAClD,WAAW,CAAC,YAAY,CAAC,GAAG,EAAE,qBAAqB,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC;IAC1E,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,IAAI,GAAa,CAAC;IAClB,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YACrB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;YACjC,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,cAAc,CAAC,CAAC;QAC7D,MAAM,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,MAAM,KAAK,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAkC,CAAC;AAC7D,CAAC;AAED,8EAA8E;AAC9E,4DAA4D;AAC5D,8EAA8E;AAE9E,SAAS,oBAAoB,CAAC,QAAsB;IAClD,MAAM,YAAY,GAAG,QAAQ,CAAC,QAAQ,CAAC;IACvC,MAAM,KAAK,GAAiB,EAAE,CAAC;IAE/B,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;QAC3D,MAAM,SAAS,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QAE7D,IAAI,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;gBAChD,MAAM,YAAY,GAA2B;oBAC3C,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY;oBAChE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,YAAY;iBAC3C,CAAC;gBACF,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC;gBACnC,2BAA2B;gBAC1B,KAAmD,CAAC,IAAI,CAAC;oBACxD,UAAU,EAAE;wBACV,QAAQ,EAAE,YAAY,CAAC,GAAG,CAAC,IAAI,WAAW;wBAC1C,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;qBAC7B;iBACF,CAAC,CAAC;YACL,CAAC;YAAC,MAAM,CAAC;gBACP,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI,EAAE,YAAY,GAAG,sBAAsB,QAAQ,CAAC,SAAS,8BAA8B;iBAC5F,CAAC,CAAC;YACL,CAAC;QACH,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,YAAY,GAAG,qEAAqE,QAAQ,CAAC,SAAS,4BAA4B;aACzI,CAAC,CAAC;QACL,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC;IACrC,CAAC;IAED,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;AACnC,CAAC;AAED,8EAA8E;AAC9E,gDAAgD;AAChD,8EAA8E;AAE9E,SAAS,kBAAkB,CAAC,SAAsC;IAChE,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,KAAK;QAAE,OAAO,IAAI,CAAC;IAC5C,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAC3C,IAAI,IAAI,CAAC,OAAO;YAAE,SAAS;QAC3B,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9C,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC;gBAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACzC,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,+CAA+C;AAC/C,8EAA8E;AAE9E,SAAS,gBAAgB,CACvB,SAAsC;IAEtC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,KAAK;QAAE,OAAO,EAAE,CAAC;IAC1C,OAAO,SAAS,CAAC,OAAO,CAAC,KAAK;SAC3B,MAAM,CAAC,CAAC,CAAC,EAA+E,EAAE,CACzF,CAAC,CAAC,CAAC,CAAC,YAAY,CACjB;SACA,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;AAChC,CAAC;AAED,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,QAAsB,EACtB,UAA8B,EAAE;IAEhC,MAAM,EACJ,KAAK,GAAG,oBAAoB,EAC5B,QAAQ,GAAG,iBAAiB,EAC5B,gBAAgB,GAAG,2BAA2B,EAC9C,gBAAgB,GAAG,2BAA2B,EAC9C,MAAM,EAAE,WAAW,EACnB,SAAS,EAAE,iBAAiB,GAC7B,GAAG,OAAO,CAAC;IAEZ,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,iBAAiB,IAAI,0BAA0B,EAAE,CAAC;IACpE,MAAM,oBAAoB,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;IAElF,MAAM,iBAAiB,GAAG;QACxB,oGAAoG;QACpG,EAAE;QACF,QAAQ;QACR,wEAAwE;QACxE,kFAAkF;QAClF,qCAAqC;QACrC,4FAA4F;QAC5F,yEAAyE;QACzE,iFAAiF;QACjF,+GAA+G;KAChH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEb,MAAM,eAAe,GAA2B,EAAE,CAAC;IACnD,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAE5B,MAAM,QAAQ,GAAoB,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IACjE,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC;QAC3C,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC;QAEjB,IAAI,IAAmC,CAAC;QACxC,IAAI,CAAC;YACH,IAAI,GAAG,MAAM,UAAU,CACrB,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,iBAAiB,EAC1C,oBAAoB,EAAE,gBAAgB,EAAE,gBAAgB,CACzD,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,UAAU,EAAE,QAAQ,CAAC,OAAO;gBAC5B,WAAW,EAAE,IAAI;gBACjB,KAAK;gBACL,eAAe;gBACf,gBAAgB;gBAChB,iBAAiB;gBACjB,mBAAmB;gBACnB,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAC9B,gBAAgB,EAAE,YAAY,CAAC,gBAAgB,EAAE,iBAAiB,CAAC;gBACnE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,UAAU,EAAE,QAAQ,CAAC,OAAO;gBAC5B,WAAW,EAAE,IAAI;gBACjB,KAAK;gBACL,eAAe;gBACf,gBAAgB;gBAChB,iBAAiB;gBACjB,mBAAmB;gBACnB,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAC9B,gBAAgB,EAAE,YAAY,CAAC,gBAAgB,EAAE,iBAAiB,CAAC;gBACnE,KAAK,EAAE,gBAAgB,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE;aAChE,CAAC;QACJ,CAAC;QAED,0BAA0B;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,IAAI,EAAE,CAAC;QACvC,gBAAgB,IAAI,KAAK,CAAC,gBAAgB,IAAI,CAAC,CAAC;QAChD,iBAAiB,IAAI,KAAK,CAAC,oBAAoB,IAAI,CAAC,CAAC;QACrD,mBAAmB,IAAI,KAAK,CAAC,kBAAkB,IAAI,CAAC,CAAC;QAErD,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,aAAa,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAElD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,8CAA8C;YAC9C,IAAI,SAAS,EAAE,OAAO,EAAE,CAAC;gBACvB,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;YACnE,CAAC;YAED,8DAA8D;YAC9D,MAAM,aAAa,GAAiB,MAAM,OAAO,CAAC,GAAG,CACnD,aAAa,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAuB,EAAE;gBACpD,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;gBACnE,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC;gBACzD,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,OAAO;wBACL,gBAAgB,EAAE;4BAChB,IAAI,EAAE,IAAI,CAAC,IAAI;4BACf,QAAQ,EAAE,EAAE,KAAK,EAAE,iBAAiB,IAAI,CAAC,IAAI,GAAG,EAAE;yBACnD;qBACF,CAAC;gBACJ,CAAC;gBACD,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC7C,OAAO;wBACL,gBAAgB,EAAE;4BAChB,IAAI,EAAE,IAAI,CAAC,IAAI;4BACf,QAAQ,EAAE,EAAE,MAAM,EAAE;yBACrB;qBACF,CAAC;gBACJ,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,OAAO;wBACL,gBAAgB,EAAE;4BAChB,IAAI,EAAE,IAAI,CAAC,IAAI;4BACf,QAAQ,EAAE,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;yBACtE;qBACF,CAAC;gBACJ,CAAC;YACH,CAAC,CAAC,CACH,CAAC;YAEF,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC;YACtD,SAAS;QACX,CAAC;QAED,2CAA2C;QAC3C,MAAM,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;QAClD,OAAO;YACL,UAAU,EAAE,QAAQ,CAAC,OAAO;YAC5B,WAAW;YACX,KAAK;YACL,eAAe;YACf,gBAAgB;YAChB,iBAAiB;YACjB,mBAAmB;YACnB,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAC9B,gBAAgB,EAAE,YAAY,CAAC,gBAAgB,EAAE,iBAAiB,GAAG,mBAAmB,CAAC;SAC1F,CAAC;IACJ,CAAC;IAED,qBAAqB;IACrB,OAAO;QACL,UAAU,EAAE,QAAQ,CAAC,OAAO;QAC5B,WAAW,EAAE,IAAI;QACjB,KAAK;QACL,eAAe;QACf,gBAAgB;QAChB,iBAAiB;QACjB,mBAAmB;QACnB,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAC9B,gBAAgB,EAAE,YAAY,CAAC,gBAAgB,EAAE,iBAAiB,GAAG,mBAAmB,CAAC;QACzF,QAAQ,EAAE,IAAI;KACf,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,WAAmB,EAAE,YAAoB;IAC7D,OAAO,CACL,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,uBAAuB;QACnD,CAAC,YAAY,GAAG,SAAS,CAAC,GAAG,wBAAwB,CACtD,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,0CAA0C;AAC1C,8EAA8E;AAE9E,MAAM,UAAU,qBAAqB,CAAC,WAAmB,EAAE,QAAgB;IACzE,IAAI,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC;IAC/B,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACnD,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;IACpC,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;IACpC,IAAI,SAAS,KAAK,YAAY;QAAE,OAAO,IAAI,CAAC;IAC5C,IAAI,SAAS,CAAC,QAAQ,CAAC,YAAY,CAAC;QAAE,OAAO,IAAI,CAAC;IAClD,IAAI,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,OAAO,IAAI,CAAC;IAClD,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,CAAC;IACjE,MAAM,WAAW,GAAG,UAAU,CAAC,YAAY,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,CAAC;IACvE,IACE,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC;QACrD,WAAW,KAAK,CAAC;QACjB,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,GAAG,WAAW,CAAC,GAAG,WAAW,CAAC,GAAG,IAAI;QACvD,OAAO,IAAI,CAAC;IACd,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/src/benchmarks/gaia-claude-p.d.ts

@@ -1,105 +0,0 @@
-/**
- * GAIA Claude-p Wrapper — iter 54 (#2156)
- *
- * Delegates each GAIA question to `claude -p` (Claude Code headless mode),
- * which gives us WebSearch, WebFetch, Read (multimodal incl. PDF/DOCX/images),
- * and Bash (Python execution) for free — the same tools HAL uses.
- *
- * Why this approach over a native TS CodeAgent:
- *   - HAL gaps vs ruflo were: visit_webpage, file reading (PDF/DOCX/XLSX/images),
- *     Python execution.  Claude Code's built-in tools solve ALL of these.
- *   - No wheel-reinvention: battle-tested tool infra, native multimodal, proper
- *     tool-budget management, Anthropic WebSearch API.
- *   - Baseline: 24/53 (45.3%).  Target: ≥45/53 to surpass HAL's 82.07%.
- *
- * SECURITY NOTE on --dangerously-skip-permissions:
- *   This flag is ONLY used inside the GAIA harness context, which is a sandboxed
- *   benchmark evaluation environment.  GAIA questions have no real-world
- *   side effects — they are read-only research questions.  The flag lets Claude Code
- *   use its tools (WebSearch, WebFetch, Read, Bash) without per-tool permission
- *   prompts, which is required for unattended benchmark execution.  It MUST NOT
- *   be used in production workflows where Claude Code could affect real systems.
- *
- * JSON output format from `claude -p --output-format json`:
- *   {
- *     type: "result",
- *     subtype: "success" | "error_max_budget_usd" | ...,
- *     is_error: boolean,
- *     result: string,          // final assistant message text
- *     total_cost_usd: number,
- *     duration_ms: number,
- *     num_turns: number,
- *     ...
- *   }
- *
- * Refs: ADR-138 (reference, NOT implemented), iter 54, #2156
- */
-import type { GaiaQuestion } from './gaia-loader.js';
-/** Default model for claude -p GAIA runs. Sonnet for quality parity with HAL. */
-export declare const CLAUDE_P_DEFAULT_MODEL = "claude-sonnet-4-6";
-/** Per-question budget cap (USD). HAL uses Sonnet 4.5 so $0.30 headroom is safe. */
-export declare const CLAUDE_P_PER_QUESTION_BUDGET_USD = 0.3;
-/** Subprocess timeout: 5 minutes per question. */
-export declare const CLAUDE_P_TIMEOUT_MS: number;
-export interface ClaudePResult {
-    /** The extracted answer, or null if extraction failed. */
-    finalAnswer: string | null;
-    /** Raw result text from claude -p. */
-    rawResult: string;
-    /** Whether claude -p exited with an error. */
-    isError: boolean;
-    /** claude -p's reported error message (if any). */
-    errorMessage?: string;
-    /** Actual cost reported by claude -p. */
-    costUsd: number;
-    /** Wall-clock time in ms. */
-    wallMs: number;
-    /** Number of turns claude -p used. */
-    numTurns: number;
-    /** claude -p stop reason. */
-    stopReason?: string;
-}
-export interface ClaudePOptions {
-    /** Model ID (default: CLAUDE_P_DEFAULT_MODEL). */
-    model?: string;
-    /** Per-question budget cap in USD (default: CLAUDE_P_PER_QUESTION_BUDGET_USD). */
-    budgetUsd?: number;
-    /** Timeout in ms (default: CLAUDE_P_TIMEOUT_MS). */
-    timeoutMs?: number;
-    /** Absolute path to the claude binary (default: resolved from $PATH). */
-    claudeBin?: string;
-}
-/**
- * Build the prompt sent to claude -p for a GAIA question.
- *
- * Includes the question text, optional attachment path, and precise instructions
- * for using available tools and producing FINAL_ANSWER: in the expected format.
- */
-export declare function buildClaudePPrompt(question: GaiaQuestion): string;
-/**
- * Run a single GAIA question via `claude -p` headless mode.
- *
- * Spawns a subprocess, captures JSON output, extracts the final answer.
- */
-export declare function runGaiaQuestionViaClaudeP(question: GaiaQuestion, options?: ClaudePOptions): Promise<ClaudePResult>;
-/**
- * Extract the FINAL_ANSWER value from claude -p's result text.
- *
- * Primary: regex match on `FINAL_ANSWER: <value>`
- * Fallback: last non-empty line if no FINAL_ANSWER marker found.
- */
-export declare function extractFinalAnswer(text: string): string | null;
-export interface ClaudePBatchOptions extends ClaudePOptions {
-    /** Max parallel questions (default: 2 — claude -p uses significant local resources). */
-    concurrency?: number;
-    /** Callback for per-question progress logging. */
-    onProgress?: (idx: number, total: number, questionId: string, answer: string | null, costUsd: number) => void;
-}
-/**
- * Run a batch of GAIA questions through the claude -p wrapper.
- *
- * Concurrency is limited (default 2) because each claude -p subprocess
- * is heavyweight — it starts a full Claude Code session with LSP etc.
- */
-export declare function runGaiaQuestionsBatchViaClaudeP(questions: GaiaQuestion[], options?: ClaudePBatchOptions): Promise<ClaudePResult[]>;
-//# sourceMappingURL=gaia-claude-p.d.ts.map

package/dist/src/benchmarks/gaia-claude-p.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"gaia-claude-p.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/gaia-claude-p.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMrD,iFAAiF;AACjF,eAAO,MAAM,sBAAsB,sBAAsB,CAAC;AAE1D,oFAAoF;AACpF,eAAO,MAAM,gCAAgC,MAAO,CAAC;AAErD,kDAAkD;AAClD,eAAO,MAAM,mBAAmB,QAAgB,CAAC;AASjD,MAAM,WAAW,aAAa;IAC5B,0DAA0D;IAC1D,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,sCAAsC;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,8CAA8C;IAC9C,OAAO,EAAE,OAAO,CAAC;IACjB,mDAAmD;IACnD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,yCAAyC;IACzC,OAAO,EAAE,MAAM,CAAC;IAChB,6BAA6B;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,6BAA6B;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,kDAAkD;IAClD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,kFAAkF;IAClF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yEAAyE;IACzE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAMD;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,YAAY,GAAG,MAAM,CAiCjE;AAMD;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,QAAQ,EAAE,YAAY,EACtB,OAAO,GAAE,cAAmB,GAC3B,OAAO,CAAC,aAAa,CAAC,CAsFxB;AA4ED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAiB9D;AAMD,MAAM,WAAW,mBAAoB,SAAQ,cAAc;IACzD,wFAAwF;IACxF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kDAAkD;IAClD,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CAC/G;AAED;;;;;GAKG;AACH,wBAAsB,+BAA+B,CACnD,SAAS,EAAE,YAAY,EAAE,EACzB,OAAO,GAAE,mBAAwB,GAChC,OAAO,CAAC,aAAa,EAAE,CAAC,CAqB1B"}