@claude-flow/cli 3.32.1 → 3.32.3

package/dist/src/benchmarks/gaia-agent-planning.smoke.d.ts

@@ -1,18 +0,0 @@
-/**
- * GAIA Agent — Planning Interval Smoke Tests
- *
- * Verifies that the planning-checkpoint injection logic fires at the correct
- * turns and stays silent when conditions are not met.  All tests are fully
- * mocked — no live Anthropic API calls, $0 cost.
- *
- * Test cases:
- *   1. 12-turn loop with interval=4 → replans at turns 4, 8, 12  (3 replans)
- *   2.  3-turn loop with interval=4 → NO replan (interval never hit)
- *   3. end_turn at turn 5          → NO replan injected (terminal state)
- *   4. all-tool_use 8-turn loop    → exactly floor(8/4) = 2 replans
- *   5. planningInterval=0          → disabled entirely, 0 replans
- *
- * Refs: ADR-133, ADR-135, iter 34, #2156
- */
-export {};
-//# sourceMappingURL=gaia-agent-planning.smoke.d.ts.map

package/dist/src/benchmarks/gaia-agent-planning.smoke.js

@@ -1,253 +0,0 @@
-/**
- * GAIA Agent — Planning Interval Smoke Tests
- *
- * Verifies that the planning-checkpoint injection logic fires at the correct
- * turns and stays silent when conditions are not met.  All tests are fully
- * mocked — no live Anthropic API calls, $0 cost.
- *
- * Test cases:
- *   1. 12-turn loop with interval=4 → replans at turns 4, 8, 12  (3 replans)
- *   2.  3-turn loop with interval=4 → NO replan (interval never hit)
- *   3. end_turn at turn 5          → NO replan injected (terminal state)
- *   4. all-tool_use 8-turn loop    → exactly floor(8/4) = 2 replans
- *   5. planningInterval=0          → disabled entirely, 0 replans
- *
- * Refs: ADR-133, ADR-135, iter 34, #2156
- */
-import assert from 'node:assert/strict';
-import { runGaiaAgent, PLANNING_INTERVAL, buildPlanningCheckpoint } from './gaia-agent.js';
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-/** Minimal GAIA question fixture. */
-const FAKE_QUESTION = {
-    task_id: 'smoke-planning-01',
-    question: 'What is 2 + 2?',
-    final_answer: '4',
-    level: 1,
-    file_name: null,
-    file_path: null,
-};
-/** A trivial tool catalogue with one no-op tool. */
-const NOOP_CATALOGUE = [
-    {
-        name: 'noop',
-        definition: {
-            name: 'noop',
-            description: 'Does nothing',
-            input_schema: { type: 'object', properties: {}, required: [] },
-        },
-        execute: async () => 'ok',
-    },
-];
-// ---------------------------------------------------------------------------
-// Mock Anthropic API call factory
-// ---------------------------------------------------------------------------
-/**
- * Build a mock `fetch` that simulates a specific turn sequence.
- *
- * `sequence` is an array of stop_reasons.  The mock returns `tool_use`
- * responses (with one `noop` tool_use block) for `tool_use` entries, and
- * `end_turn` with a FINAL_ANSWER text block for `end_turn` entries.
- *
- * The mock replaces `global.fetch` for the duration of the test and is
- * restored automatically via the returned `restore` function.
- */
-function mockFetchSequence(sequence) {
-    const calls = []; // turn indices (0-based) of actual calls
-    let callIdx = 0;
-    const originalFetch = global.fetch;
-    global.fetch = async (_url, options) => {
-        const idx = callIdx++;
-        calls.push(idx);
-        const stopReason = sequence[idx] ?? 'end_turn';
-        let content;
-        if (stopReason === 'tool_use') {
-            content = [
-                {
-                    type: 'tool_use',
-                    id: `tool-${idx}`,
-                    name: 'noop',
-                    input: {},
-                },
-            ];
-        }
-        else {
-            content = [
-                {
-                    type: 'text',
-                    text: 'FINAL_ANSWER: 4',
-                },
-            ];
-        }
-        const body = JSON.stringify({
-            id: `msg-${idx}`,
-            model: 'claude-haiku-4-5',
-            stop_reason: stopReason,
-            content,
-            usage: { input_tokens: 100, output_tokens: 50 },
-        });
-        return new Response(body, {
-            status: 200,
-            headers: { 'content-type': 'application/json' },
-        });
-    };
-    return {
-        calls,
-        restore: () => {
-            global.fetch = originalFetch;
-        },
-    };
-}
-// ---------------------------------------------------------------------------
-// Base options for all tests
-// ---------------------------------------------------------------------------
-function baseOpts(overrides = {}) {
-    return {
-        apiKey: 'sk-ant-test-key',
-        catalogue: NOOP_CATALOGUE,
-        model: 'claude-haiku-4-5',
-        perTurnTimeoutMs: 5_000,
-        ...overrides,
-    };
-}
-// ---------------------------------------------------------------------------
-// Test 1: 12-turn all-tool_use loop → 3 replans at turns 4, 8, 12
-// ---------------------------------------------------------------------------
-async function test_twelveToolUseTurns_threeReplans() {
-    // Turns 0-11 are tool_use, turn 12 (index 12) is end_turn
-    const sequence = [
-        ...Array(12).fill('tool_use'),
-        'end_turn',
-    ];
-    const { restore } = mockFetchSequence(sequence);
-    try {
-        const result = await runGaiaAgent(FAKE_QUESTION, baseOpts({ maxTurns: 13 }));
-        assert.equal(result.replanCount, 3, `Expected 3 replans, got ${result.replanCount}`);
-    }
-    finally {
-        restore();
-    }
-    console.log('  PASS test_twelveToolUseTurns_threeReplans');
-}
-// ---------------------------------------------------------------------------
-// Test 2: 3-turn loop → 0 replans (interval=4 never hit)
-// ---------------------------------------------------------------------------
-async function test_threeToolUseTurns_zeroReplans() {
-    const sequence = [
-        'tool_use',
-        'tool_use',
-        'tool_use',
-        'end_turn',
-    ];
-    const { restore } = mockFetchSequence(sequence);
-    try {
-        const result = await runGaiaAgent(FAKE_QUESTION, baseOpts({ maxTurns: 8 }));
-        assert.equal(result.replanCount, 0, `Expected 0 replans, got ${result.replanCount}`);
-    }
-    finally {
-        restore();
-    }
-    console.log('  PASS test_threeToolUseTurns_zeroReplans');
-}
-// ---------------------------------------------------------------------------
-// Test 3: end_turn at turn 5 → 1 replan (at turn 4), not at turn 5
-// ---------------------------------------------------------------------------
-async function test_endTurnAtFive_oneReplan() {
-    // Turns 0-4 are tool_use (5 of them), turn 5 is end_turn
-    // Replan fires at turn 4 (turns=4, turns%4===0), but NOT at turn 5 (end_turn → no injection)
-    const sequence = [
-        'tool_use',
-        'tool_use',
-        'tool_use',
-        'tool_use',
-        'tool_use',
-        'end_turn',
-    ];
-    const { restore } = mockFetchSequence(sequence);
-    try {
-        const result = await runGaiaAgent(FAKE_QUESTION, baseOpts({ maxTurns: 10 }));
-        assert.equal(result.replanCount, 1, `Expected 1 replan (at turn 4), got ${result.replanCount}`);
-        assert.equal(result.timedOut, undefined, 'Should not time out');
-    }
-    finally {
-        restore();
-    }
-    console.log('  PASS test_endTurnAtFive_oneReplan');
-}
-// ---------------------------------------------------------------------------
-// Test 4: all tool_use for 8 turns → floor(8/4) = 2 replans
-// ---------------------------------------------------------------------------
-async function test_eightToolUseTurns_twoReplans() {
-    const sequence = [
-        ...Array(8).fill('tool_use'),
-        'end_turn',
-    ];
-    const { restore } = mockFetchSequence(sequence);
-    try {
-        const result = await runGaiaAgent(FAKE_QUESTION, baseOpts({ maxTurns: 9 }));
-        assert.equal(result.replanCount, 2, `Expected 2 replans, got ${result.replanCount}`);
-    }
-    finally {
-        restore();
-    }
-    console.log('  PASS test_eightToolUseTurns_twoReplans');
-}
-// ---------------------------------------------------------------------------
-// Test 5: planningInterval=0 → disabled, 0 replans even on long runs
-// ---------------------------------------------------------------------------
-async function test_intervalZero_disabledReplanning() {
-    const sequence = [
-        ...Array(8).fill('tool_use'),
-        'end_turn',
-    ];
-    const { restore } = mockFetchSequence(sequence);
-    try {
-        const result = await runGaiaAgent(FAKE_QUESTION, baseOpts({ maxTurns: 9, planningInterval: 0 }));
-        assert.equal(result.replanCount, 0, `Expected 0 replans with interval=0, got ${result.replanCount}`);
-    }
-    finally {
-        restore();
-    }
-    console.log('  PASS test_intervalZero_disabledReplanning');
-}
-// ---------------------------------------------------------------------------
-// Test 6: buildPlanningCheckpoint content smoke
-// ---------------------------------------------------------------------------
-function test_buildPlanningCheckpoint_content() {
-    const text = buildPlanningCheckpoint(4, 12);
-    assert.ok(text.includes('[PLANNING CHECKPOINT'), 'Missing checkpoint header');
-    assert.ok(text.includes('turn 4/12'), 'Missing turn counter');
-    assert.ok(text.includes('4 turns so far'), 'Missing turn count phrase');
-    assert.ok(text.includes('switch strategy'), 'Missing strategy-switch instruction');
-    assert.ok(text.includes('FINAL_ANSWER'), 'Missing FINAL_ANSWER instruction');
-    console.log('  PASS test_buildPlanningCheckpoint_content');
-}
-// ---------------------------------------------------------------------------
-// Test 7: PLANNING_INTERVAL constant equals 4
-// ---------------------------------------------------------------------------
-function test_planningInterval_constant() {
-    assert.equal(PLANNING_INTERVAL, 4, `Expected PLANNING_INTERVAL=4, got ${PLANNING_INTERVAL}`);
-    console.log('  PASS test_planningInterval_constant');
-}
-// ---------------------------------------------------------------------------
-// Runner
-// ---------------------------------------------------------------------------
-async function main() {
-    console.log('\n=== GAIA Agent Planning Interval Smoke Tests ===\n');
-    // Synchronous tests first
-    test_buildPlanningCheckpoint_content();
-    test_planningInterval_constant();
-    // Async tests
-    await test_twelveToolUseTurns_threeReplans();
-    await test_threeToolUseTurns_zeroReplans();
-    await test_endTurnAtFive_oneReplan();
-    await test_eightToolUseTurns_twoReplans();
-    await test_intervalZero_disabledReplanning();
-    console.log('\n=== All planning-interval smoke tests PASSED ===\n');
-}
-main().catch((err) => {
-    console.error('Planning interval smoke failed:', err);
-    process.exit(1);
-});
-//# sourceMappingURL=gaia-agent-planning.smoke.js.map

package/dist/src/benchmarks/gaia-agent.d.ts

@@ -1,198 +0,0 @@
-/**
- * GAIA Agent — ADR-133-PR3 / ADR-135 (planning interval)
- *
- * Multi-turn Anthropic Messages API loop that drives Claude through the
- * GAIA benchmark questions using a tool-use agent pattern.
- *
- * Loop algorithm:
- *   1. Build initial message with the question and a system prompt that
- *      instructs Claude to output `FINAL_ANSWER: <value>` when done.
- *   2. Call Anthropic Messages API with the registered tool definitions.
- *   3. On `stop_reason === 'tool_use'`: execute all tool_use blocks in
- *      parallel, append results as a `user` turn, and repeat.
- *      Every PLANNING_INTERVAL turns, inject a planning-checkpoint text
- *      alongside the tool results to force strategy re-evaluation.
- *   4. On `stop_reason === 'end_turn'`: scan content for the final answer
- *      pattern and return the result.
- *   5. On timeout (maxTurns exceeded): return `{ timedOut: true }`.
- *
- * API key resolution order (mirrors resolveHfToken from gaia-loader.ts):
- *   1. `options.apiKey` (caller-supplied)
- *   2. `ANTHROPIC_API_KEY` env var
- *   3. `gcloud secrets versions access latest --secret=ANTHROPIC_API_KEY`
- *
- * Cost discipline: smoke runs use `claude-haiku-4-5` only.  The smoke
- * runner at the bottom of this file enforces that model.
- *
- * Planning interval (iter 30 finding #3):
- *   smolagents CodeAgent uses planning_interval=4 — replans every 4 steps
- *   to prevent tunnel-vision on bad strategies. Adds ~80 tokens per
- *   replan event (~$0.0001 each), negligible cost.
- *
- * Iter 53a T2 narrowing:
- *   Three precise changes from iter 52 T2 (which had net -1q: +6 recoveries, -7 regressions):
- *   1. extractFinalAnswer uses Stage 1 only (no Stage 2/3 prose fallback).
- *      Stage 2/3 fired too aggressively: overwriting correct Stage 1 answers and
- *      extracting wrong prose fragments. Now Stage 1 is the only extraction path.
- *   2. System prompt removes surrender instruction ("FINAL_ANSWER: unknown / I don't know").
- *      That instruction caused the agent to give up on questions it would have figured out.
- *      Replaced with: "When you reach a final answer, output FINAL_ANSWER: <value>."
- *   3. Reversed-text preprocessor is preserved (iter 52 T2 finding: 2d83110e has reversed text).
- *
- * Refs: ADR-133, ADR-135, iter 30, iter 52, iter 53a, #2156
- */
-import { GaiaQuestion } from './gaia-loader.js';
-import { GaiaToolCatalogue, ContentBlock } from './gaia-tools/index.js';
-/**
- * Every PLANNING_INTERVAL tool_use turns, inject a planning-checkpoint
- * message to force the agent to reassess its strategy.
- *
- * Based on iter 30 research: smolagents CodeAgent uses planning_interval=4.
- * HAL reliability analysis showed agents fail when they exhaust step
- * budgets without recalibrating.
- */
-export declare const PLANNING_INTERVAL = 4;
-/**
- * Build the planning-checkpoint text injected every PLANNING_INTERVAL turns.
- * Exported so tests can snapshot the exact wording.
- */
-export declare function buildPlanningCheckpoint(turn: number, maxTurns: number): string;
-export interface GaiaAgentResult {
-    questionId: string;
-    finalAnswer: string | null;
-    turns: number;
-    toolCallsByName: Record<string, number>;
-    totalInputTokens: number;
-    totalOutputTokens: number;
-    wallMs: number;
-    /** Number of planning-checkpoint injections during this run (0 when planning is disabled). */
-    replanCount?: number;
-    timedOut?: boolean;
-    /** Set when the convergence layer fired and committed the final answer. */
-    convergenceTrigger?: string;
-    /** True when the convergence layer recovered the answer from prior message history. */
-    convergenceUsedFallback?: boolean;
-    error?: string;
-}
-export interface GaiaAgentOptions {
-    /** Model to use (default: 'claude-haiku-4-5'). */
-    model?: string;
-    /** Maximum number of agent turns before giving up (default: 8). */
-    maxTurns?: number;
-    /** Maximum tokens per Anthropic API call (default: 2048). */
-    maxTokensPerTurn?: number;
-    /** Per-turn HTTP timeout in milliseconds (default: 60 000). */
-    perTurnTimeoutMs?: number;
-    /**
-     * Inject a planning-checkpoint every N tool_use turns (default: PLANNING_INTERVAL = 4).
-     * Set to 0 to disable planning checkpoints.
-     */
-    planningInterval?: number;
-    /**
-     * Anthropic API key.  Resolved automatically via env var + gcloud fallback
-     * if omitted.
-     */
-    apiKey?: string;
-    /**
-     * Pre-built tool catalogue.  Defaults to `createDefaultToolCatalogue()`.
-     * Exposed so callers can inject mocks for testing.
-     */
-    catalogue?: GaiaToolCatalogue;
-    /**
-     * Enable the convergence layer (default: true).
-     *
-     * When enabled, the convergence layer monitors for three failure modes:
-     *   1. max_turns hit without FINAL_ANSWER
-     *   2. Loop (same tool+args 3× in a 5-turn window)
-     *   3. Token overflow (>120k input tokens)
-     *
-     * On detection, a forced-commit phase is run: one API call with a
-     * directive prompt, no tools, then a fallback scan of prior messages.
-     * Set to false to disable (e.g. for ablation testing).
-     */
-    enableConvergence?: boolean;
-}
-/**
- * Resolve the Anthropic API key.
- *
- * Resolution order:
- *   1. Caller-supplied `apiKey`
- *   2. `ANTHROPIC_API_KEY` env var
- *   3. `gcloud secrets versions access latest --secret=ANTHROPIC_API_KEY`
- *
- * Throws with a clear message if none of the above is available.
- */
-export declare function resolveAnthropicApiKey(apiKey?: string): string;
-/**
- * If the question text appears to be reversed English, prepend a de-reversed
- * version so the agent sees both the original and the decoded form.
- *
- * Iter 52 T2 — gate 1 finding: task 2d83110e has a reversed sentence.
- * Kept in iter 53a (this is not the source of the iter 52 regressions).
- */
-declare function buildUserMessage(question: string): string;
-/** Anthropic image content block for vision API. */
-interface ImageContentBlock {
-    type: 'image';
-    source: {
-        type: 'base64';
-        media_type: string;
-        data: string;
-    };
-}
-/**
- * Parse an IMAGE_BASE64 marker returned by file_read's extractImage().
- * Returns an Anthropic image content block, or null if the marker is invalid.
- *
- * Marker format: [IMAGE_BASE64:{"mediaType":"image/png","base64":"...","path":"..."}]
- */
-export declare function parseImageMarker(marker: string): ImageContentBlock | null;
-/** Minimal types for the Anthropic Messages API response. */
-interface AnthropicResponse {
-    id: string;
-    model: string;
-    stop_reason: 'end_turn' | 'tool_use' | 'max_tokens' | string;
-    content: ContentBlock[];
-    usage: {
-        input_tokens: number;
-        output_tokens: number;
-    };
-}
-declare function extractFinalAnswer(resp: AnthropicResponse): string | null;
-/**
- * Run a GAIA question through Claude with tool use.
- *
- * @returns GaiaAgentResult with the final answer (or null if timed out),
- * turn count, token totals, and per-tool call counts.
- */
-export declare function runGaiaAgent(question: GaiaQuestion, options?: GaiaAgentOptions): Promise<GaiaAgentResult>;
-/**
- * Check whether a model answer matches the expected ground-truth answer.
- *
- * Matching rules (mirrors GAIA evaluation):
- * - Normalise: trim whitespace, lowercase.
- * - Substring match: expected is contained in model answer (handles "Paris" vs "Paris, France").
- * - Direct equality after normalisation.
- * - Numeric: parse as floats and compare with ±1% tolerance.
- */
-export declare function isAnswerCorrect(modelAnswer: string, expected: string): boolean;
-/**
- * Run all 5 SMOKE_FIXTURE questions and report results to stdout.
- *
- * Pass criteria: ≥3/5 correct (60% pass rate).
- *
- * Cost estimate is printed at the end using Haiku pricing.
- *
- * This function is exported so tests can call it directly and capture output;
- * it also runs when this file is executed directly via `node gaia-agent.js --smoke`.
- */
-export declare function runSmokeTest(opts?: {
-    verbose?: boolean;
-    apiKey?: string;
-}): Promise<{
-    passRate: number;
-    passed: number;
-    total: number;
-}>;
-export { extractFinalAnswer as _extractFinalAnswerForTest, buildUserMessage as _buildUserMessageForTest, };
-//# sourceMappingURL=gaia-agent.d.ts.map