claude-flow 3.10.4 → 3.10.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-flow",
-  "version": "3.10.4",
+  "version": "3.10.5",
   "description": "Ruflo - Enterprise AI agent orchestration for Claude Code. Deploy 60+ specialized agents in coordinated swarms with self-learning, fault-tolerant consensus, vector memory, and MCP integration",
   "main": "dist/index.js",
   "type": "module",

package/v3/@claude-flow/cli/dist/src/benchmarks/gaia-agent-planning.smoke.d.ts

@@ -0,0 +1,18 @@
+/**
+ * GAIA Agent — Planning Interval Smoke Tests
+ *
+ * Verifies that the planning-checkpoint injection logic fires at the correct
+ * turns and stays silent when conditions are not met.  All tests are fully
+ * mocked — no live Anthropic API calls, $0 cost.
+ *
+ * Test cases:
+ *   1. 12-turn loop with interval=4 → replans at turns 4, 8, 12  (3 replans)
+ *   2.  3-turn loop with interval=4 → NO replan (interval never hit)
+ *   3. end_turn at turn 5          → NO replan injected (terminal state)
+ *   4. all-tool_use 8-turn loop    → exactly floor(8/4) = 2 replans
+ *   5. planningInterval=0          → disabled entirely, 0 replans
+ *
+ * Refs: ADR-133, ADR-135, iter 34, #2156
+ */
+export {};
+//# sourceMappingURL=gaia-agent-planning.smoke.d.ts.map

package/v3/@claude-flow/cli/dist/src/benchmarks/gaia-agent-planning.smoke.js

@@ -0,0 +1,253 @@
+/**
+ * GAIA Agent — Planning Interval Smoke Tests
+ *
+ * Verifies that the planning-checkpoint injection logic fires at the correct
+ * turns and stays silent when conditions are not met.  All tests are fully
+ * mocked — no live Anthropic API calls, $0 cost.
+ *
+ * Test cases:
+ *   1. 12-turn loop with interval=4 → replans at turns 4, 8, 12  (3 replans)
+ *   2.  3-turn loop with interval=4 → NO replan (interval never hit)
+ *   3. end_turn at turn 5          → NO replan injected (terminal state)
+ *   4. all-tool_use 8-turn loop    → exactly floor(8/4) = 2 replans
+ *   5. planningInterval=0          → disabled entirely, 0 replans
+ *
+ * Refs: ADR-133, ADR-135, iter 34, #2156
+ */
+import assert from 'node:assert/strict';
+import { runGaiaAgent, PLANNING_INTERVAL, buildPlanningCheckpoint } from './gaia-agent.js';
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/** Minimal GAIA question fixture. */
+const FAKE_QUESTION = {
+    task_id: 'smoke-planning-01',
+    question: 'What is 2 + 2?',
+    final_answer: '4',
+    level: 1,
+    file_name: null,
+    file_path: null,
+};
+/** A trivial tool catalogue with one no-op tool. */
+const NOOP_CATALOGUE = [
+    {
+        name: 'noop',
+        definition: {
+            name: 'noop',
+            description: 'Does nothing',
+            input_schema: { type: 'object', properties: {}, required: [] },
+        },
+        execute: async () => 'ok',
+    },
+];
+// ---------------------------------------------------------------------------
+// Mock Anthropic API call factory
+// ---------------------------------------------------------------------------
+/**
+ * Build a mock `fetch` that simulates a specific turn sequence.
+ *
+ * `sequence` is an array of stop_reasons.  The mock returns `tool_use`
+ * responses (with one `noop` tool_use block) for `tool_use` entries, and
+ * `end_turn` with a FINAL_ANSWER text block for `end_turn` entries.
+ *
+ * The mock replaces `global.fetch` for the duration of the test and is
+ * restored automatically via the returned `restore` function.
+ */
+function mockFetchSequence(sequence) {
+    const calls = []; // turn indices (0-based) of actual calls
+    let callIdx = 0;
+    const originalFetch = global.fetch;
+    global.fetch = async (_url, options) => {
+        const idx = callIdx++;
+        calls.push(idx);
+        const stopReason = sequence[idx] ?? 'end_turn';
+        let content;
+        if (stopReason === 'tool_use') {
+            content = [
+                {
+                    type: 'tool_use',
+                    id: `tool-${idx}`,
+                    name: 'noop',
+                    input: {},
+                },
+            ];
+        }
+        else {
+            content = [
+                {
+                    type: 'text',
+                    text: 'FINAL_ANSWER: 4',
+                },
+            ];
+        }
+        const body = JSON.stringify({
+            id: `msg-${idx}`,
+            model: 'claude-haiku-4-5',
+            stop_reason: stopReason,
+            content,
+            usage: { input_tokens: 100, output_tokens: 50 },
+        });
+        return new Response(body, {
+            status: 200,
+            headers: { 'content-type': 'application/json' },
+        });
+    };
+    return {
+        calls,
+        restore: () => {
+            global.fetch = originalFetch;
+        },
+    };
+}
+// ---------------------------------------------------------------------------
+// Base options for all tests
+// ---------------------------------------------------------------------------
+function baseOpts(overrides = {}) {
+    return {
+        apiKey: 'sk-ant-test-key',
+        catalogue: NOOP_CATALOGUE,
+        model: 'claude-haiku-4-5',
+        perTurnTimeoutMs: 5_000,
+        ...overrides,
+    };
+}
+// ---------------------------------------------------------------------------
+// Test 1: 12-turn all-tool_use loop → 3 replans at turns 4, 8, 12
+// ---------------------------------------------------------------------------
+async function test_twelveToolUseTurns_threeReplans() {
+    // Turns 0-11 are tool_use, turn 12 (index 12) is end_turn
+    const sequence = [
+        ...Array(12).fill('tool_use'),
+        'end_turn',
+    ];
+    const { restore } = mockFetchSequence(sequence);
+    try {
+        const result = await runGaiaAgent(FAKE_QUESTION, baseOpts({ maxTurns: 13 }));
+        assert.equal(result.replanCount, 3, `Expected 3 replans, got ${result.replanCount}`);
+    }
+    finally {
+        restore();
+    }
+    console.log('  PASS test_twelveToolUseTurns_threeReplans');
+}
+// ---------------------------------------------------------------------------
+// Test 2: 3-turn loop → 0 replans (interval=4 never hit)
+// ---------------------------------------------------------------------------
+async function test_threeToolUseTurns_zeroReplans() {
+    const sequence = [
+        'tool_use',
+        'tool_use',
+        'tool_use',
+        'end_turn',
+    ];
+    const { restore } = mockFetchSequence(sequence);
+    try {
+        const result = await runGaiaAgent(FAKE_QUESTION, baseOpts({ maxTurns: 8 }));
+        assert.equal(result.replanCount, 0, `Expected 0 replans, got ${result.replanCount}`);
+    }
+    finally {
+        restore();
+    }
+    console.log('  PASS test_threeToolUseTurns_zeroReplans');
+}
+// ---------------------------------------------------------------------------
+// Test 3: end_turn at turn 5 → 1 replan (at turn 4), not at turn 5
+// ---------------------------------------------------------------------------
+async function test_endTurnAtFive_oneReplan() {
+    // Turns 0-4 are tool_use (5 of them), turn 5 is end_turn
+    // Replan fires at turn 4 (turns=4, turns%4===0), but NOT at turn 5 (end_turn → no injection)
+    const sequence = [
+        'tool_use',
+        'tool_use',
+        'tool_use',
+        'tool_use',
+        'tool_use',
+        'end_turn',
+    ];
+    const { restore } = mockFetchSequence(sequence);
+    try {
+        const result = await runGaiaAgent(FAKE_QUESTION, baseOpts({ maxTurns: 10 }));
+        assert.equal(result.replanCount, 1, `Expected 1 replan (at turn 4), got ${result.replanCount}`);
+        assert.equal(result.timedOut, undefined, 'Should not time out');
+    }
+    finally {
+        restore();
+    }
+    console.log('  PASS test_endTurnAtFive_oneReplan');
+}
+// ---------------------------------------------------------------------------
+// Test 4: all tool_use for 8 turns → floor(8/4) = 2 replans
+// ---------------------------------------------------------------------------
+async function test_eightToolUseTurns_twoReplans() {
+    const sequence = [
+        ...Array(8).fill('tool_use'),
+        'end_turn',
+    ];
+    const { restore } = mockFetchSequence(sequence);
+    try {
+        const result = await runGaiaAgent(FAKE_QUESTION, baseOpts({ maxTurns: 9 }));
+        assert.equal(result.replanCount, 2, `Expected 2 replans, got ${result.replanCount}`);
+    }
+    finally {
+        restore();
+    }
+    console.log('  PASS test_eightToolUseTurns_twoReplans');
+}
+// ---------------------------------------------------------------------------
+// Test 5: planningInterval=0 → disabled, 0 replans even on long runs
+// ---------------------------------------------------------------------------
+async function test_intervalZero_disabledReplanning() {
+    const sequence = [
+        ...Array(8).fill('tool_use'),
+        'end_turn',
+    ];
+    const { restore } = mockFetchSequence(sequence);
+    try {
+        const result = await runGaiaAgent(FAKE_QUESTION, baseOpts({ maxTurns: 9, planningInterval: 0 }));
+        assert.equal(result.replanCount, 0, `Expected 0 replans with interval=0, got ${result.replanCount}`);
+    }
+    finally {
+        restore();
+    }
+    console.log('  PASS test_intervalZero_disabledReplanning');
+}
+// ---------------------------------------------------------------------------
+// Test 6: buildPlanningCheckpoint content smoke
+// ---------------------------------------------------------------------------
+function test_buildPlanningCheckpoint_content() {
+    const text = buildPlanningCheckpoint(4, 12);
+    assert.ok(text.includes('[PLANNING CHECKPOINT'), 'Missing checkpoint header');
+    assert.ok(text.includes('turn 4/12'), 'Missing turn counter');
+    assert.ok(text.includes('4 turns so far'), 'Missing turn count phrase');
+    assert.ok(text.includes('switch strategy'), 'Missing strategy-switch instruction');
+    assert.ok(text.includes('FINAL_ANSWER'), 'Missing FINAL_ANSWER instruction');
+    console.log('  PASS test_buildPlanningCheckpoint_content');
+}
+// ---------------------------------------------------------------------------
+// Test 7: PLANNING_INTERVAL constant equals 4
+// ---------------------------------------------------------------------------
+function test_planningInterval_constant() {
+    assert.equal(PLANNING_INTERVAL, 4, `Expected PLANNING_INTERVAL=4, got ${PLANNING_INTERVAL}`);
+    console.log('  PASS test_planningInterval_constant');
+}
+// ---------------------------------------------------------------------------
+// Runner
+// ---------------------------------------------------------------------------
+async function main() {
+    console.log('\n=== GAIA Agent Planning Interval Smoke Tests ===\n');
+    // Synchronous tests first
+    test_buildPlanningCheckpoint_content();
+    test_planningInterval_constant();
+    // Async tests
+    await test_twelveToolUseTurns_threeReplans();
+    await test_threeToolUseTurns_zeroReplans();
+    await test_endTurnAtFive_oneReplan();
+    await test_eightToolUseTurns_twoReplans();
+    await test_intervalZero_disabledReplanning();
+    console.log('\n=== All planning-interval smoke tests PASSED ===\n');
+}
+main().catch((err) => {
+    console.error('Planning interval smoke failed:', err);
+    process.exit(1);
+});
+//# sourceMappingURL=gaia-agent-planning.smoke.js.map

package/v3/@claude-flow/cli/dist/src/benchmarks/gaia-agent.d.ts

@@ -0,0 +1,198 @@
+/**
+ * GAIA Agent — ADR-133-PR3 / ADR-135 (planning interval)
+ *
+ * Multi-turn Anthropic Messages API loop that drives Claude through the
+ * GAIA benchmark questions using a tool-use agent pattern.
+ *
+ * Loop algorithm:
+ *   1. Build initial message with the question and a system prompt that
+ *      instructs Claude to output `FINAL_ANSWER: <value>` when done.
+ *   2. Call Anthropic Messages API with the registered tool definitions.
+ *   3. On `stop_reason === 'tool_use'`: execute all tool_use blocks in
+ *      parallel, append results as a `user` turn, and repeat.
+ *      Every PLANNING_INTERVAL turns, inject a planning-checkpoint text
+ *      alongside the tool results to force strategy re-evaluation.
+ *   4. On `stop_reason === 'end_turn'`: scan content for the final answer
+ *      pattern and return the result.
+ *   5. On timeout (maxTurns exceeded): return `{ timedOut: true }`.
+ *
+ * API key resolution order (mirrors resolveHfToken from gaia-loader.ts):
+ *   1. `options.apiKey` (caller-supplied)
+ *   2. `ANTHROPIC_API_KEY` env var
+ *   3. `gcloud secrets versions access latest --secret=ANTHROPIC_API_KEY`
+ *
+ * Cost discipline: smoke runs use `claude-haiku-4-5` only.  The smoke
+ * runner at the bottom of this file enforces that model.
+ *
+ * Planning interval (iter 30 finding #3):
+ *   smolagents CodeAgent uses planning_interval=4 — replans every 4 steps
+ *   to prevent tunnel-vision on bad strategies. Adds ~80 tokens per
+ *   replan event (~$0.0001 each), negligible cost.
+ *
+ * Iter 53a T2 narrowing:
+ *   Three precise changes from iter 52 T2 (which had net -1q: +6 recoveries, -7 regressions):
+ *   1. extractFinalAnswer uses Stage 1 only (no Stage 2/3 prose fallback).
+ *      Stage 2/3 fired too aggressively: overwriting correct Stage 1 answers and
+ *      extracting wrong prose fragments. Now Stage 1 is the only extraction path.
+ *   2. System prompt removes surrender instruction ("FINAL_ANSWER: unknown / I don't know").
+ *      That instruction caused the agent to give up on questions it would have figured out.
+ *      Replaced with: "When you reach a final answer, output FINAL_ANSWER: <value>."
+ *   3. Reversed-text preprocessor is preserved (iter 52 T2 finding: 2d83110e has reversed text).
+ *
+ * Refs: ADR-133, ADR-135, iter 30, iter 52, iter 53a, #2156
+ */
+import { GaiaQuestion } from './gaia-loader.js';
+import { GaiaToolCatalogue, ContentBlock } from './gaia-tools/index.js';
+/**
+ * Every PLANNING_INTERVAL tool_use turns, inject a planning-checkpoint
+ * message to force the agent to reassess its strategy.
+ *
+ * Based on iter 30 research: smolagents CodeAgent uses planning_interval=4.
+ * HAL reliability analysis showed agents fail when they exhaust step
+ * budgets without recalibrating.
+ */
+export declare const PLANNING_INTERVAL = 4;
+/**
+ * Build the planning-checkpoint text injected every PLANNING_INTERVAL turns.
+ * Exported so tests can snapshot the exact wording.
+ */
+export declare function buildPlanningCheckpoint(turn: number, maxTurns: number): string;
+export interface GaiaAgentResult {
+    questionId: string;
+    finalAnswer: string | null;
+    turns: number;
+    toolCallsByName: Record<string, number>;
+    totalInputTokens: number;
+    totalOutputTokens: number;
+    wallMs: number;
+    /** Number of planning-checkpoint injections during this run (0 when planning is disabled). */
+    replanCount?: number;
+    timedOut?: boolean;
+    /** Set when the convergence layer fired and committed the final answer. */
+    convergenceTrigger?: string;
+    /** True when the convergence layer recovered the answer from prior message history. */
+    convergenceUsedFallback?: boolean;
+    error?: string;
+}
+export interface GaiaAgentOptions {
+    /** Model to use (default: 'claude-haiku-4-5'). */
+    model?: string;
+    /** Maximum number of agent turns before giving up (default: 8). */
+    maxTurns?: number;
+    /** Maximum tokens per Anthropic API call (default: 2048). */
+    maxTokensPerTurn?: number;
+    /** Per-turn HTTP timeout in milliseconds (default: 60 000). */
+    perTurnTimeoutMs?: number;
+    /**
+     * Inject a planning-checkpoint every N tool_use turns (default: PLANNING_INTERVAL = 4).
+     * Set to 0 to disable planning checkpoints.
+     */
+    planningInterval?: number;
+    /**
+     * Anthropic API key.  Resolved automatically via env var + gcloud fallback
+     * if omitted.
+     */
+    apiKey?: string;
+    /**
+     * Pre-built tool catalogue.  Defaults to `createDefaultToolCatalogue()`.
+     * Exposed so callers can inject mocks for testing.
+     */
+    catalogue?: GaiaToolCatalogue;
+    /**
+     * Enable the convergence layer (default: true).
+     *
+     * When enabled, the convergence layer monitors for three failure modes:
+     *   1. max_turns hit without FINAL_ANSWER
+     *   2. Loop (same tool+args 3× in a 5-turn window)
+     *   3. Token overflow (>120k input tokens)
+     *
+     * On detection, a forced-commit phase is run: one API call with a
+     * directive prompt, no tools, then a fallback scan of prior messages.
+     * Set to false to disable (e.g. for ablation testing).
+     */
+    enableConvergence?: boolean;
+}
+/**
+ * Resolve the Anthropic API key.
+ *
+ * Resolution order:
+ *   1. Caller-supplied `apiKey`
+ *   2. `ANTHROPIC_API_KEY` env var
+ *   3. `gcloud secrets versions access latest --secret=ANTHROPIC_API_KEY`
+ *
+ * Throws with a clear message if none of the above is available.
+ */
+export declare function resolveAnthropicApiKey(apiKey?: string): string;
+/**
+ * If the question text appears to be reversed English, prepend a de-reversed
+ * version so the agent sees both the original and the decoded form.
+ *
+ * Iter 52 T2 — gate 1 finding: task 2d83110e has a reversed sentence.
+ * Kept in iter 53a (this is not the source of the iter 52 regressions).
+ */
+declare function buildUserMessage(question: string): string;
+/** Anthropic image content block for vision API. */
+interface ImageContentBlock {
+    type: 'image';
+    source: {
+        type: 'base64';
+        media_type: string;
+        data: string;
+    };
+}
+/**
+ * Parse an IMAGE_BASE64 marker returned by file_read's extractImage().
+ * Returns an Anthropic image content block, or null if the marker is invalid.
+ *
+ * Marker format: [IMAGE_BASE64:{"mediaType":"image/png","base64":"...","path":"..."}]
+ */
+export declare function parseImageMarker(marker: string): ImageContentBlock | null;
+/** Minimal types for the Anthropic Messages API response. */
+interface AnthropicResponse {
+    id: string;
+    model: string;
+    stop_reason: 'end_turn' | 'tool_use' | 'max_tokens' | string;
+    content: ContentBlock[];
+    usage: {
+        input_tokens: number;
+        output_tokens: number;
+    };
+}
+declare function extractFinalAnswer(resp: AnthropicResponse): string | null;
+/**
+ * Run a GAIA question through Claude with tool use.
+ *
+ * @returns GaiaAgentResult with the final answer (or null if timed out),
+ * turn count, token totals, and per-tool call counts.
+ */
+export declare function runGaiaAgent(question: GaiaQuestion, options?: GaiaAgentOptions): Promise<GaiaAgentResult>;
+/**
+ * Check whether a model answer matches the expected ground-truth answer.
+ *
+ * Matching rules (mirrors GAIA evaluation):
+ * - Normalise: trim whitespace, lowercase.
+ * - Substring match: expected is contained in model answer (handles "Paris" vs "Paris, France").
+ * - Direct equality after normalisation.
+ * - Numeric: parse as floats and compare with ±1% tolerance.
+ */
+export declare function isAnswerCorrect(modelAnswer: string, expected: string): boolean;
+/**
+ * Run all 5 SMOKE_FIXTURE questions and report results to stdout.
+ *
+ * Pass criteria: ≥3/5 correct (60% pass rate).
+ *
+ * Cost estimate is printed at the end using Haiku pricing.
+ *
+ * This function is exported so tests can call it directly and capture output;
+ * it also runs when this file is executed directly via `node gaia-agent.js --smoke`.
+ */
+export declare function runSmokeTest(opts?: {
+    verbose?: boolean;
+    apiKey?: string;
+}): Promise<{
+    passRate: number;
+    passed: number;
+    total: number;
+}>;
+export { extractFinalAnswer as _extractFinalAnswerForTest, buildUserMessage as _buildUserMessageForTest, };
+//# sourceMappingURL=gaia-agent.d.ts.map