claude-flow 3.10.4 → 3.10.5

package/v3/@claude-flow/cli/dist/src/benchmarks/gaia-causal-memory.smoke.js

@@ -0,0 +1,300 @@
+/**
+ * Smoke tests — ADR-135 Track I: Causal failure-avoidance edges
+ *
+ * All I/O is mocked (no real filesystem or network calls).
+ *
+ * Tests:
+ *   1. Record one failure → retrieve same-signature question → returns hint
+ *   2. Record 3 failures → retrieve unrelated question → returns empty hint
+ *   3. Same edge recorded twice → occurrenceCount=2, not duplicated
+ *   4. File doesn't exist → graceful empty result on retrieve
+ *   5. Corrupted JSONL line → skipped, doesn't crash
+ *   6. maxEdgesPerSignature cap respected (no new edge after cap)
+ *   7. Signature computation is deterministic (same input → same output)
+ *   8. Correct answer trajectory → no edges recorded
+ *
+ * Usage:
+ *   npx tsx src/benchmarks/gaia-causal-memory.smoke.ts
+ *
+ * Refs: ADR-135, #2156
+ */
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { computeQuestionSignature, inferFailureType, recordCausalFailures, retrieveCausalHints, } from './gaia-causal-memory.js';
+// ---------------------------------------------------------------------------
+// Mini test runner (zero external deps)
+// ---------------------------------------------------------------------------
+let passed = 0;
+let failed = 0;
+const results = [];
+async function test(name, fn) {
+    try {
+        await fn();
+        passed++;
+        results.push({ name, ok: true });
+        console.log(`  PASS  ${name}`);
+    }
+    catch (err) {
+        failed++;
+        const msg = err instanceof Error ? err.message : String(err);
+        results.push({ name, ok: false, error: msg });
+        console.error(`  FAIL  ${name}`);
+        console.error(`        ${msg}`);
+    }
+}
+function assert(condition, message) {
+    if (!condition) {
+        throw new Error(`Assertion failed: ${message}`);
+    }
+}
+function assertEqual(actual, expected, label) {
+    if (actual !== expected) {
+        throw new Error(`${label}: expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`);
+    }
+}
+// ---------------------------------------------------------------------------
+// Fixtures
+// ---------------------------------------------------------------------------
+function makeQuestion(text) {
+    return {
+        task_id: 'smoke-' + text.slice(0, 8),
+        level: 1,
+        question: text,
+        final_answer: 'ignored-in-smoke',
+        file_name: null,
+        file_path: null,
+    };
+}
+function makeResult(overrides = {}) {
+    return {
+        questionId: 'smoke-q1',
+        finalAnswer: 'wrong answer',
+        turns: 3,
+        toolCallsByName: { web_search: 2 },
+        totalInputTokens: 100,
+        totalOutputTokens: 50,
+        wallMs: 1500,
+        ...overrides,
+    };
+}
+// ---------------------------------------------------------------------------
+// Test suite
+// ---------------------------------------------------------------------------
+async function runAll() {
+    console.log('\nADR-135 Track I — causal failure-avoidance edges (smoke)\n');
+    // We use a fresh temp dir per test group to isolate state.
+    const tmpBase = path.join(os.tmpdir(), 'ruflo-causal-smoke-' + process.pid);
+    // -------------------------------------------------------------------------
+    // Test 1: Record one failure → retrieve same-signature question → hint returned
+    // -------------------------------------------------------------------------
+    await test('1. record one failure → retrieve same question → hint returned', async () => {
+        const dir = path.join(tmpBase, 't1');
+        fs.mkdirSync(dir, { recursive: true });
+        const storePath = path.join(dir, 'edges.jsonl');
+        const question = makeQuestion('What is the capital of France?');
+        const result = makeResult({ finalAnswer: 'London' });
+        const recRes = await recordCausalFailures(question, result, false, { storePath });
+        assert(recRes.edgesRecorded > 0, 'should have recorded at least one edge');
+        const retRes = await retrieveCausalHints(question, { storePath });
+        assert(retRes.edgesMatched > 0, 'should match the stored edge');
+        assert(retRes.hint.length > 0, 'hint should be non-empty');
+        assert(retRes.hint.includes('[PRIOR FAILURES]'), 'hint should contain [PRIOR FAILURES] header');
+        assert(retRes.hint.includes('web_search'), 'hint should mention the failed tool');
+    });
+    // -------------------------------------------------------------------------
+    // Test 2: Record 3 failures → retrieve unrelated question → empty hint
+    // -------------------------------------------------------------------------
+    await test('2. record 3 failures → retrieve unrelated question → empty hint', async () => {
+        const dir = path.join(tmpBase, 't2');
+        fs.mkdirSync(dir, { recursive: true });
+        const storePath = path.join(dir, 'edges.jsonl');
+        const questions = [
+            makeQuestion('What is the capital of France?'),
+            makeQuestion('Who wrote Hamlet?'),
+            makeQuestion('What is 2 + 2?'),
+        ];
+        for (const q of questions) {
+            await recordCausalFailures(q, makeResult(), false, { storePath });
+        }
+        const unrelated = makeQuestion('Completely different and unrelated question about astrophysics and quantum phenomena XYZ123');
+        const retRes = await retrieveCausalHints(unrelated, { storePath });
+        assertEqual(retRes.edgesMatched, 0, 'edgesMatched for unrelated question');
+        assertEqual(retRes.hint, '', 'hint for unrelated question');
+    });
+    // -------------------------------------------------------------------------
+    // Test 3: Same edge recorded twice → occurrenceCount=2, not duplicated
+    // -------------------------------------------------------------------------
+    await test('3. same edge recorded twice → occurrenceCount=2, not duplicated', async () => {
+        const dir = path.join(tmpBase, 't3');
+        fs.mkdirSync(dir, { recursive: true });
+        const storePath = path.join(dir, 'edges.jsonl');
+        const question = makeQuestion('What year did World War II end?');
+        const result = makeResult({ toolCallsByName: { web_search: 1 } });
+        // Record same failure twice
+        await recordCausalFailures(question, result, false, { storePath });
+        await recordCausalFailures(question, result, false, { storePath });
+        // Read the JSONL directly and count lines / occurrence counts
+        const raw = fs.readFileSync(storePath, 'utf8');
+        const lines = raw.trim().split('\n').filter((l) => l.trim() !== '');
+        const edges = lines.map((l) => JSON.parse(l));
+        // Should have exactly one edge for this signature+tool+step
+        const sig = computeQuestionSignature(question.question);
+        const sigEdges = edges.filter((e) => e.questionSignature === sig && e.failedTool === 'web_search');
+        assertEqual(sigEdges.length, 1, 'should have exactly one edge per signature+tool+step');
+        assertEqual(sigEdges[0].occurrenceCount, 2, 'occurrenceCount after two recordings');
+    });
+    // -------------------------------------------------------------------------
+    // Test 4: File doesn't exist → graceful empty result on retrieve
+    // -------------------------------------------------------------------------
+    await test('4. file does not exist → graceful empty result on retrieve', async () => {
+        const storePath = path.join(tmpBase, 'nonexistent', 'edges.jsonl');
+        const question = makeQuestion('How many legs does a spider have?');
+        const retRes = await retrieveCausalHints(question, { storePath });
+        assertEqual(retRes.edgesMatched, 0, 'edgesMatched when file absent');
+        assertEqual(retRes.hint, '', 'hint when file absent');
+    });
+    // -------------------------------------------------------------------------
+    // Test 5: Corrupted JSONL line → skipped, doesn't crash
+    // -------------------------------------------------------------------------
+    await test('5. corrupted JSONL line → skipped, does not crash', async () => {
+        const dir = path.join(tmpBase, 't5');
+        fs.mkdirSync(dir, { recursive: true });
+        const storePath = path.join(dir, 'edges.jsonl');
+        const goodQuestion = makeQuestion('Name the largest planet in the solar system.');
+        const goodEdge = {
+            questionSignature: computeQuestionSignature(goodQuestion.question),
+            failedTool: 'python_exec',
+            failedTrajectoryStep: 'python_exec called once; failure type=wrong_answer',
+            observedFailureType: 'wrong_answer',
+            createdAt: new Date().toISOString(),
+            occurrenceCount: 1,
+        };
+        // Write one valid edge + one corrupted line
+        fs.mkdirSync(path.dirname(storePath), { recursive: true });
+        fs.writeFileSync(storePath, JSON.stringify(goodEdge) + '\n' +
+            '{"this": is not valid JSON!!!\n' +
+            '{"partial":true\n', 'utf8');
+        // retrieve should work and not throw
+        const retRes = await retrieveCausalHints(goodQuestion, { storePath });
+        assertEqual(retRes.edgesMatched, 1, 'should still find the one valid edge');
+        assert(retRes.hint.includes('[PRIOR FAILURES]'), 'hint should be returned for valid edge');
+        assert(retRes.hint.includes('python_exec'), 'hint should reference python_exec');
+    });
+    // -------------------------------------------------------------------------
+    // Test 6: maxEdgesPerSignature cap respected
+    // -------------------------------------------------------------------------
+    await test('6. maxEdgesPerSignature cap respected', async () => {
+        const dir = path.join(tmpBase, 't6');
+        fs.mkdirSync(dir, { recursive: true });
+        const storePath = path.join(dir, 'edges.jsonl');
+        const maxEdgesPerSignature = 2;
+        const question = makeQuestion('Explain the theory of relativity in simple terms.');
+        // Record 3 failures with different tools — only first 2 should be stored
+        for (const tool of ['web_search', 'file_read', 'python_exec']) {
+            const result = makeResult({ toolCallsByName: { [tool]: 1 } });
+            await recordCausalFailures(question, result, false, {
+                storePath,
+                maxEdgesPerSignature,
+            });
+        }
+        const raw = fs.readFileSync(storePath, 'utf8');
+        const lines = raw.trim().split('\n').filter((l) => l.trim() !== '');
+        const sig = computeQuestionSignature(question.question);
+        const sigEdges = lines
+            .map((l) => JSON.parse(l))
+            .filter((e) => e.questionSignature === sig);
+        assert(sigEdges.length <= maxEdgesPerSignature, `should have at most ${maxEdgesPerSignature} edges, got ${sigEdges.length}`);
+    });
+    // -------------------------------------------------------------------------
+    // Test 7: Signature computation is deterministic
+    // -------------------------------------------------------------------------
+    await test('7. signature computation is deterministic', async () => {
+        const text = 'What is the boiling point of water at sea level?';
+        const sig1 = computeQuestionSignature(text);
+        const sig2 = computeQuestionSignature(text);
+        const sig3 = computeQuestionSignature(text.trim()); // trim should not change result
+        const sig4 = computeQuestionSignature(text.toUpperCase().toLowerCase()); // normalisation
+        assertEqual(sig1, sig2, 'same input produces same signature (call 1 vs 2)');
+        assertEqual(sig1, sig3, 'trimmed input produces same signature');
+        assertEqual(sig1, sig4, 'case-normalised input produces same signature');
+        assertEqual(sig1.length, 16, 'signature is 16 hex characters');
+        // Different text should produce different signature (with overwhelming probability)
+        const sigOther = computeQuestionSignature('Completely different question about astronomy.');
+        assert(sig1 !== sigOther, 'different inputs produce different signatures');
+    });
+    // -------------------------------------------------------------------------
+    // Test 8: Correct answer trajectory → no edges recorded
+    // -------------------------------------------------------------------------
+    await test('8. correct answer trajectory → no edges recorded', async () => {
+        const dir = path.join(tmpBase, 't8');
+        fs.mkdirSync(dir, { recursive: true });
+        const storePath = path.join(dir, 'edges.jsonl');
+        const question = makeQuestion('What color is the sky on a clear day?');
+        const result = makeResult({ finalAnswer: 'blue' });
+        // wasCorrect = true
+        const recRes = await recordCausalFailures(question, result, true, { storePath });
+        assertEqual(recRes.edgesRecorded, 0, 'no edges for correct trajectory');
+        assert(!fs.existsSync(storePath), 'store file should not be created for correct answer');
+    });
+    // -------------------------------------------------------------------------
+    // Bonus: inferFailureType unit checks
+    // -------------------------------------------------------------------------
+    await test('inferFailureType: null for correct answers', async () => {
+        const result = makeResult({ finalAnswer: 'blue' });
+        const ft = inferFailureType(result, true);
+        assertEqual(ft, null, 'inferFailureType returns null for wasCorrect=true');
+    });
+    await test('inferFailureType: timeout when timedOut=true', async () => {
+        const result = makeResult({ timedOut: true });
+        const ft = inferFailureType(result, false);
+        assertEqual(ft, 'timeout', 'inferFailureType=timeout when timedOut flag set');
+    });
+    await test('inferFailureType: tool_error when error present', async () => {
+        const result = makeResult({ error: 'network timeout' });
+        const ft = inferFailureType(result, false);
+        assertEqual(ft, 'tool_error', 'inferFailureType=tool_error when error field set');
+    });
+    await test('inferFailureType: empty_result when finalAnswer is null', async () => {
+        const result = makeResult({ finalAnswer: null });
+        const ft = inferFailureType(result, false);
+        assertEqual(ft, 'empty_result', 'inferFailureType=empty_result when finalAnswer=null');
+    });
+    await test('inferFailureType: wrong_answer for normal failure', async () => {
+        const result = makeResult({ finalAnswer: 'incorrect value' });
+        const ft = inferFailureType(result, false);
+        assertEqual(ft, 'wrong_answer', 'inferFailureType=wrong_answer for normal wrong answer');
+    });
+    // -------------------------------------------------------------------------
+    // Cleanup
+    // -------------------------------------------------------------------------
+    try {
+        fs.rmSync(tmpBase, { recursive: true, force: true });
+    }
+    catch {
+        // Non-fatal cleanup failure; temp files will be cleared by OS.
+    }
+    // -------------------------------------------------------------------------
+    // Summary
+    // -------------------------------------------------------------------------
+    console.log(`\n${'─'.repeat(60)}`);
+    console.log(`Results: ${passed} passed, ${failed} failed (${passed + failed} total)`);
+    if (failed > 0) {
+        console.error('\nFailed tests:');
+        for (const r of results.filter((r) => !r.ok)) {
+            console.error(`  ${r.name}: ${r.error}`);
+        }
+        process.exit(1);
+    }
+    else {
+        console.log('\nAll smoke tests passed.');
+        console.log('\nNote: Track I lift is compound (+0pp first run, +2-5pp after 5+ runs).');
+        console.log('Wiring into gaia-bench.ts is a follow-up PR (in-flight conflict avoidance).');
+        process.exit(0);
+    }
+}
+runAll().catch((err) => {
+    console.error('Unhandled error in smoke runner:', err);
+    process.exit(1);
+});
+//# sourceMappingURL=gaia-causal-memory.smoke.js.map

package/v3/@claude-flow/cli/dist/src/benchmarks/gaia-convergence.d.ts

@@ -0,0 +1,138 @@
+/**
+ * GAIA Convergence Layer — deterministic finalization for saturated agent loops.
+ *
+ * Detects three failure modes that cause empty FINAL_ANSWER extraction:
+ *   1. max_turns hit without a final_answer call
+ *   2. Loop detected (same tool + same args called 3× in a 5-turn window)
+ *   3. Token budget exceeded (>120k tokens of conversation context)
+ *
+ * On detection, runs a FORCED COMMIT phase:
+ *   - Injects a stripped-down summary prompt: "Based on all observations, answer with
+ *     FINAL_ANSWER: X. Do NOT explore further."
+ *   - Makes 1 final API call with strict instruction and no tools.
+ *   - If still no FINAL_ANSWER in the response, runs the Stage 1 extraction cascade
+ *     against ALL prior assistant messages (last to first), returns first non-empty hit.
+ *   - If still empty: returns null but logs the failure mode.
+ *
+ * This is NOT a new cognition layer — it is a convergence controller.
+ * It STRIPS information rather than adding it: one final chance to commit,
+ * no tools available, prior context summarized rather than appended.
+ *
+ * Architecture principle (iter 60 post-mortem):
+ *   "More information can reduce agent reliability. Past a certain point —
+ *    retrieval depth, context size, browsing breadth, tool diversity — information
+ *    increases trajectory entropy. The system becomes less likely to finalize coherently."
+ *   This layer is the entropy-reducer.
+ *
+ * Refs: #2156, iter 60 (19/25 empty FINAL_ANSWER failures), iter 62
+ */
+/** Token threshold (sum of input tokens across all turns) that triggers overflow detection. */
+export declare const TOKEN_OVERFLOW_THRESHOLD = 120000;
+/** Number of repeated identical tool+args calls in a window that signals a loop. */
+export declare const LOOP_REPEAT_THRESHOLD = 3;
+/** Sliding window size (turns) for loop detection. */
+export declare const LOOP_WINDOW_SIZE = 5;
+/** Tracks state that the convergence layer needs to evaluate triggers. */
+export interface ConvergenceState {
+    /** How many agent turns have elapsed so far. */
+    turnCount: number;
+    /** Sum of input tokens across all turns (used for overflow detection). */
+    totalTokens: number;
+    /** Ordered log of tool calls for loop detection. */
+    toolCalls: Array<{
+        name: string;
+        argsHash: string;
+        turn: number;
+    }>;
+    /** Set by checkConvergenceTriggers when a failure mode is detected. */
+    detectedFailureMode: 'max_turns' | 'loop' | 'token_overflow' | null;
+}
+/** Result of a forced-commit attempt. */
+export interface ForceCommitResult {
+    /** The extracted answer, or null if even forced commit could not extract one. */
+    answer: string | null;
+    /** True when the answer was recovered from prior message history rather than
+     *  from the forced-commit API call. */
+    usedFallback: boolean;
+    /** The failure mode that triggered this forced commit. */
+    triggerMode: ConvergenceState['detectedFailureMode'];
+}
+/**
+ * Produce a stable hash of a tool call's name + args for loop detection.
+ *
+ * Uses SHA-256 truncated to 16 hex chars — collision probability is
+ * negligible for the small call volumes in a single agent run.
+ *
+ * The hash is deterministic: same toolName + same args always → same hash.
+ * Different args always → different hash (within SHA-256 collision bounds).
+ */
+export declare function argsHash(toolName: string, args: object): string;
+/**
+ * Evaluate the current ConvergenceState and return the first failure mode
+ * detected, or null if no trigger has fired.
+ *
+ * Evaluation order:
+ *   1. max_turns — turnCount >= maxTurns
+ *   2. token_overflow — totalTokens >= TOKEN_OVERFLOW_THRESHOLD
+ *   3. loop — same tool+argsHash appears >= LOOP_REPEAT_THRESHOLD times
+ *              in the last LOOP_WINDOW_SIZE entries of toolCalls
+ *
+ * Only the FIRST matching trigger is returned (stops at first detection).
+ * The caller is responsible for setting state.detectedFailureMode.
+ */
+export declare function checkConvergenceTriggers(state: ConvergenceState, maxTurns: number): ConvergenceState['detectedFailureMode'];
+/**
+ * Run Stage 1 extraction (FINAL_ANSWER: pattern) against a raw text string.
+ * Returns the matched answer or null.
+ */
+export declare function extractFinalAnswerFromText(text: string): string | null;
+/**
+ * Scan prior assistant messages from last to first, looking for FINAL_ANSWER.
+ *
+ * This is the fallback path when the forced-commit API call still does not
+ * produce a FINAL_ANSWER. We search backwards because the most recent
+ * assistant message is most likely to contain the best answer.
+ */
+export declare function extractFromPriorMessages(messages: Array<{
+    role: string;
+    content: string | unknown;
+}>): string | null;
+/**
+ * Run the forced-commit phase for a saturated agent loop.
+ *
+ * Steps:
+ *   1. Append a stripped directive prompt to the message history.
+ *   2. Call the model once with NO tools — forces text-only response.
+ *   3. If FINAL_ANSWER found in the response → return it.
+ *   4. If not → scan prior assistant messages last-to-first for FINAL_ANSWER.
+ *   5. If still empty → return { answer: null, usedFallback: false }.
+ *
+ * The callModel callback must return the raw text response (assistant turn
+ * text blocks concatenated). It should be called WITHOUT tool definitions
+ * so the model cannot call tools in this turn.
+ *
+ * @param messages  Current conversation history (mutated: 1 user turn appended)
+ * @param callModel Callback that makes a single API call and returns the response text
+ * @param triggerMode  The failure mode that triggered this forced commit
+ */
+export declare function forceCommit(messages: Array<{
+    role: string;
+    content: string | unknown;
+}>, callModel: (messages: Array<{
+    role: string;
+    content: string | unknown;
+}>) => Promise<string>, triggerMode?: ConvergenceState['detectedFailureMode']): Promise<ForceCommitResult>;
+/** Create a new ConvergenceState with zero counters. */
+export declare function createConvergenceState(): ConvergenceState;
+/**
+ * Update ConvergenceState after each agent turn completes.
+ *
+ * @param state       Mutable state object (modified in place)
+ * @param inputTokens Tokens consumed in this turn
+ * @param toolCallsThisTurn  Tool calls made in this turn (name + args)
+ */
+export declare function recordTurn(state: ConvergenceState, inputTokens: number, toolCallsThisTurn: Array<{
+    name: string;
+    args: object;
+}>): void;
+//# sourceMappingURL=gaia-convergence.d.ts.map