claude-flow 3.10.4 → 3.10.5

package/v3/@claude-flow/cli/dist/src/benchmarks/gaia-critic.js

@@ -0,0 +1,312 @@
+/**
+ * GAIA Adversarial Critic — ADR-135 Track D
+ *
+ * After the main agent produces a candidate answer, a Sonnet pass reviews it.
+ * If the critic verdict is "fail", the orchestrator re-runs the agent with the
+ * critique injected as additional context.
+ *
+ * Motivation (iter 29 finding): tool quality is the bottleneck on L1 (~20.8%).
+ * The critic catches wrong-because-of-bad-tool-result answers BEFORE submission,
+ * without requiring better search backends.  Expected L1 lift: +3-5pp.
+ *
+ * Design:
+ *  - NEW file only; NOT wired into gaia-bench.ts yet to avoid merge conflicts
+ *    with in-flight iter 29/31/34 branches.  Wiring is a small follow-up PR.
+ *  - `criticReview()` — single Sonnet call, returns structured verdict.
+ *  - `runGaiaAgentWithCritic()` — orchestration wrapper: runs agent, calls
+ *    critic, retries once on "fail".  "uncertain" is treated as "pass" (don't
+ *    burn retries on borderline cases).
+ *  - API errors and malformed JSON from the critic are caught; original
+ *    candidate is returned with an error-flagged verdict.
+ *  - Default opt-in: enableCritic=false.  Set true via RunWithCriticOptions.
+ *
+ * Cost discipline:
+ *  - Critic uses claude-sonnet-4-6 (separate from the agent's default Haiku).
+ *  - One critic call + one optional retry = max 2 extra Sonnet calls per Q.
+ *  - Approximate extra cost per question: ~$0.003-0.005 (well within budget).
+ *
+ * Plugin sync TODO (follow-up PR after gaia-bench wiring):
+ *  - Update plugins/ruflo-workflows/commands/gaia-run.md with --enable-critic flag.
+ *  - Update plugins/ruflo-workflows/skills/gaia-debugging/SKILL.md: add critic
+ *    as a recommended diagnostic step for wrong-answer analysis.
+ *
+ * Refs: ADR-135, ADR-133, iter 29 finding, #2156
+ */
+import { execSync } from 'node:child_process';
+import { runGaiaAgent, } from './gaia-agent.js';
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+const ANTHROPIC_API_URL = 'https://api.anthropic.com/v1/messages';
+const ANTHROPIC_API_VERSION = '2023-06-01';
+/** Default model for the critic — Sonnet for higher reasoning quality. */
+const DEFAULT_CRITIC_MODEL = 'claude-sonnet-4-6';
+/** Max tokens for the critic response (verdict JSON is short). */
+const CRITIC_MAX_TOKENS = 512;
+/** Sonnet pricing (input/output per million tokens, 2026-05-27).
+ *  Used only for cost estimation in CriticVerdict. */
+const SONNET_INPUT_COST_PER_M = 3.0;
+const SONNET_OUTPUT_COST_PER_M = 15.0;
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+/** Resolve Anthropic API key using the same precedence as gaia-agent.ts. */
+function resolveApiKey(override) {
+    if (override)
+        return override;
+    const fromEnv = process.env['ANTHROPIC_API_KEY'];
+    if (fromEnv)
+        return fromEnv;
+    try {
+        return execSync('gcloud secrets versions access latest --secret=ANTHROPIC_API_KEY', { encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }).trim();
+    }
+    catch {
+        throw new Error('ANTHROPIC_API_KEY not set and gcloud fallback failed. ' +
+            'Set ANTHROPIC_API_KEY env var or ensure gcloud access.');
+    }
+}
+/**
+ * Build the critic system prompt.
+ * Instructs Sonnet to act as an adversarial reviewer and respond in JSON.
+ */
+function buildCriticSystemPrompt() {
+    return `You are an adversarial reviewer of agent answers on the GAIA benchmark. \
+Your job is to find flaws in candidate answers before they are submitted.
+
+Given a question, a candidate answer, and a summary of the agent's reasoning \
+trajectory, you must decide whether the candidate answer is correct.
+
+Evaluation criteria:
+1. Does the answer directly address what the question asks?
+2. Is there evidence in the trajectory that supports the answer?
+3. Are there obvious flaws (wrong unit, wrong format, missed constraint, \
+fabricated source, off-by-one error, truncation)?
+4. For numeric answers: is the magnitude and unit plausible?
+5. For list answers: are all required items present and correctly ordered?
+
+Respond ONLY with a JSON object — no markdown, no prose outside the JSON:
+{"verdict":"pass"|"fail"|"uncertain","reasoning":"<1-3 sentences>","suggestedRevision":"<corrected answer or empty string>"}
+
+Use "uncertain" only when you genuinely cannot determine correctness from the \
+available information. Never use "uncertain" to avoid making a decision when \
+evidence is available.`;
+}
+/**
+ * Build the critic user message combining the question, candidate answer,
+ * and a compressed view of the agent trajectory.
+ */
+function buildCriticUserMessage(question, candidateAnswer, trajectory) {
+    // Summarise trajectory: first 2 + last 2 steps to keep tokens bounded.
+    const steps = trajectory.steps ?? [];
+    const summarised = steps.length <= 4
+        ? steps
+        : [...steps.slice(0, 2), ...steps.slice(-2)];
+    const trajectoryText = summarised.length === 0
+        ? '(no tool calls recorded)'
+        : summarised.map((s, i) => {
+            const label = s.tool ? `Step ${i + 1} [${s.tool}]` : `Step ${i + 1}`;
+            const result = s.result ? ` → ${s.result.slice(0, 200)}` : '';
+            return `${label}${result}`;
+        }).join('\n');
+    return `QUESTION: ${question.question}
+
+CANDIDATE ANSWER: ${candidateAnswer}
+
+AGENT TRAJECTORY (${trajectory.turns} turns, summarised):
+${trajectoryText}`;
+}
+/**
+ * Attempt to extract a verdict from a potentially malformed JSON string.
+ * Falls back gracefully to "uncertain" with the raw text preserved.
+ */
+function parseVerdictFallback(raw) {
+    // Try standard JSON parse first.
+    try {
+        const parsed = JSON.parse(raw);
+        const verdict = parsed['verdict'];
+        if (verdict === 'pass' || verdict === 'fail' || verdict === 'uncertain') {
+            return {
+                verdict,
+                reasoning: parsed['reasoning'] ?? '',
+                suggestedRevision: parsed['suggestedRevision'] ?? '',
+            };
+        }
+    }
+    catch {
+        // Fall through to regex extraction.
+    }
+    // Regex extraction for embedded JSON in prose.
+    const jsonMatch = raw.match(/\{[\s\S]*?"verdict"\s*:\s*"(pass|fail|uncertain)"[\s\S]*?\}/);
+    if (jsonMatch) {
+        try {
+            const parsed = JSON.parse(jsonMatch[0]);
+            return {
+                verdict: parsed['verdict'],
+                reasoning: parsed['reasoning'] ?? '',
+                suggestedRevision: parsed['suggestedRevision'] ?? '',
+            };
+        }
+        catch {
+            // Fall through.
+        }
+    }
+    // Last resort: extract verdict keyword from raw text.
+    const verdictMatch = raw.match(/\b(pass|fail|uncertain)\b/i);
+    return {
+        verdict: verdictMatch
+            ? verdictMatch[1].toLowerCase()
+            : 'uncertain',
+        reasoning: `Critic returned malformed JSON; extracted verdict heuristically.`,
+        suggestedRevision: '',
+        rawResponse: raw.slice(0, 500),
+    };
+}
+// ---------------------------------------------------------------------------
+// Core critic function
+// ---------------------------------------------------------------------------
+/**
+ * Run the adversarial critic against a candidate answer.
+ *
+ * @param question     - The GAIA question being evaluated.
+ * @param candidateAnswer - The agent's proposed final answer.
+ * @param trajectory   - Lightweight trajectory summary from the agent run.
+ * @param options      - Critic configuration (model, apiKey).
+ * @returns            CriticVerdict with verdict, reasoning, cost.
+ */
+export async function criticReview(question, candidateAnswer, trajectory, options) {
+    const model = options?.model ?? DEFAULT_CRITIC_MODEL;
+    const apiKey = resolveApiKey(options?.apiKey);
+    const t0 = Date.now();
+    const requestBody = {
+        model,
+        max_tokens: CRITIC_MAX_TOKENS,
+        system: buildCriticSystemPrompt(),
+        messages: [
+            {
+                role: 'user',
+                content: buildCriticUserMessage(question, candidateAnswer, trajectory),
+            },
+        ],
+    };
+    let rawResponseText = '';
+    try {
+        const response = await fetch(ANTHROPIC_API_URL, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'x-api-key': apiKey,
+                'anthropic-version': ANTHROPIC_API_VERSION,
+            },
+            body: JSON.stringify(requestBody),
+        });
+        if (!response.ok) {
+            const errText = await response.text().catch(() => '');
+            throw new Error(`Anthropic API error ${response.status}: ${errText.slice(0, 200)}`);
+        }
+        const data = await response.json();
+        const inputTokens = data.usage?.input_tokens ?? 0;
+        const outputTokens = data.usage?.output_tokens ?? 0;
+        const costUsd = (inputTokens / 1_000_000) * SONNET_INPUT_COST_PER_M +
+            (outputTokens / 1_000_000) * SONNET_OUTPUT_COST_PER_M;
+        rawResponseText = data.content
+            .filter(b => b.type === 'text')
+            .map(b => b.text ?? '')
+            .join('');
+        const parsed = parseVerdictFallback(rawResponseText);
+        return {
+            verdict: parsed.verdict ?? 'uncertain',
+            reasoning: parsed.reasoning ?? '',
+            suggestedRevision: parsed.suggestedRevision ?? '',
+            costUsd,
+            ...(parsed.rawResponse ? { rawResponse: parsed.rawResponse } : {}),
+        };
+    }
+    catch (err) {
+        const wallMs = Date.now() - t0;
+        const errMsg = err instanceof Error ? err.message : String(err);
+        // Graceful fallback: treat critic error as "uncertain" so agent result
+        // is still returned rather than throwing.
+        return {
+            verdict: 'uncertain',
+            reasoning: `Critic call failed after ${wallMs}ms: ${errMsg.slice(0, 200)}`,
+            suggestedRevision: '',
+            costUsd: 0,
+            error: true,
+            rawResponse: rawResponseText.slice(0, 200) || undefined,
+        };
+    }
+}
+// ---------------------------------------------------------------------------
+// Orchestration wrapper
+// ---------------------------------------------------------------------------
+/**
+ * Run the GAIA agent with an optional adversarial critic pass.
+ *
+ * When enableCritic=false (default), this is a thin pass-through to
+ * runGaiaAgent with an empty criticVerdicts array.
+ *
+ * When enableCritic=true:
+ *  1. Run runGaiaAgent normally.
+ *  2. If the agent produced a finalAnswer, call criticReview.
+ *  3. If verdict is "fail" and retriesAttempted < maxRetries:
+ *     a. Re-run the agent with the critique injected as additional context.
+ *     b. Call criticReview again on the new answer.
+ *  4. Return the final result with all critic verdicts attached.
+ *
+ * Note on "uncertain": treated as "pass" (no retry triggered).
+ */
+export async function runGaiaAgentWithCritic(question, options = {}) {
+    const { enableCritic = false, criticOptions, ...agentOptions } = options;
+    // Fast path: critic disabled.
+    if (!enableCritic) {
+        const result = await runGaiaAgent(question, agentOptions);
+        return { ...result, criticVerdicts: [], retriesAttempted: 0 };
+    }
+    const maxRetries = criticOptions?.maxRetries ?? 1;
+    const criticVerdicts = [];
+    let retriesAttempted = 0;
+    // First agent run.
+    let agentResult = await runGaiaAgent(question, agentOptions);
+    // If agent timed out or errored with no answer, skip critic.
+    if (agentResult.finalAnswer == null) {
+        return { ...agentResult, criticVerdicts, retriesAttempted };
+    }
+    // Build a lightweight trajectory summary from the agent result.
+    // gaia-agent.ts doesn't expose step-level detail in GaiaAgentResult, so we
+    // synthesise from the available fields (tool call counts, turn count).
+    const makeTrajectory = (result) => ({
+        turns: result.turns,
+        steps: Object.entries(result.toolCallsByName).map(([tool, count]) => ({
+            tool,
+            result: `called ${count} time(s)`,
+        })),
+    });
+    // First critic pass.
+    let verdict = await criticReview(question, agentResult.finalAnswer, makeTrajectory(agentResult), criticOptions);
+    criticVerdicts.push(verdict);
+    // Retry loop on "fail".
+    while (verdict.verdict === 'fail' && retriesAttempted < maxRetries) {
+        retriesAttempted += 1;
+        // Inject critique as additional context via the system prompt append.
+        // We pass it through GaiaAgentOptions.criticFeedback — gaia-agent.ts does
+        // NOT yet read this field (wiring is follow-up PR), but storing it here
+        // makes the contract explicit and harmless (unknown options are ignored).
+        const retryOptions = {
+            ...agentOptions,
+            criticFeedback: `Previous answer "${agentResult.finalAnswer}" was flagged by adversarial review: ` +
+                `${verdict.reasoning}` +
+                (verdict.suggestedRevision
+                    ? ` Suggested revision: ${verdict.suggestedRevision}`
+                    : ''),
+        };
+        agentResult = await runGaiaAgent(question, retryOptions);
+        if (agentResult.finalAnswer == null) {
+            // Retry yielded no answer; stop retrying.
+            break;
+        }
+        verdict = await criticReview(question, agentResult.finalAnswer, makeTrajectory(agentResult), criticOptions);
+        criticVerdicts.push(verdict);
+    }
+    return { ...agentResult, criticVerdicts, retriesAttempted };
+}
+//# sourceMappingURL=gaia-critic.js.map

package/v3/@claude-flow/cli/dist/src/benchmarks/gaia-critic.smoke.d.ts

@@ -0,0 +1,21 @@
+/**
+ * GAIA Critic Smoke Tests — ADR-135 Track D
+ *
+ * Tests for the adversarial critic agent in gaia-critic.ts.
+ * All tests use mocked responses — NO live API calls.
+ *
+ * Test coverage:
+ *  1. Critic returns "pass" → no retry, immediately returns candidate
+ *  2. Critic returns "fail" with suggestedRevision → triggers one retry
+ *  3. Critic returns "fail" twice → retries exhausted, returns last candidate
+ *  4. Critic returns "uncertain" → treated as "pass", no retry
+ *  5. API error in critic → graceful fallback, returns candidate as-is
+ *  6. Malformed JSON from critic → fallback parser extracts verdict
+ *
+ * Usage (no API key required):
+ *   npx tsx src/benchmarks/gaia-critic.smoke.ts
+ *
+ * Refs: ADR-135, #2156
+ */
+export {};
+//# sourceMappingURL=gaia-critic.smoke.d.ts.map

package/v3/@claude-flow/cli/dist/src/benchmarks/gaia-critic.smoke.js

@@ -0,0 +1,327 @@
+/**
+ * GAIA Critic Smoke Tests — ADR-135 Track D
+ *
+ * Tests for the adversarial critic agent in gaia-critic.ts.
+ * All tests use mocked responses — NO live API calls.
+ *
+ * Test coverage:
+ *  1. Critic returns "pass" → no retry, immediately returns candidate
+ *  2. Critic returns "fail" with suggestedRevision → triggers one retry
+ *  3. Critic returns "fail" twice → retries exhausted, returns last candidate
+ *  4. Critic returns "uncertain" → treated as "pass", no retry
+ *  5. API error in critic → graceful fallback, returns candidate as-is
+ *  6. Malformed JSON from critic → fallback parser extracts verdict
+ *
+ * Usage (no API key required):
+ *   npx tsx src/benchmarks/gaia-critic.smoke.ts
+ *
+ * Refs: ADR-135, #2156
+ */
+import { criticReview, runGaiaAgentWithCritic, } from './gaia-critic.js';
+// ---------------------------------------------------------------------------
+// Test helpers
+// ---------------------------------------------------------------------------
+let passed = 0;
+let failed = 0;
+function assert(condition, message) {
+    if (condition) {
+        console.log(`  PASS  ${message}`);
+        passed++;
+    }
+    else {
+        console.error(`  FAIL  ${message}`);
+        failed++;
+    }
+}
+function assertEqual(actual, expected, message) {
+    assert(actual === expected, `${message} (expected ${String(expected)}, got ${String(actual)})`);
+}
+/** A minimal GaiaQuestion fixture. */
+const FIXTURE_QUESTION = {
+    task_id: 'smoke-001',
+    level: 1,
+    question: 'What is the capital of France?',
+    final_answer: 'Paris',
+    file_name: null,
+    file_path: null,
+};
+/** Minimal GaiaAgentResult with a given finalAnswer. */
+function makeAgentResult(finalAnswer, turns = 2) {
+    return {
+        questionId: FIXTURE_QUESTION.task_id,
+        finalAnswer,
+        turns,
+        toolCallsByName: { web_search: 1 },
+        totalInputTokens: 100,
+        totalOutputTokens: 50,
+        wallMs: 1200,
+    };
+}
+function installFetchMock(responses) {
+    let callIndex = 0;
+    const original = globalThis.fetch;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    globalThis.fetch = async (_url, _init) => {
+        const resp = responses[Math.min(callIndex, responses.length - 1)];
+        callIndex++;
+        if (!resp.ok) {
+            return {
+                ok: false,
+                status: resp.status ?? 500,
+                text: async () => resp.text ?? 'Internal error',
+                json: async () => { throw new Error('not ok'); },
+            };
+        }
+        return {
+            ok: true,
+            status: 200,
+            json: async () => resp.body ?? {},
+            text: async () => JSON.stringify(resp.body ?? {}),
+        };
+    };
+    return () => { globalThis.fetch = original; };
+}
+/**
+ * Build a mock Anthropic response body containing a critic verdict JSON.
+ */
+function mockCriticResponse(verdict, reasoning, suggestedRevision = '') {
+    const content = JSON.stringify({ verdict, reasoning, suggestedRevision });
+    return {
+        content: [{ type: 'text', text: content }],
+        usage: { input_tokens: 200, output_tokens: 40 },
+    };
+}
+/**
+ * Mock for runGaiaAgent that immediately returns a fixed result.
+ * We monkey-patch the module-level import by wrapping runGaiaAgentWithCritic
+ * via its options.catalogue approach — but since gaia-agent's runGaiaAgent is
+ * imported directly, we mock at the fetch level instead (the agent also calls
+ * the Anthropic API, so we intercept there).
+ *
+ * For the smoke tests we only exercise criticReview directly in Tests 1-6, and
+ * use a simplified version of runGaiaAgentWithCritic that pre-supplies a
+ * mocked agent result rather than actually calling the API for the agent run.
+ *
+ * This avoids needing to rewrite the import mechanism just for smoke tests.
+ */
+// ---------------------------------------------------------------------------
+// Test 1: critic returns "pass" → no retry
+// ---------------------------------------------------------------------------
+async function test1_criticPass() {
+    console.log('\nTest 1: critic returns "pass" → no retry');
+    const restore = installFetchMock([
+        { ok: true, body: mockCriticResponse('pass', 'Answer is correct.') },
+    ]);
+    try {
+        const verdict = await criticReview(FIXTURE_QUESTION, 'Paris', { steps: [{ tool: 'web_search', result: 'Paris is the capital' }], turns: 2 }, { model: 'claude-sonnet-4-6', apiKey: 'test-key' });
+        assertEqual(verdict.verdict, 'pass', 'verdict is "pass"');
+        assert(verdict.reasoning.length > 0, 'reasoning is non-empty');
+        assert(verdict.costUsd >= 0, 'costUsd is non-negative');
+        assert(!verdict.error, 'no error flag');
+    }
+    finally {
+        restore();
+    }
+}
+// ---------------------------------------------------------------------------
+// Test 2: critic returns "fail" with suggestedRevision
+// ---------------------------------------------------------------------------
+async function test2_criticFail() {
+    console.log('\nTest 2: critic returns "fail" with suggestedRevision');
+    const restore = installFetchMock([
+        {
+            ok: true,
+            body: mockCriticResponse('fail', 'The answer is the wrong city.', 'Paris'),
+        },
+    ]);
+    try {
+        const verdict = await criticReview(FIXTURE_QUESTION, 'Lyon', { steps: [{ tool: 'web_search', result: 'Lyon is in France' }], turns: 1 }, { model: 'claude-sonnet-4-6', apiKey: 'test-key' });
+        assertEqual(verdict.verdict, 'fail', 'verdict is "fail"');
+        assert((verdict.suggestedRevision ?? '').length > 0, 'suggestedRevision is non-empty');
+        assertEqual(verdict.suggestedRevision, 'Paris', 'suggestedRevision is "Paris"');
+    }
+    finally {
+        restore();
+    }
+}
+// ---------------------------------------------------------------------------
+// Test 3: critic returns "fail" twice → retries exhausted
+// ---------------------------------------------------------------------------
+async function test3_retriesExhausted() {
+    console.log('\nTest 3: critic fails twice → retries exhausted, returns last candidate');
+    // We test this at the runGaiaAgentWithCritic level.
+    // We need to mock BOTH the critic fetch calls AND the agent API calls.
+    // Strategy: sequence the mock responses in the order they will be called.
+    //
+    // Call sequence (with enableCritic=true, maxRetries=1):
+    //   1. runGaiaAgent attempt 1   → agent Anthropic API call (returns answer "Lyon")
+    //   2. criticReview attempt 1   → critic API call (returns "fail")
+    //   3. runGaiaAgent attempt 2   → agent Anthropic API call (returns answer "Marseille")
+    //   4. criticReview attempt 2   → critic API call (returns "fail")
+    //
+    // For simplicity we make the agent calls also return valid Anthropic responses
+    // that produce a FINAL_ANSWER (the agent code parses stop_reason=end_turn).
+    const agentResponseLyon = {
+        id: 'msg_01',
+        type: 'message',
+        role: 'assistant',
+        stop_reason: 'end_turn',
+        content: [{ type: 'text', text: 'FINAL_ANSWER: Lyon' }],
+        usage: { input_tokens: 150, output_tokens: 20 },
+        model: 'claude-haiku-4-5',
+    };
+    const agentResponseMarseille = {
+        id: 'msg_02',
+        type: 'message',
+        role: 'assistant',
+        stop_reason: 'end_turn',
+        content: [{ type: 'text', text: 'FINAL_ANSWER: Marseille' }],
+        usage: { input_tokens: 150, output_tokens: 20 },
+        model: 'claude-haiku-4-5',
+    };
+    const restore = installFetchMock([
+        { ok: true, body: agentResponseLyon }, // agent call 1
+        { ok: true, body: mockCriticResponse('fail', 'Wrong city.', 'Paris') }, // critic 1
+        { ok: true, body: agentResponseMarseille }, // agent call 2 (retry)
+        { ok: true, body: mockCriticResponse('fail', 'Still wrong.', 'Paris') }, // critic 2
+    ]);
+    try {
+        const result = await runGaiaAgentWithCritic(FIXTURE_QUESTION, {
+            enableCritic: true,
+            apiKey: 'test-key',
+            criticOptions: { apiKey: 'test-key', maxRetries: 1 },
+        });
+        assertEqual(result.retriesAttempted, 1, 'retriesAttempted is 1');
+        assertEqual(result.criticVerdicts.length, 2, 'two critic verdicts collected');
+        assertEqual(result.criticVerdicts[0].verdict, 'fail', 'first verdict is fail');
+        assertEqual(result.criticVerdicts[1].verdict, 'fail', 'second verdict is fail');
+        // Last candidate is returned regardless.
+        assert(result.finalAnswer !== null, 'finalAnswer is non-null (last candidate returned)');
+    }
+    finally {
+        restore();
+    }
+}
+// ---------------------------------------------------------------------------
+// Test 4: critic returns "uncertain" → treated as "pass", no retry
+// ---------------------------------------------------------------------------
+async function test4_uncertainAsPass() {
+    console.log('\nTest 4: critic returns "uncertain" → treated as pass, no retry');
+    const restore = installFetchMock([
+        {
+            ok: true,
+            body: mockCriticResponse('uncertain', 'Cannot verify without more context.'),
+        },
+    ]);
+    try {
+        const verdict = await criticReview(FIXTURE_QUESTION, 'Paris', { steps: [], turns: 1 }, { model: 'claude-sonnet-4-6', apiKey: 'test-key' });
+        assertEqual(verdict.verdict, 'uncertain', 'verdict is "uncertain"');
+        // Verify orchestrator treats uncertain as pass (no retry fired).
+        // We simulate by checking the logic directly: uncertain !== 'fail', so loop
+        // body is skipped.  We test it via runGaiaAgentWithCritic with a minimal
+        // agent mock that returns a final answer.
+        const agentResponseParis = {
+            id: 'msg_03',
+            type: 'message',
+            role: 'assistant',
+            stop_reason: 'end_turn',
+            content: [{ type: 'text', text: 'FINAL_ANSWER: Paris' }],
+            usage: { input_tokens: 100, output_tokens: 15 },
+            model: 'claude-haiku-4-5',
+        };
+        const restore2 = installFetchMock([
+            { ok: true, body: agentResponseParis }, // agent call
+            { ok: true, body: mockCriticResponse('uncertain', 'Cannot verify.') }, // critic
+        ]);
+        try {
+            const result = await runGaiaAgentWithCritic(FIXTURE_QUESTION, {
+                enableCritic: true,
+                apiKey: 'test-key',
+                criticOptions: { apiKey: 'test-key', maxRetries: 1 },
+            });
+            assertEqual(result.retriesAttempted, 0, 'no retries for uncertain verdict');
+            assertEqual(result.criticVerdicts.length, 1, 'one critic verdict collected');
+            assertEqual(result.criticVerdicts[0].verdict, 'uncertain', 'verdict is uncertain');
+        }
+        finally {
+            restore2();
+        }
+    }
+    finally {
+        restore();
+    }
+}
+// ---------------------------------------------------------------------------
+// Test 5: API error in critic → graceful fallback
+// ---------------------------------------------------------------------------
+async function test5_apiErrorFallback() {
+    console.log('\nTest 5: API error in critic → graceful fallback, original candidate returned');
+    const restore = installFetchMock([
+        { ok: false, status: 529, text: 'Overloaded' },
+    ]);
+    try {
+        const verdict = await criticReview(FIXTURE_QUESTION, 'Paris', { steps: [], turns: 1 }, { model: 'claude-sonnet-4-6', apiKey: 'test-key' });
+        // On API error, critic returns uncertain with error flag — does not throw.
+        assertEqual(verdict.verdict, 'uncertain', 'verdict is "uncertain" on API error');
+        assert(verdict.error === true, 'error flag is set');
+        assert(verdict.costUsd === 0, 'costUsd is 0 on error');
+        assert(verdict.reasoning.includes('failed'), 'reasoning mentions failure');
+    }
+    finally {
+        restore();
+    }
+}
+// ---------------------------------------------------------------------------
+// Test 6: malformed JSON from critic → fallback parser
+// ---------------------------------------------------------------------------
+async function test6_malformedJson() {
+    console.log('\nTest 6: malformed JSON from critic → fallback parser extracts verdict');
+    // Simulate Sonnet returning prose with embedded JSON fragment.
+    const malformedBody = {
+        content: [{
+                type: 'text',
+                text: 'After careful review, I believe the answer is wrong. {"verdict":"fail","reasoning":"Incorrect city","suggestedRevision":"Paris"} The agent should try again.',
+            }],
+        usage: { input_tokens: 180, output_tokens: 60 },
+    };
+    const restore = installFetchMock([
+        { ok: true, body: malformedBody },
+    ]);
+    try {
+        const verdict = await criticReview(FIXTURE_QUESTION, 'Lyon', { steps: [], turns: 1 }, { model: 'claude-sonnet-4-6', apiKey: 'test-key' });
+        // Fallback parser should extract "fail" from the embedded JSON.
+        assertEqual(verdict.verdict, 'fail', 'fallback parser extracts "fail" verdict');
+        assert(!verdict.error, 'no error flag for recoverable parse');
+    }
+    finally {
+        restore();
+    }
+}
+// ---------------------------------------------------------------------------
+// Run all tests
+// ---------------------------------------------------------------------------
+async function main() {
+    console.log('=== GAIA Critic Smoke Tests (ADR-135 Track D) ===');
+    console.log('All tests use mocked responses — no live API calls.\n');
+    try {
+        await test1_criticPass();
+        await test2_criticFail();
+        await test3_retriesExhausted();
+        await test4_uncertainAsPass();
+        await test5_apiErrorFallback();
+        await test6_malformedJson();
+    }
+    catch (err) {
+        console.error('\nUnexpected test runner error:', err);
+        process.exit(1);
+    }
+    console.log(`\n=== Results: ${passed} passed, ${failed} failed ===`);
+    if (failed > 0) {
+        process.exit(1);
+    }
+}
+main().catch(err => {
+    console.error(err);
+    process.exit(1);
+});
+//# sourceMappingURL=gaia-critic.smoke.js.map