@claude-flow/cli 3.32.3 → 3.32.5

package/dist/src/benchmarks/gaia-hardness/train-data-loader.js

@@ -0,0 +1,179 @@
+/**
+ * GAIA Hardness Predictor — Training Data Loader (ADR-136 Track Q)
+ *
+ * Loads labelled training examples from prior bench-run result JSONs
+ * (iter-15, iter-23, iter-28 outputs) and converts them into the
+ * `LabeledExample[]` format consumed by `HardnessPredictor.train()`.
+ *
+ * Expected result JSON schema (matches gaia-bench --output json):
+ * {
+ *   level: number,
+ *   model: string,
+ *   summary: { total, passed, passRate, estCostUsd, meanTurns, meanWallMs },
+ *   results: [
+ *     {
+ *       task_id: string, question: string, model: string, correct: boolean,
+ *       answer: string | null, expected_output: string, error?: string,
+ *       turns?: number, wallMs?: number, inputTokens?: number, outputTokens?: number
+ *     }
+ *   ]
+ * }
+ *
+ * The file may contain either:
+ *   (a) a single JSON object (one model run), or
+ *   (b) a JSON array of objects (multi-model run from --models a,b,c), or
+ *   (c) a text preamble followed by JSON (raw output from gaia-bench text mode
+ *       — we scan for the first '[' or '{' and parse from there).
+ *
+ * Missing files are silently skipped (returns empty array).
+ * Malformed files emit a warning to stderr and are skipped.
+ *
+ * Default search paths (tried in order, first found wins per iter):
+ *   /tmp/gaia-l1-full.json
+ *   /tmp/gaia-l1-haiku.json
+ *   /tmp/gaia-all-p1b.json
+ *   /tmp/gaia-all-p2.json
+ *   <custom paths passed by caller>
+ *
+ * Refs: ADR-136, #2156
+ */
+import * as fs from 'node:fs';
+// ---------------------------------------------------------------------------
+// Default result-file search paths
+// ---------------------------------------------------------------------------
+/** Default candidate paths for historical bench-run result JSONs. */
+export const DEFAULT_RESULT_PATHS = [
+    '/tmp/gaia-l1-full.json',
+    '/tmp/gaia-l1-haiku.json',
+    '/tmp/gaia-all-p1b.json',
+    '/tmp/gaia-all-p2.json',
+    '/tmp/gaia-all-probe.json',
+];
+// ---------------------------------------------------------------------------
+// JSON extraction helper
+// ---------------------------------------------------------------------------
+/**
+ * Attempt to extract and parse a JSON value (object or array) from a string
+ * that may have a text preamble before the JSON.
+ *
+ * Strategy:
+ *   1. Find the first '[' — parse as array.
+ *   2. Find the first '{' — parse as object.
+ *   3. Return null on failure.
+ */
+function extractJson(content) {
+    const bracketIdx = content.indexOf('[');
+    const braceIdx = content.indexOf('{');
+    // Prefer whichever appears first (handles both array and object formats).
+    const candidates = [];
+    if (bracketIdx >= 0)
+        candidates.push([bracketIdx, '[']);
+    if (braceIdx >= 0)
+        candidates.push([braceIdx, '{']);
+    candidates.sort((a, b) => a[0] - b[0]);
+    for (const [startIdx] of candidates) {
+        try {
+            return JSON.parse(content.slice(startIdx));
+        }
+        catch {
+            // Try next candidate.
+        }
+    }
+    return null;
+}
+// ---------------------------------------------------------------------------
+// Parse a single file into RawBenchOutput[]
+// ---------------------------------------------------------------------------
+function parseBenchFile(filePath) {
+    let content;
+    try {
+        content = fs.readFileSync(filePath, 'utf-8');
+    }
+    catch {
+        return [];
+    }
+    const parsed = extractJson(content);
+    if (parsed === null) {
+        process.stderr.write(`[gaia-hardness] Warning: could not extract JSON from ${filePath}\n`);
+        return [];
+    }
+    // Normalise to array of RawBenchOutput.
+    const outputs = [];
+    if (Array.isArray(parsed)) {
+        for (const item of parsed) {
+            if (item && typeof item === 'object' && Array.isArray(item.results)) {
+                outputs.push(item);
+            }
+        }
+    }
+    else if (parsed !== null &&
+        typeof parsed === 'object' &&
+        Array.isArray(parsed.results)) {
+        outputs.push(parsed);
+    }
+    if (outputs.length === 0) {
+        process.stderr.write(`[gaia-hardness] Warning: no valid bench outputs found in ${filePath}\n`);
+    }
+    return outputs;
+}
+// ---------------------------------------------------------------------------
+// Convert RawQuestionResult → LabeledExample
+// ---------------------------------------------------------------------------
+function toGaiaQuestion(r) {
+    return {
+        task_id: r.task_id,
+        level: 1, // level not stored in result JSON; default to 1
+        question: r.question,
+        final_answer: r.expected_output,
+        file_name: null,
+        file_path: null,
+    };
+}
+function toBenchLabeledExample(r) {
+    return {
+        question: toGaiaQuestion(r),
+        wasCorrect: Boolean(r.correct),
+        turns: r.turns,
+    };
+}
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Load labelled training examples from historical bench-run result JSONs.
+ *
+ * @param additionalPaths - Extra file paths to scan beyond the defaults.
+ * @param verbose - If true, log loaded example counts to stderr.
+ * @returns Deduplicated array of LabeledExample (dedup by task_id, last write wins).
+ */
+export function loadTrainingData(additionalPaths = [], verbose = false) {
+    const allPaths = [...DEFAULT_RESULT_PATHS, ...additionalPaths];
+    const seen = new Set();
+    const examples = [];
+    for (const filePath of allPaths) {
+        if (!fs.existsSync(filePath))
+            continue;
+        const outputs = parseBenchFile(filePath);
+        let fileCount = 0;
+        for (const output of outputs) {
+            for (const result of output.results) {
+                if (!result.task_id || typeof result.correct !== 'boolean')
+                    continue;
+                const key = result.task_id;
+                if (seen.has(key))
+                    continue; // first file wins (chronological order)
+                seen.add(key);
+                examples.push(toBenchLabeledExample(result));
+                fileCount++;
+            }
+        }
+        if (verbose && fileCount > 0) {
+            process.stderr.write(`[gaia-hardness] Loaded ${fileCount} examples from ${filePath}\n`);
+        }
+    }
+    if (verbose) {
+        process.stderr.write(`[gaia-hardness] Total training examples: ${examples.length}\n`);
+    }
+    return examples;
+}
+//# sourceMappingURL=train-data-loader.js.map

package/dist/src/benchmarks/gaia-judge.d.ts

@@ -0,0 +1,88 @@
+/**
+ * GAIA Judge — ADR-133-PR6
+ *
+ * Two-stage answer scorer for the GAIA benchmark:
+ *
+ *   Stage 1 — Fast path: normalized exact-match.
+ *     Normalise = lowercase + strip surrounding whitespace + strip surrounding
+ *     single/double quotes + collapse internal whitespace runs to one space.
+ *     Roughly 30 % of GAIA Level-1 answers satisfy this; no API call required.
+ *
+ *   Stage 2 — LLM-as-judge: when exact-match fails, ask Claude Sonnet whether
+ *     the candidate answer is semantically equivalent to the ground truth.
+ *     The prompt embeds GAIA's official evaluation guideline (see
+ *     https://huggingface.co/datasets/gaia-benchmark/GAIA for full spec).
+ *
+ * Caching: judgment results are persisted under
+ *   ~/.cache/ruflo/gaia/judgments/<hash>.json
+ * keyed on (question_id, candidate_answer, model_id, JUDGE_PROMPT_VERSION).
+ * Re-running the same pair hits the cache and returns instantly.
+ *
+ * API pattern: raw fetch() against https://api.anthropic.com/v1/messages —
+ * mirrors the pattern established in gaia-agent.ts (ADR-133-PR3).
+ *
+ * Refs: ADR-133, #2156
+ */
+export interface JudgeResult {
+    questionId: string;
+    passed: boolean;
+    scoringPath: 'exact-match' | 'llm-judge' | 'cache';
+    candidateAnswer: string;
+    groundTruth: string;
+    judgeReason?: string;
+    judgeModel?: string;
+    judgeTokensIn?: number;
+    judgeTokensOut?: number;
+    judgeCostUsd?: number;
+}
+export interface JudgeOptions {
+    /** Default: 'claude-sonnet-4-6' */
+    judgeModel?: string;
+    /** Default: '~/.cache/ruflo/gaia/judgments/' */
+    cacheDir?: string;
+    skipCache?: boolean;
+    apiKey?: string;
+}
+/**
+ * GAIA normalisation as specified in the dataset paper:
+ *   - strip surrounding whitespace
+ *   - lowercase
+ *   - strip a single pair of surrounding quotes (single or double)
+ *   - collapse internal whitespace runs to one space
+ */
+export declare function normaliseAnswer(raw: string | null | undefined): string;
+/**
+ * Attempt to match a candidate numeric answer to an expected answer where the
+ * question implies a unit scale.
+ *
+ * Examples that this catches:
+ *   candidate="17000", expected="17", question contains "thousand"
+ *     → candidate / 1000 ≈ expected → MATCH
+ *   candidate="17", expected="17000", question contains "thousand"
+ *     → candidate × 1000 ≈ expected → MATCH (reverse direction)
+ *
+ * Returns true only when a numeric match is found under one of the scale
+ * multipliers mentioned in the question text.  Returns false for non-numeric
+ * inputs or when no multiplier matches.
+ *
+ * @param candidate    - The raw string from the model (may include commas/spaces).
+ * @param expected     - The raw ground-truth string.
+ * @param questionText - The original question (used to detect multiplier words).
+ */
+export declare function unitAwareNumberMatch(candidate: string, expected: string, questionText?: string): boolean;
+/**
+ * Judge a single GAIA answer.
+ *
+ * @param question   - Object with `id` (task_id), `expected` (ground truth),
+ *                     and optional `questionText` (the full question string,
+ *                     used for unit-aware numeric matching in Stage 1).
+ * @param candidateAnswer - The answer produced by the agent; `null` counts as a miss.
+ * @param options    - Optional overrides (model, cache dir, API key, etc.).
+ * @returns          - JudgeResult with pass/fail, scoring path, and cost metrics.
+ */
+export declare function judgeAnswer(question: {
+    id: string;
+    expected: string;
+    questionText?: string;
+}, candidateAnswer: string | null, options?: JudgeOptions): Promise<JudgeResult>;
+//# sourceMappingURL=gaia-judge.d.ts.map