@claude-flow/cli 3.32.2 → 3.32.3

package/dist/src/benchmarks/gaia-hardness/train-data-loader.js

@@ -1,179 +0,0 @@
-/**
- * GAIA Hardness Predictor — Training Data Loader (ADR-136 Track Q)
- *
- * Loads labelled training examples from prior bench-run result JSONs
- * (iter-15, iter-23, iter-28 outputs) and converts them into the
- * `LabeledExample[]` format consumed by `HardnessPredictor.train()`.
- *
- * Expected result JSON schema (matches gaia-bench --output json):
- * {
- *   level: number,
- *   model: string,
- *   summary: { total, passed, passRate, estCostUsd, meanTurns, meanWallMs },
- *   results: [
- *     {
- *       task_id: string, question: string, model: string, correct: boolean,
- *       answer: string | null, expected_output: string, error?: string,
- *       turns?: number, wallMs?: number, inputTokens?: number, outputTokens?: number
- *     }
- *   ]
- * }
- *
- * The file may contain either:
- *   (a) a single JSON object (one model run), or
- *   (b) a JSON array of objects (multi-model run from --models a,b,c), or
- *   (c) a text preamble followed by JSON (raw output from gaia-bench text mode
- *       — we scan for the first '[' or '{' and parse from there).
- *
- * Missing files are silently skipped (returns empty array).
- * Malformed files emit a warning to stderr and are skipped.
- *
- * Default search paths (tried in order, first found wins per iter):
- *   /tmp/gaia-l1-full.json
- *   /tmp/gaia-l1-haiku.json
- *   /tmp/gaia-all-p1b.json
- *   /tmp/gaia-all-p2.json
- *   <custom paths passed by caller>
- *
- * Refs: ADR-136, #2156
- */
-import * as fs from 'node:fs';
-// ---------------------------------------------------------------------------
-// Default result-file search paths
-// ---------------------------------------------------------------------------
-/** Default candidate paths for historical bench-run result JSONs. */
-export const DEFAULT_RESULT_PATHS = [
-    '/tmp/gaia-l1-full.json',
-    '/tmp/gaia-l1-haiku.json',
-    '/tmp/gaia-all-p1b.json',
-    '/tmp/gaia-all-p2.json',
-    '/tmp/gaia-all-probe.json',
-];
-// ---------------------------------------------------------------------------
-// JSON extraction helper
-// ---------------------------------------------------------------------------
-/**
- * Attempt to extract and parse a JSON value (object or array) from a string
- * that may have a text preamble before the JSON.
- *
- * Strategy:
- *   1. Find the first '[' — parse as array.
- *   2. Find the first '{' — parse as object.
- *   3. Return null on failure.
- */
-function extractJson(content) {
-    const bracketIdx = content.indexOf('[');
-    const braceIdx = content.indexOf('{');
-    // Prefer whichever appears first (handles both array and object formats).
-    const candidates = [];
-    if (bracketIdx >= 0)
-        candidates.push([bracketIdx, '[']);
-    if (braceIdx >= 0)
-        candidates.push([braceIdx, '{']);
-    candidates.sort((a, b) => a[0] - b[0]);
-    for (const [startIdx] of candidates) {
-        try {
-            return JSON.parse(content.slice(startIdx));
-        }
-        catch {
-            // Try next candidate.
-        }
-    }
-    return null;
-}
-// ---------------------------------------------------------------------------
-// Parse a single file into RawBenchOutput[]
-// ---------------------------------------------------------------------------
-function parseBenchFile(filePath) {
-    let content;
-    try {
-        content = fs.readFileSync(filePath, 'utf-8');
-    }
-    catch {
-        return [];
-    }
-    const parsed = extractJson(content);
-    if (parsed === null) {
-        process.stderr.write(`[gaia-hardness] Warning: could not extract JSON from ${filePath}\n`);
-        return [];
-    }
-    // Normalise to array of RawBenchOutput.
-    const outputs = [];
-    if (Array.isArray(parsed)) {
-        for (const item of parsed) {
-            if (item && typeof item === 'object' && Array.isArray(item.results)) {
-                outputs.push(item);
-            }
-        }
-    }
-    else if (parsed !== null &&
-        typeof parsed === 'object' &&
-        Array.isArray(parsed.results)) {
-        outputs.push(parsed);
-    }
-    if (outputs.length === 0) {
-        process.stderr.write(`[gaia-hardness] Warning: no valid bench outputs found in ${filePath}\n`);
-    }
-    return outputs;
-}
-// ---------------------------------------------------------------------------
-// Convert RawQuestionResult → LabeledExample
-// ---------------------------------------------------------------------------
-function toGaiaQuestion(r) {
-    return {
-        task_id: r.task_id,
-        level: 1, // level not stored in result JSON; default to 1
-        question: r.question,
-        final_answer: r.expected_output,
-        file_name: null,
-        file_path: null,
-    };
-}
-function toBenchLabeledExample(r) {
-    return {
-        question: toGaiaQuestion(r),
-        wasCorrect: Boolean(r.correct),
-        turns: r.turns,
-    };
-}
-// ---------------------------------------------------------------------------
-// Public API
-// ---------------------------------------------------------------------------
-/**
- * Load labelled training examples from historical bench-run result JSONs.
- *
- * @param additionalPaths - Extra file paths to scan beyond the defaults.
- * @param verbose - If true, log loaded example counts to stderr.
- * @returns Deduplicated array of LabeledExample (dedup by task_id, last write wins).
- */
-export function loadTrainingData(additionalPaths = [], verbose = false) {
-    const allPaths = [...DEFAULT_RESULT_PATHS, ...additionalPaths];
-    const seen = new Set();
-    const examples = [];
-    for (const filePath of allPaths) {
-        if (!fs.existsSync(filePath))
-            continue;
-        const outputs = parseBenchFile(filePath);
-        let fileCount = 0;
-        for (const output of outputs) {
-            for (const result of output.results) {
-                if (!result.task_id || typeof result.correct !== 'boolean')
-                    continue;
-                const key = result.task_id;
-                if (seen.has(key))
-                    continue; // first file wins (chronological order)
-                seen.add(key);
-                examples.push(toBenchLabeledExample(result));
-                fileCount++;
-            }
-        }
-        if (verbose && fileCount > 0) {
-            process.stderr.write(`[gaia-hardness] Loaded ${fileCount} examples from ${filePath}\n`);
-        }
-    }
-    if (verbose) {
-        process.stderr.write(`[gaia-hardness] Total training examples: ${examples.length}\n`);
-    }
-    return examples;
-}
-//# sourceMappingURL=train-data-loader.js.map

package/dist/src/benchmarks/gaia-judge.d.ts

@@ -1,88 +0,0 @@
-/**
- * GAIA Judge — ADR-133-PR6
- *
- * Two-stage answer scorer for the GAIA benchmark:
- *
- *   Stage 1 — Fast path: normalized exact-match.
- *     Normalise = lowercase + strip surrounding whitespace + strip surrounding
- *     single/double quotes + collapse internal whitespace runs to one space.
- *     Roughly 30 % of GAIA Level-1 answers satisfy this; no API call required.
- *
- *   Stage 2 — LLM-as-judge: when exact-match fails, ask Claude Sonnet whether
- *     the candidate answer is semantically equivalent to the ground truth.
- *     The prompt embeds GAIA's official evaluation guideline (see
- *     https://huggingface.co/datasets/gaia-benchmark/GAIA for full spec).
- *
- * Caching: judgment results are persisted under
- *   ~/.cache/ruflo/gaia/judgments/<hash>.json
- * keyed on (question_id, candidate_answer, model_id, JUDGE_PROMPT_VERSION).
- * Re-running the same pair hits the cache and returns instantly.
- *
- * API pattern: raw fetch() against https://api.anthropic.com/v1/messages —
- * mirrors the pattern established in gaia-agent.ts (ADR-133-PR3).
- *
- * Refs: ADR-133, #2156
- */
-export interface JudgeResult {
-    questionId: string;
-    passed: boolean;
-    scoringPath: 'exact-match' | 'llm-judge' | 'cache';
-    candidateAnswer: string;
-    groundTruth: string;
-    judgeReason?: string;
-    judgeModel?: string;
-    judgeTokensIn?: number;
-    judgeTokensOut?: number;
-    judgeCostUsd?: number;
-}
-export interface JudgeOptions {
-    /** Default: 'claude-sonnet-4-6' */
-    judgeModel?: string;
-    /** Default: '~/.cache/ruflo/gaia/judgments/' */
-    cacheDir?: string;
-    skipCache?: boolean;
-    apiKey?: string;
-}
-/**
- * GAIA normalisation as specified in the dataset paper:
- *   - strip surrounding whitespace
- *   - lowercase
- *   - strip a single pair of surrounding quotes (single or double)
- *   - collapse internal whitespace runs to one space
- */
-export declare function normaliseAnswer(raw: string | null | undefined): string;
-/**
- * Attempt to match a candidate numeric answer to an expected answer where the
- * question implies a unit scale.
- *
- * Examples that this catches:
- *   candidate="17000", expected="17", question contains "thousand"
- *     → candidate / 1000 ≈ expected → MATCH
- *   candidate="17", expected="17000", question contains "thousand"
- *     → candidate × 1000 ≈ expected → MATCH (reverse direction)
- *
- * Returns true only when a numeric match is found under one of the scale
- * multipliers mentioned in the question text.  Returns false for non-numeric
- * inputs or when no multiplier matches.
- *
- * @param candidate    - The raw string from the model (may include commas/spaces).
- * @param expected     - The raw ground-truth string.
- * @param questionText - The original question (used to detect multiplier words).
- */
-export declare function unitAwareNumberMatch(candidate: string, expected: string, questionText?: string): boolean;
-/**
- * Judge a single GAIA answer.
- *
- * @param question   - Object with `id` (task_id), `expected` (ground truth),
- *                     and optional `questionText` (the full question string,
- *                     used for unit-aware numeric matching in Stage 1).
- * @param candidateAnswer - The answer produced by the agent; `null` counts as a miss.
- * @param options    - Optional overrides (model, cache dir, API key, etc.).
- * @returns          - JudgeResult with pass/fail, scoring path, and cost metrics.
- */
-export declare function judgeAnswer(question: {
-    id: string;
-    expected: string;
-    questionText?: string;
-}, candidateAnswer: string | null, options?: JudgeOptions): Promise<JudgeResult>;
-//# sourceMappingURL=gaia-judge.d.ts.map