@claude-flow/cli 3.10.3 → 3.10.5

package/dist/src/benchmarks/gaia-hardness/predictor.smoke.js

@@ -0,0 +1,235 @@
+/**
+ * Smoke tests for gaia-hardness predictor — ADR-136 Track Q
+ *
+ * All tests are mock-based (no live API calls, no HF token, $0 cost).
+ *
+ * Test matrix:
+ *   1. Feature extraction: math question fires requiresMath + qword_calc
+ *   2. Feature extraction: simple factual question has low complexity signal
+ *   3. Feature extraction: question with file attachment fires has_file_attachment
+ *   4. Feature extraction: multi-hop question fires multi_hop_signal
+ *   5. Cold-start fallback: untrained predictor returns medium with confidence=0.5
+ *   6. Train on mock data: predictor becomes trained after ≥ 10 examples
+ *   7. Trained predictor: math-heavy question predicts harder than trivial question
+ *   8. Export/import: weights round-trip through JSON without loss
+ *
+ * Refs: ADR-136, #2156
+ */
+import assert from 'node:assert/strict';
+import { extractFeatures, FEATURE_LABELS } from './features.js';
+import { HardnessPredictor } from './predictor.js';
+// ---------------------------------------------------------------------------
+// Fixture helpers
+// ---------------------------------------------------------------------------
+function makeQuestion(text, overrides = {}) {
+    return {
+        task_id: 'smoke-test',
+        level: 1,
+        question: text,
+        final_answer: 'X',
+        file_name: null,
+        file_path: null,
+        ...overrides,
+    };
+}
+function labelIndex(label) {
+    return FEATURE_LABELS.indexOf(label);
+}
+// ---------------------------------------------------------------------------
+// Test 1: Math question fires expected features
+// ---------------------------------------------------------------------------
+function testMathQuestionFeatures() {
+    const q = makeQuestion('What is 15 multiplied by 4? Calculate the exact product.');
+    const fv = extractFeatures(q);
+    const calcIdx = labelIndex('qword_calc_compute');
+    const mathIdx = labelIndex('requires_math');
+    const numIdx = labelIndex('has_numeric');
+    assert.ok(calcIdx >= 0, 'qword_calc_compute feature not found');
+    assert.ok(mathIdx >= 0, 'requires_math feature not found');
+    assert.ok(numIdx >= 0, 'has_numeric feature not found');
+    assert.equal(fv.values[numIdx], 1, 'has_numeric should be 1 for "15 multiplied by 4"');
+    // "Calculate" or "what" triggers calc compute or requires_math
+    const eitherMath = fv.values[calcIdx] === 1 || fv.values[mathIdx] === 1;
+    assert.ok(eitherMath, 'math question should fire qword_calc_compute or requires_math');
+    console.log('  [PASS] Test 1: math question fires expected features');
+}
+// ---------------------------------------------------------------------------
+// Test 2: Simple factual question has low complexity
+// ---------------------------------------------------------------------------
+function testSimpleFactualQuestion() {
+    const q = makeQuestion('What is the capital of France?');
+    const fv = extractFeatures(q);
+    assert.equal(fv.values.length, 17, 'feature vector should have 17 dims');
+    const multiHopIdx = labelIndex('multi_hop_signal');
+    const toolIdx = labelIndex('tool_implication_norm');
+    const fileIdx = labelIndex('has_file_attachment');
+    assert.equal(fv.values[multiHopIdx], 0, 'simple question should have no multi-hop signal');
+    assert.equal(fv.values[toolIdx], 0, 'simple question should have no tool implication');
+    assert.equal(fv.values[fileIdx], 0, 'simple question should have no file attachment');
+    // Word count should be normalised well below 1
+    const wordNormIdx = labelIndex('len_words_norm');
+    assert.ok(fv.values[wordNormIdx] < 0.3, 'short question should have normalised word count < 0.3');
+    console.log('  [PASS] Test 2: simple factual question has low complexity signal');
+}
+// ---------------------------------------------------------------------------
+// Test 3: File attachment fires has_file_attachment
+// ---------------------------------------------------------------------------
+function testFileAttachmentFeature() {
+    const q = makeQuestion('Analyse the attached PDF document and extract the table.', {
+        file_name: 'document.pdf',
+        file_path: '/tmp/document.pdf',
+    });
+    const fv = extractFeatures(q);
+    const fileIdx = labelIndex('has_file_attachment');
+    const toolIdx = labelIndex('tool_implication_norm');
+    assert.equal(fv.values[fileIdx], 1, 'question with file_name should have has_file_attachment=1');
+    assert.ok(fv.values[toolIdx] > 0, 'PDF question should have tool_implication > 0');
+    console.log('  [PASS] Test 3: file attachment fires has_file_attachment');
+}
+// ---------------------------------------------------------------------------
+// Test 4: Multi-hop question fires multi_hop_signal
+// ---------------------------------------------------------------------------
+function testMultiHopFeature() {
+    const q = makeQuestion('Who was the president of the country that won the most gold medals at the Olympics that was held in the city that hosted the 1936 Games?');
+    const fv = extractFeatures(q);
+    const multiHopIdx = labelIndex('multi_hop_signal');
+    assert.equal(fv.values[multiHopIdx], 1, 'multi-hop question should fire multi_hop_signal');
+    console.log('  [PASS] Test 4: multi-hop question fires multi_hop_signal');
+}
+// ---------------------------------------------------------------------------
+// Test 5: Cold-start fallback returns medium with confidence=0.5
+// ---------------------------------------------------------------------------
+function testColdStartFallback() {
+    const predictor = new HardnessPredictor();
+    assert.equal(predictor.isTrained, false, 'new predictor should not be trained');
+    const q = makeQuestion('What is the capital of France?');
+    const result = predictor.predict(q);
+    assert.equal(result.difficulty, 'medium', 'cold-start should return medium');
+    assert.equal(result.confidence, 0.5, 'cold-start should return confidence=0.5');
+    assert.equal(result.budget.model, 'sonnet', 'medium budget should use sonnet');
+    assert.equal(result.budget.maxTurns, 8, 'medium budget should have maxTurns=8');
+    assert.equal(result.budget.votingAttempts, 1, 'medium budget should have votingAttempts=1');
+    console.log('  [PASS] Test 5: cold-start fallback returns medium with confidence=0.5');
+}
+// ---------------------------------------------------------------------------
+// Test 6: Becomes trained after ≥ 10 labelled examples
+// ---------------------------------------------------------------------------
+function testBecomesTrainedWith10Examples() {
+    const predictor = new HardnessPredictor();
+    // Generate 15 mock examples (mix of correct/incorrect)
+    const examples = [];
+    for (let i = 0; i < 15; i++) {
+        examples.push({
+            question: makeQuestion(`Mock question number ${i} about topic ${i % 3}`),
+            wasCorrect: i % 3 !== 0,
+            turns: 3 + (i % 6),
+        });
+    }
+    assert.equal(predictor.isTrained, false, 'should not be trained before training');
+    predictor.train(examples);
+    assert.equal(predictor.isTrained, true, 'should be trained after ≥10 examples');
+    console.log('  [PASS] Test 6: predictor becomes trained after ≥ 10 examples');
+}
+// ---------------------------------------------------------------------------
+// Test 7: Trained predictor scores math question harder than trivial question
+// ---------------------------------------------------------------------------
+function testTrainedRelativeHardness() {
+    const predictor = new HardnessPredictor({ conservativeMode: false });
+    // Train on 20 examples with clear pattern:
+    // - Short simple questions → correct (easy)
+    // - Long multi-step questions → incorrect (hard)
+    const examples = [];
+    for (let i = 0; i < 10; i++) {
+        examples.push({
+            question: makeQuestion(`What is the capital of country ${i}?`),
+            wasCorrect: true,
+            turns: 2,
+        });
+        examples.push({
+            question: makeQuestion(`Calculate the exact percentage difference between ${i * 100} and ${i * 100 + 50}, ` +
+                `then multiply by the number of years between 1990 and 2023. ` +
+                `Express as a decimal rounded to 3 places.`),
+            wasCorrect: false,
+            turns: 12,
+        });
+    }
+    predictor.train(examples);
+    const trivialQ = makeQuestion('What is the capital of France?');
+    const mathQ = makeQuestion('Calculate the exact percentage difference between 750 and 1250, ' +
+        'then multiply by the number of years between 1990 and 2023. ' +
+        'Express as a decimal rounded to 3 places.');
+    const trivialResult = predictor.predict(trivialQ);
+    const mathResult = predictor.predict(mathQ);
+    // The difficulty ordering should not put math as easier than trivial.
+    const ORDER = { easy: 0, medium: 1, hard: 2 };
+    assert.ok(ORDER[mathResult.difficulty] >= ORDER[trivialResult.difficulty], `math question (${mathResult.difficulty}) should be ≥ trivial (${trivialResult.difficulty})`);
+    console.log(`  [PASS] Test 7: math="${mathResult.difficulty}" (conf=${mathResult.confidence.toFixed(2)}) ≥ trivial="${trivialResult.difficulty}" (conf=${trivialResult.confidence.toFixed(2)})`);
+}
+// ---------------------------------------------------------------------------
+// Test 8: Weights round-trip through JSON export/import
+// ---------------------------------------------------------------------------
+function testWeightRoundTrip() {
+    const predictor = new HardnessPredictor();
+    const examples = [];
+    for (let i = 0; i < 12; i++) {
+        examples.push({
+            question: makeQuestion(`Question ${i}`),
+            wasCorrect: i % 2 === 0,
+            turns: 4 + i,
+        });
+    }
+    predictor.train(examples);
+    const exported = predictor.export();
+    assert.ok(exported !== null, 'exported state should not be null after training');
+    const predictor2 = new HardnessPredictor();
+    predictor2.import(exported);
+    assert.equal(predictor2.isTrained, true, 'imported predictor should be trained');
+    // Both predictors should give the same result for the same question.
+    const q = makeQuestion('How many sides does a hexagon have?');
+    const r1 = predictor.predict(q);
+    const r2 = predictor2.predict(q);
+    assert.equal(r1.difficulty, r2.difficulty, 'exported/imported predictor should give same difficulty');
+    assert.ok(Math.abs(r1.confidence - r2.confidence) < 0.001, 'exported/imported predictor should give same confidence');
+    console.log('  [PASS] Test 8: weights round-trip through export/import without loss');
+}
+// ---------------------------------------------------------------------------
+// Runner
+// ---------------------------------------------------------------------------
+async function runSmoke() {
+    console.log('\n=== gaia-hardness predictor.smoke.ts — ADR-136 Track Q ===');
+    console.log('(mock-based, no live API, $0 cost)\n');
+    let passed = 0;
+    let total = 0;
+    function run(name, fn) {
+        total++;
+        try {
+            fn();
+            passed++;
+        }
+        catch (err) {
+            console.error(`  [FAIL] ${name}: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    run('Feature extraction: math question', testMathQuestionFeatures);
+    run('Feature extraction: simple factual', testSimpleFactualQuestion);
+    run('Feature extraction: file attachment', testFileAttachmentFeature);
+    run('Feature extraction: multi-hop signal', testMultiHopFeature);
+    run('Cold-start fallback: medium + confidence=0.5', testColdStartFallback);
+    run('Train: becomes trained with ≥10 examples', testBecomesTrainedWith10Examples);
+    run('Trained: math harder than trivial', testTrainedRelativeHardness);
+    run('Export/import: weights round-trip', testWeightRoundTrip);
+    console.log(`\n=== Summary: ${passed}/${total} tests passed ===`);
+    if (passed < total) {
+        process.exit(1);
+    }
+}
+// Run when invoked directly.
+if (process.argv[1]?.endsWith('predictor.smoke.js') ||
+    process.argv[1]?.endsWith('predictor.smoke.ts')) {
+    runSmoke().catch((err) => {
+        console.error('Smoke crashed:', err);
+        process.exit(2);
+    });
+}
+export { runSmoke };
+//# sourceMappingURL=predictor.smoke.js.map

package/dist/src/benchmarks/gaia-hardness/predictor.smoke.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"predictor.smoke.js","sourceRoot":"","sources":["../../../../src/benchmarks/gaia-hardness/predictor.smoke.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAA6C,MAAM,gBAAgB,CAAC;AAG9F,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E,SAAS,YAAY,CAAC,IAAY,EAAE,YAAmC,EAAE;IACvE,OAAO;QACL,OAAO,EAAE,YAAY;QACrB,KAAK,EAAE,CAAC;QACR,QAAQ,EAAE,IAAI;QACd,YAAY,EAAE,GAAG;QACjB,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,IAAI;QACf,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AACvC,CAAC;AAED,8EAA8E;AAC9E,gDAAgD;AAChD,8EAA8E;AAE9E,SAAS,wBAAwB;IAC/B,MAAM,CAAC,GAAG,YAAY,CACpB,0DAA0D,CAC3D,CAAC;IACF,MAAM,EAAE,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;IAE9B,MAAM,OAAO,GAAG,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACjD,MAAM,OAAO,GAAG,UAAU,CAAC,eAAe,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,UAAU,CAAC,aAAa,CAAC,CAAC;IAEzC,MAAM,CAAC,EAAE,CAAC,OAAO,IAAI,CAAC,EAAE,sCAAsC,CAAC,CAAC;IAChE,MAAM,CAAC,EAAE,CAAC,OAAO,IAAI,CAAC,EAAE,iCAAiC,CAAC,CAAC;IAC3D,MAAM,CAAC,EAAE,CAAC,MAAM,IAAI,CAAC,EAAE,+BAA+B,CAAC,CAAC;IAExD,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,kDAAkD,CAAC,CAAC;IACvF,+DAA+D;IAC/D,MAAM,UAAU,GAAG,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACxE,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,+DAA+D,CAAC,CAAC;IAEvF,OAAO,CAAC,GAAG,CAAC,wDAAwD,CAAC,CAAC;AACxE,CAAC;AAED,8EAA8E;AAC9E,qDAAqD;AACrD,8EAA8E;AAE9E,SAAS,yBAAyB;IAChC,MAAM,CAAC,GAAG,YAAY,CAAC,gCAAgC,CAAC,CAAC;IACzD,MAAM,EAAE,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;IAE9B,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,EAAE,oCAAoC,CAAC,CAAC;IAEzE,MAAM,WAAW,GAAG,UAAU,CAAC,kBAAkB,CAAC,CAAC;IACnD,MAAM,OAAO,GAAG,UAAU,CAAC,uBAAuB,CAAC,CAAC;IACpD,MAAM,OAAO,GAAG,UAAU,CAAC,qBAAqB,CAAC,CAAC;IAElD,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,iDAAiD,CAAC,CAAC;IAC3F,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,iDAAiD,CAAC,CAAC;IACvF,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,gDAAgD,CAAC,CAAC;IAEtF,+CAA+C;IAC/C,MAAM,WAAW,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC;IACjD,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,GAAG,EAAE,wDAAwD,CAAC,CAAC;IAElG,OAAO,CAAC,GAAG,CAAC,oEAAoE,CAAC,CAAC;AACpF,CAAC;AAED,8EAA8E;AAC9E,oDAAoD;AACpD,8EAA8E;AAE9E,SAAS,yBAAyB;IAChC,MAAM,CAAC,GAAG,YAAY,CAAC,0DAA0D,EAAE;QACjF,SAAS,EAAE,cAAc;QACzB,SAAS,EAAE,mBAAmB;KAC/B,CAAC,CAAC;IACH,MAAM,EAAE,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;IAE9B,MAAM,OAAO,GAAG,UAAU,CAAC,qBAAqB,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,UAAU,CAAC,uBAAuB,CAAC,CAAC;IAEpD,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,2DAA2D,CAAC,CAAC;IACjG,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,+CAA+C,CAAC,CAAC;IAEnF,OAAO,CAAC,GAAG,CAAC,4DAA4D,CAAC,CAAC;AAC5E,CAAC;AAED,8EAA8E;AAC9E,oDAAoD;AACpD,8EAA8E;AAE9E,SAAS,mBAAmB;IAC1B,MAAM,CAAC,GAAG,YAAY,CACpB,0IAA0I,CAC3I,CAAC;IACF,MAAM,EAAE,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;IAE9B,MAAM,WAAW,GAAG,UAAU,CAAC,kBAAkB,CAAC,CAAC;IACnD,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,iDAAiD,CAAC,CAAC;IAE3F,OAAO,CAAC,GAAG,CAAC,4DAA4D,CAAC,CAAC;AAC5E,CAAC;AAED,8EAA8E;AAC9E,iEAAiE;AACjE,8EAA8E;AAE9E,SAAS,qBAAqB;IAC5B,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,SAAS,EAAE,KAAK,EAAE,qCAAqC,CAAC,CAAC;IAEhF,MAAM,CAAC,GAAG,YAAY,CAAC,gCAAgC,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAEpC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,QAAQ,EAAE,iCAAiC,CAAC,CAAC;IAC7E,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,GAAG,EAAE,yCAAyC,CAAC,CAAC;IAChF,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,iCAAiC,CAAC,CAAC;IAC/E,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,EAAE,sCAAsC,CAAC,CAAC;IAChF,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC,EAAE,4CAA4C,CAAC,CAAC;IAE5F,OAAO,CAAC,GAAG,CAAC,yEAAyE,CAAC,CAAC;AACzF,CAAC;AAED,8EAA8E;AAC9E,uDAAuD;AACvD,8EAA8E;AAE9E,SAAS,gCAAgC;IACvC,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAE1C,uDAAuD;IACvD,MAAM,QAAQ,GAAqB,EAAE,CAAC;IACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,QAAQ,CAAC,IAAI,CAAC;YACZ,QAAQ,EAAE,YAAY,CAAC,wBAAwB,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;YACxE,UAAU,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC;YACvB,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;SACnB,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,SAAS,EAAE,KAAK,EAAE,uCAAuC,CAAC,CAAC;IAClF,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC1B,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,sCAAsC,CAAC,CAAC;IAEhF,OAAO,CAAC,GAAG,CAAC,gEAAgE,CAAC,CAAC;AAChF,CAAC;AAED,8EAA8E;AAC9E,8EAA8E;AAC9E,8EAA8E;AAE9E,SAAS,2BAA2B;IAClC,MAAM,SAAS,GAAG,IAAI,iBAAiB,CAAC,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAAC,CAAC;IAErE,2CAA2C;IAC3C,4CAA4C;IAC5C,iDAAiD;IACjD,MAAM,QAAQ,GAAqB,EAAE,CAAC;IACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,QAAQ,CAAC,IAAI,CAAC;YACZ,QAAQ,EAAE,YAAY,CAAC,kCAAkC,CAAC,GAAG,CAAC;YAC9D,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,QAAQ,CAAC,IAAI,CAAC;YACZ,QAAQ,EAAE,YAAY,CACpB,qDAAqD,CAAC,GAAG,GAAG,QAAQ,CAAC,GAAG,GAAG,GAAG,EAAE,IAAI;gBACpF,8DAA8D;gBAC9D,2CAA2C,CAC5C;YACD,UAAU,EAAE,KAAK;YACjB,KAAK,EAAE,EAAE;SACV,CAAC,CAAC;IACL,CAAC;IAED,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAE1B,MAAM,QAAQ,GAAG,YAAY,CAAC,gCAAgC,CAAC,CAAC;IAChE,MAAM,KAAK,GAAG,YAAY,CACxB,kEAAkE;QAClE,8DAA8D;QAC9D,2CAA2C,CAC5C,CAAC;IAEF,MAAM,aAAa,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAClD,MAAM,UAAU,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAE5C,sEAAsE;IACtE,MAAM,KAAK,GAAoC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IAC/E,MAAM,CAAC,EAAE,CACP,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,aAAa,CAAC,UAAU,CAAC,EAC/D,kBAAkB,UAAU,CAAC,UAAU,0BAA0B,aAAa,CAAC,UAAU,GAAG,CAC7F,CAAC;IAEF,OAAO,CAAC,GAAG,CACT,0BAA0B,UAAU,CAAC,UAAU,WAAW,UAAU,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,gBAAgB,aAAa,CAAC,UAAU,WAAW,aAAa,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACpL,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,wDAAwD;AACxD,8EAA8E;AAE9E,SAAS,mBAAmB;IAC1B,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAE1C,MAAM,QAAQ,GAAqB,EAAE,CAAC;IACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,QAAQ,CAAC,IAAI,CAAC;YACZ,QAAQ,EAAE,YAAY,CAAC,YAAY,CAAC,EAAE,CAAC;YACvC,UAAU,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC;YACvB,KAAK,EAAE,CAAC,GAAG,CAAC;SACb,CAAC,CAAC;IACL,CAAC;IACD,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAE1B,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC;IACpC,MAAM,CAAC,EAAE,CAAC,QAAQ,KAAK,IAAI,EAAE,kDAAkD,CAAC,CAAC;IAEjF,MAAM,UAAU,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC3C,UAAU,CAAC,MAAM,CAAC,QAAS,CAAC,CAAC;IAC7B,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,EAAE,IAAI,EAAE,sCAAsC,CAAC,CAAC;IAEjF,qEAAqE;IACrE,MAAM,CAAC,GAAG,YAAY,CAAC,qCAAqC,CAAC,CAAC;IAC9D,MAAM,EAAE,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAChC,MAAM,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACjC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,EAAE,yDAAyD,CAAC,CAAC;IACtG,MAAM,CAAC,EAAE,CACP,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,GAAG,KAAK,EAC/C,yDAAyD,CAC1D,CAAC;IAEF,OAAO,CAAC,GAAG,CAAC,wEAAwE,CAAC,CAAC;AACxF,CAAC;AAED,8EAA8E;AAC9E,SAAS;AACT,8EAA8E;AAE9E,KAAK,UAAU,QAAQ;IACrB,OAAO,CAAC,GAAG,CAAC,8DAA8D,CAAC,CAAC;IAC5E,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;IAEpD,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,SAAS,GAAG,CAAC,IAAY,EAAE,EAAc;QACvC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC;YACH,EAAE,EAAE,CAAC;YACL,MAAM,EAAE,CAAC;QACX,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,YAAY,IAAI,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzF,CAAC;IACH,CAAC;IAED,GAAG,CAAC,mCAAmC,EAAE,wBAAwB,CAAC,CAAC;IACnE,GAAG,CAAC,oCAAoC,EAAE,yBAAyB,CAAC,CAAC;IACrE,GAAG,CAAC,qCAAqC,EAAE,yBAAyB,CAAC,CAAC;IACtE,GAAG,CAAC,sCAAsC,EAAE,mBAAmB,CAAC,CAAC;IACjE,GAAG,CAAC,8CAA8C,EAAE,qBAAqB,CAAC,CAAC;IAC3E,GAAG,CAAC,0CAA0C,EAAE,gCAAgC,CAAC,CAAC;IAClF,GAAG,CAAC,mCAAmC,EAAE,2BAA2B,CAAC,CAAC;IACtE,GAAG,CAAC,mCAAmC,EAAE,mBAAmB,CAAC,CAAC;IAE9D,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,IAAI,KAAK,mBAAmB,CAAC,CAAC;IAElE,IAAI,MAAM,GAAG,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,6BAA6B;AAC7B,IACE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,oBAAoB,CAAC;IAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,oBAAoB,CAAC,EAC/C,CAAC;IACD,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACvB,OAAO,CAAC,KAAK,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC;QACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,OAAO,EAAE,QAAQ,EAAE,CAAC"}

package/dist/src/benchmarks/gaia-hardness/train-data-loader.d.ts

@@ -0,0 +1,51 @@
+/**
+ * GAIA Hardness Predictor — Training Data Loader (ADR-136 Track Q)
+ *
+ * Loads labelled training examples from prior bench-run result JSONs
+ * (iter-15, iter-23, iter-28 outputs) and converts them into the
+ * `LabeledExample[]` format consumed by `HardnessPredictor.train()`.
+ *
+ * Expected result JSON schema (matches gaia-bench --output json):
+ * {
+ *   level: number,
+ *   model: string,
+ *   summary: { total, passed, passRate, estCostUsd, meanTurns, meanWallMs },
+ *   results: [
+ *     {
+ *       task_id: string, question: string, model: string, correct: boolean,
+ *       answer: string | null, expected_output: string, error?: string,
+ *       turns?: number, wallMs?: number, inputTokens?: number, outputTokens?: number
+ *     }
+ *   ]
+ * }
+ *
+ * The file may contain either:
+ *   (a) a single JSON object (one model run), or
+ *   (b) a JSON array of objects (multi-model run from --models a,b,c), or
+ *   (c) a text preamble followed by JSON (raw output from gaia-bench text mode
+ *       — we scan for the first '[' or '{' and parse from there).
+ *
+ * Missing files are silently skipped (returns empty array).
+ * Malformed files emit a warning to stderr and are skipped.
+ *
+ * Default search paths (tried in order, first found wins per iter):
+ *   /tmp/gaia-l1-full.json
+ *   /tmp/gaia-l1-haiku.json
+ *   /tmp/gaia-all-p1b.json
+ *   /tmp/gaia-all-p2.json
+ *   <custom paths passed by caller>
+ *
+ * Refs: ADR-136, #2156
+ */
+import type { LabeledExample } from './predictor.js';
+/** Default candidate paths for historical bench-run result JSONs. */
+export declare const DEFAULT_RESULT_PATHS: readonly string[];
+/**
+ * Load labelled training examples from historical bench-run result JSONs.
+ *
+ * @param additionalPaths - Extra file paths to scan beyond the defaults.
+ * @param verbose - If true, log loaded example counts to stderr.
+ * @returns Deduplicated array of LabeledExample (dedup by task_id, last write wins).
+ */
+export declare function loadTrainingData(additionalPaths?: string[], verbose?: boolean): LabeledExample[];
+//# sourceMappingURL=train-data-loader.d.ts.map

package/dist/src/benchmarks/gaia-hardness/train-data-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"train-data-loader.d.ts","sourceRoot":"","sources":["../../../../src/benchmarks/gaia-hardness/train-data-loader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAIH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAMrD,qEAAqE;AACrE,eAAO,MAAM,oBAAoB,EAAE,SAAS,MAAM,EAMxC,CAAC;AAqIX;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC9B,eAAe,GAAE,MAAM,EAAO,EAC9B,OAAO,UAAQ,GACd,cAAc,EAAE,CAoClB"}

package/dist/src/benchmarks/gaia-hardness/train-data-loader.js

@@ -0,0 +1,179 @@
+/**
+ * GAIA Hardness Predictor — Training Data Loader (ADR-136 Track Q)
+ *
+ * Loads labelled training examples from prior bench-run result JSONs
+ * (iter-15, iter-23, iter-28 outputs) and converts them into the
+ * `LabeledExample[]` format consumed by `HardnessPredictor.train()`.
+ *
+ * Expected result JSON schema (matches gaia-bench --output json):
+ * {
+ *   level: number,
+ *   model: string,
+ *   summary: { total, passed, passRate, estCostUsd, meanTurns, meanWallMs },
+ *   results: [
+ *     {
+ *       task_id: string, question: string, model: string, correct: boolean,
+ *       answer: string | null, expected_output: string, error?: string,
+ *       turns?: number, wallMs?: number, inputTokens?: number, outputTokens?: number
+ *     }
+ *   ]
+ * }
+ *
+ * The file may contain either:
+ *   (a) a single JSON object (one model run), or
+ *   (b) a JSON array of objects (multi-model run from --models a,b,c), or
+ *   (c) a text preamble followed by JSON (raw output from gaia-bench text mode
+ *       — we scan for the first '[' or '{' and parse from there).
+ *
+ * Missing files are silently skipped (returns empty array).
+ * Malformed files emit a warning to stderr and are skipped.
+ *
+ * Default search paths (tried in order, first found wins per iter):
+ *   /tmp/gaia-l1-full.json
+ *   /tmp/gaia-l1-haiku.json
+ *   /tmp/gaia-all-p1b.json
+ *   /tmp/gaia-all-p2.json
+ *   <custom paths passed by caller>
+ *
+ * Refs: ADR-136, #2156
+ */
+import * as fs from 'node:fs';
+// ---------------------------------------------------------------------------
+// Default result-file search paths
+// ---------------------------------------------------------------------------
+/** Default candidate paths for historical bench-run result JSONs. */
+export const DEFAULT_RESULT_PATHS = [
+    '/tmp/gaia-l1-full.json',
+    '/tmp/gaia-l1-haiku.json',
+    '/tmp/gaia-all-p1b.json',
+    '/tmp/gaia-all-p2.json',
+    '/tmp/gaia-all-probe.json',
+];
+// ---------------------------------------------------------------------------
+// JSON extraction helper
+// ---------------------------------------------------------------------------
+/**
+ * Attempt to extract and parse a JSON value (object or array) from a string
+ * that may have a text preamble before the JSON.
+ *
+ * Strategy:
+ *   1. Find the first '[' — parse as array.
+ *   2. Find the first '{' — parse as object.
+ *   3. Return null on failure.
+ */
+function extractJson(content) {
+    const bracketIdx = content.indexOf('[');
+    const braceIdx = content.indexOf('{');
+    // Prefer whichever appears first (handles both array and object formats).
+    const candidates = [];
+    if (bracketIdx >= 0)
+        candidates.push([bracketIdx, '[']);
+    if (braceIdx >= 0)
+        candidates.push([braceIdx, '{']);
+    candidates.sort((a, b) => a[0] - b[0]);
+    for (const [startIdx] of candidates) {
+        try {
+            return JSON.parse(content.slice(startIdx));
+        }
+        catch {
+            // Try next candidate.
+        }
+    }
+    return null;
+}
+// ---------------------------------------------------------------------------
+// Parse a single file into RawBenchOutput[]
+// ---------------------------------------------------------------------------
+function parseBenchFile(filePath) {
+    let content;
+    try {
+        content = fs.readFileSync(filePath, 'utf-8');
+    }
+    catch {
+        return [];
+    }
+    const parsed = extractJson(content);
+    if (parsed === null) {
+        process.stderr.write(`[gaia-hardness] Warning: could not extract JSON from ${filePath}\n`);
+        return [];
+    }
+    // Normalise to array of RawBenchOutput.
+    const outputs = [];
+    if (Array.isArray(parsed)) {
+        for (const item of parsed) {
+            if (item && typeof item === 'object' && Array.isArray(item.results)) {
+                outputs.push(item);
+            }
+        }
+    }
+    else if (parsed !== null &&
+        typeof parsed === 'object' &&
+        Array.isArray(parsed.results)) {
+        outputs.push(parsed);
+    }
+    if (outputs.length === 0) {
+        process.stderr.write(`[gaia-hardness] Warning: no valid bench outputs found in ${filePath}\n`);
+    }
+    return outputs;
+}
+// ---------------------------------------------------------------------------
+// Convert RawQuestionResult → LabeledExample
+// ---------------------------------------------------------------------------
+function toGaiaQuestion(r) {
+    return {
+        task_id: r.task_id,
+        level: 1, // level not stored in result JSON; default to 1
+        question: r.question,
+        final_answer: r.expected_output,
+        file_name: null,
+        file_path: null,
+    };
+}
+function toBenchLabeledExample(r) {
+    return {
+        question: toGaiaQuestion(r),
+        wasCorrect: Boolean(r.correct),
+        turns: r.turns,
+    };
+}
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Load labelled training examples from historical bench-run result JSONs.
+ *
+ * @param additionalPaths - Extra file paths to scan beyond the defaults.
+ * @param verbose - If true, log loaded example counts to stderr.
+ * @returns Deduplicated array of LabeledExample (dedup by task_id, last write wins).
+ */
+export function loadTrainingData(additionalPaths = [], verbose = false) {
+    const allPaths = [...DEFAULT_RESULT_PATHS, ...additionalPaths];
+    const seen = new Set();
+    const examples = [];
+    for (const filePath of allPaths) {
+        if (!fs.existsSync(filePath))
+            continue;
+        const outputs = parseBenchFile(filePath);
+        let fileCount = 0;
+        for (const output of outputs) {
+            for (const result of output.results) {
+                if (!result.task_id || typeof result.correct !== 'boolean')
+                    continue;
+                const key = result.task_id;
+                if (seen.has(key))
+                    continue; // first file wins (chronological order)
+                seen.add(key);
+                examples.push(toBenchLabeledExample(result));
+                fileCount++;
+            }
+        }
+        if (verbose && fileCount > 0) {
+            process.stderr.write(`[gaia-hardness] Loaded ${fileCount} examples from ${filePath}\n`);
+        }
+    }
+    if (verbose) {
+        process.stderr.write(`[gaia-hardness] Total training examples: ${examples.length}\n`);
+    }
+    return examples;
+}
+//# sourceMappingURL=train-data-loader.js.map

package/dist/src/benchmarks/gaia-hardness/train-data-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"train-data-loader.js","sourceRoot":"","sources":["../../../../src/benchmarks/gaia-hardness/train-data-loader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAI9B,8EAA8E;AAC9E,mCAAmC;AACnC,8EAA8E;AAE9E,qEAAqE;AACrE,MAAM,CAAC,MAAM,oBAAoB,GAAsB;IACrD,wBAAwB;IACxB,yBAAyB;IACzB,wBAAwB;IACxB,uBAAuB;IACvB,0BAA0B;CAClB,CAAC;AA2BX,8EAA8E;AAC9E,yBAAyB;AACzB,8EAA8E;AAE9E;;;;;;;;GAQG;AACH,SAAS,WAAW,CAAC,OAAe;IAClC,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAEtC,0EAA0E;IAC1E,MAAM,UAAU,GAA4B,EAAE,CAAC;IAC/C,IAAI,UAAU,IAAI,CAAC;QAAE,UAAU,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC;IACxD,IAAI,QAAQ,IAAI,CAAC;QAAE,UAAU,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC;IACpD,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEvC,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,UAAU,EAAE,CAAC;QACpC,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC7C,CAAC;QAAC,MAAM,CAAC;YACP,sBAAsB;QACxB,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,4CAA4C;AAC5C,8EAA8E;AAE9E,SAAS,cAAc,CAAC,QAAgB;IACtC,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IACpC,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QACpB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,wDAAwD,QAAQ,IAAI,CACrE,CAAC;QACF,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,wCAAwC;IACxC,MAAM,OAAO,GAAqB,EAAE,CAAC;IAErC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAE,IAAuB,CAAC,OAAO,CAAC,EAAE,CAAC;gBACxF,OAAO,CAAC,IAAI,CAAC,IAAsB,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;IACH,CAAC;SAAM,IACL,MAAM,KAAK,IAAI;QACf,OAAO,MAAM,KAAK,QAAQ;QAC1B,KAAK,CAAC,OAAO,CAAE,MAAyB,CAAC,OAAO,CAAC,EACjD,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,MAAwB,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,4DAA4D,QAAQ,IAAI,CACzE,CAAC;IACJ,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,8EAA8E;AAC9E,6CAA6C;AAC7C,8EAA8E;AAE9E,SAAS,cAAc,CAAC,CAAoB;IAC1C,OAAO;QACL,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,KAAK,EAAE,CAAC,EAAE,gDAAgD;QAC1D,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,YAAY,EAAE,CAAC,CAAC,eAAe;QAC/B,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,IAAI;KAChB,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,CAAoB;IACjD,OAAO;QACL,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAC;QAC3B,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;QAC9B,KAAK,EAAE,CAAC,CAAC,KAAK;KACf,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAC9B,kBAA4B,EAAE,EAC9B,OAAO,GAAG,KAAK;IAEf,MAAM,QAAQ,GAAG,CAAC,GAAG,oBAAoB,EAAE,GAAG,eAAe,CAAC,CAAC;IAC/D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,QAAQ,GAAqB,EAAE,CAAC;IAEtC,KAAK,MAAM,QAAQ,IAAI,QAAQ,EAAE,CAAC;QAChC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,SAAS;QAEvC,MAAM,OAAO,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACpC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,SAAS;oBAAE,SAAS;gBACrE,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC;gBAC3B,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS,CAAC,wCAAwC;gBACrE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACd,QAAQ,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC,CAAC;gBAC7C,SAAS,EAAE,CAAC;YACd,CAAC;QACH,CAAC;QAED,IAAI,OAAO,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAC7B,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,0BAA0B,SAAS,kBAAkB,QAAQ,IAAI,CAClE,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,4CAA4C,QAAQ,CAAC,MAAM,IAAI,CAChE,CAAC;IACJ,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/src/benchmarks/gaia-judge.d.ts

@@ -0,0 +1,88 @@
+/**
+ * GAIA Judge — ADR-133-PR6
+ *
+ * Two-stage answer scorer for the GAIA benchmark:
+ *
+ *   Stage 1 — Fast path: normalized exact-match.
+ *     Normalise = lowercase + strip surrounding whitespace + strip surrounding
+ *     single/double quotes + collapse internal whitespace runs to one space.
+ *     Roughly 30 % of GAIA Level-1 answers satisfy this; no API call required.
+ *
+ *   Stage 2 — LLM-as-judge: when exact-match fails, ask Claude Sonnet whether
+ *     the candidate answer is semantically equivalent to the ground truth.
+ *     The prompt embeds GAIA's official evaluation guideline (see
+ *     https://huggingface.co/datasets/gaia-benchmark/GAIA for full spec).
+ *
+ * Caching: judgment results are persisted under
+ *   ~/.cache/ruflo/gaia/judgments/<hash>.json
+ * keyed on (question_id, candidate_answer, model_id, JUDGE_PROMPT_VERSION).
+ * Re-running the same pair hits the cache and returns instantly.
+ *
+ * API pattern: raw fetch() against https://api.anthropic.com/v1/messages —
+ * mirrors the pattern established in gaia-agent.ts (ADR-133-PR3).
+ *
+ * Refs: ADR-133, #2156
+ */
+export interface JudgeResult {
+    questionId: string;
+    passed: boolean;
+    scoringPath: 'exact-match' | 'llm-judge' | 'cache';
+    candidateAnswer: string;
+    groundTruth: string;
+    judgeReason?: string;
+    judgeModel?: string;
+    judgeTokensIn?: number;
+    judgeTokensOut?: number;
+    judgeCostUsd?: number;
+}
+export interface JudgeOptions {
+    /** Default: 'claude-sonnet-4-6' */
+    judgeModel?: string;
+    /** Default: '~/.cache/ruflo/gaia/judgments/' */
+    cacheDir?: string;
+    skipCache?: boolean;
+    apiKey?: string;
+}
+/**
+ * GAIA normalisation as specified in the dataset paper:
+ *   - strip surrounding whitespace
+ *   - lowercase
+ *   - strip a single pair of surrounding quotes (single or double)
+ *   - collapse internal whitespace runs to one space
+ */
+export declare function normaliseAnswer(raw: string | null | undefined): string;
+/**
+ * Attempt to match a candidate numeric answer to an expected answer where the
+ * question implies a unit scale.
+ *
+ * Examples that this catches:
+ *   candidate="17000", expected="17", question contains "thousand"
+ *     → candidate / 1000 ≈ expected → MATCH
+ *   candidate="17", expected="17000", question contains "thousand"
+ *     → candidate × 1000 ≈ expected → MATCH (reverse direction)
+ *
+ * Returns true only when a numeric match is found under one of the scale
+ * multipliers mentioned in the question text.  Returns false for non-numeric
+ * inputs or when no multiplier matches.
+ *
+ * @param candidate    - The raw string from the model (may include commas/spaces).
+ * @param expected     - The raw ground-truth string.
+ * @param questionText - The original question (used to detect multiplier words).
+ */
+export declare function unitAwareNumberMatch(candidate: string, expected: string, questionText?: string): boolean;
+/**
+ * Judge a single GAIA answer.
+ *
+ * @param question   - Object with `id` (task_id), `expected` (ground truth),
+ *                     and optional `questionText` (the full question string,
+ *                     used for unit-aware numeric matching in Stage 1).
+ * @param candidateAnswer - The answer produced by the agent; `null` counts as a miss.
+ * @param options    - Optional overrides (model, cache dir, API key, etc.).
+ * @returns          - JudgeResult with pass/fail, scoring path, and cost metrics.
+ */
+export declare function judgeAnswer(question: {
+    id: string;
+    expected: string;
+    questionText?: string;
+}, candidateAnswer: string | null, options?: JudgeOptions): Promise<JudgeResult>;
+//# sourceMappingURL=gaia-judge.d.ts.map

package/dist/src/benchmarks/gaia-judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gaia-judge.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/gaia-judge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AA+BH,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,OAAO,CAAC;IAChB,WAAW,EAAE,aAAa,GAAG,WAAW,GAAG,OAAO,CAAC;IACnD,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,mCAAmC;IACnC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gDAAgD;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAMD;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,CAWtE;AAMD;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,oBAAoB,CAClC,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAoCT;AAwMD;;;;;;;;;GASG;AACH,wBAAsB,WAAW,CAC/B,QAAQ,EAAE;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,EACjE,eAAe,EAAE,MAAM,GAAG,IAAI,EAC9B,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC,WAAW,CAAC,CAyFtB"}