@claude-flow/cli 3.10.4 → 3.10.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/benchmarks/capability-tasks.d.ts +126 -0
- package/dist/src/benchmarks/capability-tasks.d.ts.map +1 -0
- package/dist/src/benchmarks/capability-tasks.js +147 -0
- package/dist/src/benchmarks/capability-tasks.js.map +1 -0
- package/dist/src/benchmarks/gaia-agent-contrastive.d.ts +53 -0
- package/dist/src/benchmarks/gaia-agent-contrastive.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-agent-contrastive.js +177 -0
- package/dist/src/benchmarks/gaia-agent-contrastive.js.map +1 -0
- package/dist/src/benchmarks/gaia-agent-gemini.d.ts +51 -0
- package/dist/src/benchmarks/gaia-agent-gemini.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-agent-gemini.js +352 -0
- package/dist/src/benchmarks/gaia-agent-gemini.js.map +1 -0
- package/dist/src/benchmarks/gaia-agent-planning.smoke.d.ts +18 -0
- package/dist/src/benchmarks/gaia-agent-planning.smoke.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-agent-planning.smoke.js +253 -0
- package/dist/src/benchmarks/gaia-agent-planning.smoke.js.map +1 -0
- package/dist/src/benchmarks/gaia-agent.d.ts +198 -0
- package/dist/src/benchmarks/gaia-agent.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-agent.js +651 -0
- package/dist/src/benchmarks/gaia-agent.js.map +1 -0
- package/dist/src/benchmarks/gaia-causal-memory.d.ts +133 -0
- package/dist/src/benchmarks/gaia-causal-memory.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-causal-memory.js +281 -0
- package/dist/src/benchmarks/gaia-causal-memory.js.map +1 -0
- package/dist/src/benchmarks/gaia-causal-memory.smoke.d.ts +22 -0
- package/dist/src/benchmarks/gaia-causal-memory.smoke.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-causal-memory.smoke.js +300 -0
- package/dist/src/benchmarks/gaia-causal-memory.smoke.js.map +1 -0
- package/dist/src/benchmarks/gaia-claude-p.d.ts +105 -0
- package/dist/src/benchmarks/gaia-claude-p.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-claude-p.js +269 -0
- package/dist/src/benchmarks/gaia-claude-p.js.map +1 -0
- package/dist/src/benchmarks/gaia-claude-p.smoke.d.ts +20 -0
- package/dist/src/benchmarks/gaia-claude-p.smoke.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-claude-p.smoke.js +168 -0
- package/dist/src/benchmarks/gaia-claude-p.smoke.js.map +1 -0
- package/dist/src/benchmarks/gaia-codeagent-runner.py +556 -0
- package/dist/src/benchmarks/gaia-codeagent.d.ts +129 -0
- package/dist/src/benchmarks/gaia-codeagent.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-codeagent.js +590 -0
- package/dist/src/benchmarks/gaia-codeagent.js.map +1 -0
- package/dist/src/benchmarks/gaia-codeagent.smoke.d.ts +33 -0
- package/dist/src/benchmarks/gaia-codeagent.smoke.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-codeagent.smoke.js +241 -0
- package/dist/src/benchmarks/gaia-codeagent.smoke.js.map +1 -0
- package/dist/src/benchmarks/gaia-convergence.d.ts +138 -0
- package/dist/src/benchmarks/gaia-convergence.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-convergence.js +260 -0
- package/dist/src/benchmarks/gaia-convergence.js.map +1 -0
- package/dist/src/benchmarks/gaia-convergence.smoke.d.ts +19 -0
- package/dist/src/benchmarks/gaia-convergence.smoke.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-convergence.smoke.js +246 -0
- package/dist/src/benchmarks/gaia-convergence.smoke.js.map +1 -0
- package/dist/src/benchmarks/gaia-critic.d.ts +123 -0
- package/dist/src/benchmarks/gaia-critic.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-critic.js +312 -0
- package/dist/src/benchmarks/gaia-critic.js.map +1 -0
- package/dist/src/benchmarks/gaia-critic.smoke.d.ts +21 -0
- package/dist/src/benchmarks/gaia-critic.smoke.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-critic.smoke.js +327 -0
- package/dist/src/benchmarks/gaia-critic.smoke.js.map +1 -0
- package/dist/src/benchmarks/gaia-dag-pilot.d.ts +22 -0
- package/dist/src/benchmarks/gaia-dag-pilot.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-dag-pilot.js +158 -0
- package/dist/src/benchmarks/gaia-dag-pilot.js.map +1 -0
- package/dist/src/benchmarks/gaia-dag.d.ts +110 -0
- package/dist/src/benchmarks/gaia-dag.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-dag.js +658 -0
- package/dist/src/benchmarks/gaia-dag.js.map +1 -0
- package/dist/src/benchmarks/gaia-dag.smoke.d.ts +21 -0
- package/dist/src/benchmarks/gaia-dag.smoke.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-dag.smoke.js +290 -0
- package/dist/src/benchmarks/gaia-dag.smoke.js.map +1 -0
- package/dist/src/benchmarks/gaia-decomposer.d.ts +125 -0
- package/dist/src/benchmarks/gaia-decomposer.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-decomposer.js +350 -0
- package/dist/src/benchmarks/gaia-decomposer.js.map +1 -0
- package/dist/src/benchmarks/gaia-decomposer.smoke.d.ts +21 -0
- package/dist/src/benchmarks/gaia-decomposer.smoke.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-decomposer.smoke.js +228 -0
- package/dist/src/benchmarks/gaia-decomposer.smoke.js.map +1 -0
- package/dist/src/benchmarks/gaia-e2e-smoke.d.ts +27 -0
- package/dist/src/benchmarks/gaia-e2e-smoke.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-e2e-smoke.js +136 -0
- package/dist/src/benchmarks/gaia-e2e-smoke.js.map +1 -0
- package/dist/src/benchmarks/gaia-ensemble-pilot.d.ts +17 -0
- package/dist/src/benchmarks/gaia-ensemble-pilot.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-ensemble-pilot.js +106 -0
- package/dist/src/benchmarks/gaia-ensemble-pilot.js.map +1 -0
- package/dist/src/benchmarks/gaia-ensemble.d.ts +103 -0
- package/dist/src/benchmarks/gaia-ensemble.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-ensemble.js +462 -0
- package/dist/src/benchmarks/gaia-ensemble.js.map +1 -0
- package/dist/src/benchmarks/gaia-extract.smoke.d.ts +45 -0
- package/dist/src/benchmarks/gaia-extract.smoke.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-extract.smoke.js +242 -0
- package/dist/src/benchmarks/gaia-extract.smoke.js.map +1 -0
- package/dist/src/benchmarks/gaia-gemini-pilot.d.ts +17 -0
- package/dist/src/benchmarks/gaia-gemini-pilot.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-gemini-pilot.js +168 -0
- package/dist/src/benchmarks/gaia-gemini-pilot.js.map +1 -0
- package/dist/src/benchmarks/gaia-hardness/features.d.ts +46 -0
- package/dist/src/benchmarks/gaia-hardness/features.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-hardness/features.js +170 -0
- package/dist/src/benchmarks/gaia-hardness/features.js.map +1 -0
- package/dist/src/benchmarks/gaia-hardness/predictor.d.ts +105 -0
- package/dist/src/benchmarks/gaia-hardness/predictor.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-hardness/predictor.js +260 -0
- package/dist/src/benchmarks/gaia-hardness/predictor.js.map +1 -0
- package/dist/src/benchmarks/gaia-hardness/predictor.smoke.d.ts +20 -0
- package/dist/src/benchmarks/gaia-hardness/predictor.smoke.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-hardness/predictor.smoke.js +235 -0
- package/dist/src/benchmarks/gaia-hardness/predictor.smoke.js.map +1 -0
- package/dist/src/benchmarks/gaia-hardness/train-data-loader.d.ts +51 -0
- package/dist/src/benchmarks/gaia-hardness/train-data-loader.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-hardness/train-data-loader.js +179 -0
- package/dist/src/benchmarks/gaia-hardness/train-data-loader.js.map +1 -0
- package/dist/src/benchmarks/gaia-judge.d.ts +88 -0
- package/dist/src/benchmarks/gaia-judge.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-judge.js +437 -0
- package/dist/src/benchmarks/gaia-judge.js.map +1 -0
- package/dist/src/benchmarks/gaia-loader.d.ts +87 -0
- package/dist/src/benchmarks/gaia-loader.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-loader.js +326 -0
- package/dist/src/benchmarks/gaia-loader.js.map +1 -0
- package/dist/src/benchmarks/gaia-mode-router.d.ts +67 -0
- package/dist/src/benchmarks/gaia-mode-router.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-mode-router.js +126 -0
- package/dist/src/benchmarks/gaia-mode-router.js.map +1 -0
- package/dist/src/benchmarks/gaia-mode-router.smoke.d.ts +15 -0
- package/dist/src/benchmarks/gaia-mode-router.smoke.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-mode-router.smoke.js +131 -0
- package/dist/src/benchmarks/gaia-mode-router.smoke.js.map +1 -0
- package/dist/src/benchmarks/gaia-tools/file_read.d.ts +35 -0
- package/dist/src/benchmarks/gaia-tools/file_read.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-tools/file_read.js +403 -0
- package/dist/src/benchmarks/gaia-tools/file_read.js.map +1 -0
- package/dist/src/benchmarks/gaia-tools/grounded_query.d.ts +126 -0
- package/dist/src/benchmarks/gaia-tools/grounded_query.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-tools/grounded_query.js +225 -0
- package/dist/src/benchmarks/gaia-tools/grounded_query.js.map +1 -0
- package/dist/src/benchmarks/gaia-tools/grounded_query.smoke.d.ts +24 -0
- package/dist/src/benchmarks/gaia-tools/grounded_query.smoke.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-tools/grounded_query.smoke.js +276 -0
- package/dist/src/benchmarks/gaia-tools/grounded_query.smoke.js.map +1 -0
- package/dist/src/benchmarks/gaia-tools/image_describe.d.ts +45 -0
- package/dist/src/benchmarks/gaia-tools/image_describe.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-tools/image_describe.js +222 -0
- package/dist/src/benchmarks/gaia-tools/image_describe.js.map +1 -0
- package/dist/src/benchmarks/gaia-tools/image_describe.smoke.d.ts +24 -0
- package/dist/src/benchmarks/gaia-tools/image_describe.smoke.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-tools/image_describe.smoke.js +169 -0
- package/dist/src/benchmarks/gaia-tools/image_describe.smoke.js.map +1 -0
- package/dist/src/benchmarks/gaia-tools/index.d.ts +32 -0
- package/dist/src/benchmarks/gaia-tools/index.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-tools/index.js +36 -0
- package/dist/src/benchmarks/gaia-tools/index.js.map +1 -0
- package/dist/src/benchmarks/gaia-tools/pdf_read.d.ts +27 -0
- package/dist/src/benchmarks/gaia-tools/pdf_read.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-tools/pdf_read.js +226 -0
- package/dist/src/benchmarks/gaia-tools/pdf_read.js.map +1 -0
- package/dist/src/benchmarks/gaia-tools/python_exec.d.ts +31 -0
- package/dist/src/benchmarks/gaia-tools/python_exec.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-tools/python_exec.js +94 -0
- package/dist/src/benchmarks/gaia-tools/python_exec.js.map +1 -0
- package/dist/src/benchmarks/gaia-tools/python_exec.smoke.d.ts +13 -0
- package/dist/src/benchmarks/gaia-tools/python_exec.smoke.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-tools/python_exec.smoke.js +98 -0
- package/dist/src/benchmarks/gaia-tools/python_exec.smoke.js.map +1 -0
- package/dist/src/benchmarks/gaia-tools/types.d.ts +62 -0
- package/dist/src/benchmarks/gaia-tools/types.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-tools/types.js +12 -0
- package/dist/src/benchmarks/gaia-tools/types.js.map +1 -0
- package/dist/src/benchmarks/gaia-tools/visit_webpage.d.ts +52 -0
- package/dist/src/benchmarks/gaia-tools/visit_webpage.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-tools/visit_webpage.js +304 -0
- package/dist/src/benchmarks/gaia-tools/visit_webpage.js.map +1 -0
- package/dist/src/benchmarks/gaia-tools/web_browse.d.ts +61 -0
- package/dist/src/benchmarks/gaia-tools/web_browse.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-tools/web_browse.js +217 -0
- package/dist/src/benchmarks/gaia-tools/web_browse.js.map +1 -0
- package/dist/src/benchmarks/gaia-tools/web_browse.smoke.d.ts +26 -0
- package/dist/src/benchmarks/gaia-tools/web_browse.smoke.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-tools/web_browse.smoke.js +152 -0
- package/dist/src/benchmarks/gaia-tools/web_browse.smoke.js.map +1 -0
- package/dist/src/benchmarks/gaia-tools/web_search.d.ts +30 -0
- package/dist/src/benchmarks/gaia-tools/web_search.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-tools/web_search.js +210 -0
- package/dist/src/benchmarks/gaia-tools/web_search.js.map +1 -0
- package/dist/src/benchmarks/gaia-tools/web_search.smoke.d.ts +24 -0
- package/dist/src/benchmarks/gaia-tools/web_search.smoke.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-tools/web_search.smoke.js +311 -0
- package/dist/src/benchmarks/gaia-tools/web_search.smoke.js.map +1 -0
- package/dist/src/benchmarks/gaia-visit-webpage.smoke.d.ts +25 -0
- package/dist/src/benchmarks/gaia-visit-webpage.smoke.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-visit-webpage.smoke.js +304 -0
- package/dist/src/benchmarks/gaia-visit-webpage.smoke.js.map +1 -0
- package/dist/src/benchmarks/gaia-voting.d.ts +88 -0
- package/dist/src/benchmarks/gaia-voting.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-voting.js +297 -0
- package/dist/src/benchmarks/gaia-voting.js.map +1 -0
- package/dist/src/benchmarks/gaia-voting.smoke.d.ts +20 -0
- package/dist/src/benchmarks/gaia-voting.smoke.d.ts.map +1 -0
- package/dist/src/benchmarks/gaia-voting.smoke.js +332 -0
- package/dist/src/benchmarks/gaia-voting.smoke.js.map +1 -0
- package/dist/src/commands/gaia-bench.d.ts +40 -0
- package/dist/src/commands/gaia-bench.d.ts.map +1 -0
- package/dist/src/commands/gaia-bench.js +597 -0
- package/dist/src/commands/gaia-bench.js.map +1 -0
- package/dist/src/commands/index.d.ts.map +1 -1
- package/dist/src/commands/index.js +2 -0
- package/dist/src/commands/index.js.map +1 -1
- package/dist/src/commands/init.d.ts.map +1 -1
- package/dist/src/commands/init.js +41 -4
- package/dist/src/commands/init.js.map +1 -1
- package/dist/src/commands/performance-capability.d.ts +30 -0
- package/dist/src/commands/performance-capability.d.ts.map +1 -0
- package/dist/src/commands/performance-capability.js +349 -0
- package/dist/src/commands/performance-capability.js.map +1 -0
- package/dist/src/commands/route.d.ts.map +1 -1
- package/dist/src/commands/route.js +9 -0
- package/dist/src/commands/route.js.map +1 -1
- package/dist/src/init/executor.d.ts.map +1 -1
- package/dist/src/init/executor.js +25 -6
- package/dist/src/init/executor.js.map +1 -1
- package/dist/src/init/mcp-generator.d.ts.map +1 -1
- package/dist/src/init/mcp-generator.js +9 -4
- package/dist/src/init/mcp-generator.js.map +1 -1
- package/dist/src/init/statusline-generator.d.ts.map +1 -1
- package/dist/src/init/statusline-generator.js +25 -2
- package/dist/src/init/statusline-generator.js.map +1 -1
- package/dist/src/mcp-tools/agentdb-tools.d.ts.map +1 -1
- package/dist/src/mcp-tools/agentdb-tools.js +20 -3
- package/dist/src/mcp-tools/agentdb-tools.js.map +1 -1
- package/dist/src/mcp-tools/system-tools.d.ts.map +1 -1
- package/dist/src/mcp-tools/system-tools.js +12 -1
- package/dist/src/mcp-tools/system-tools.js.map +1 -1
- package/dist/src/memory/memory-bridge.d.ts.map +1 -1
- package/dist/src/memory/memory-bridge.js +34 -0
- package/dist/src/memory/memory-bridge.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Smoke tests — ADR-135 Track I: Causal failure-avoidance edges
|
|
3
|
+
*
|
|
4
|
+
* All I/O is mocked (no real filesystem or network calls).
|
|
5
|
+
*
|
|
6
|
+
* Tests:
|
|
7
|
+
* 1. Record one failure → retrieve same-signature question → returns hint
|
|
8
|
+
* 2. Record 3 failures → retrieve unrelated question → returns empty hint
|
|
9
|
+
* 3. Same edge recorded twice → occurrenceCount=2, not duplicated
|
|
10
|
+
* 4. File doesn't exist → graceful empty result on retrieve
|
|
11
|
+
* 5. Corrupted JSONL line → skipped, doesn't crash
|
|
12
|
+
* 6. maxEdgesPerSignature cap respected (no new edge after cap)
|
|
13
|
+
* 7. Signature computation is deterministic (same input → same output)
|
|
14
|
+
* 8. Correct answer trajectory → no edges recorded
|
|
15
|
+
*
|
|
16
|
+
* Usage:
|
|
17
|
+
* npx tsx src/benchmarks/gaia-causal-memory.smoke.ts
|
|
18
|
+
*
|
|
19
|
+
* Refs: ADR-135, #2156
|
|
20
|
+
*/
|
|
21
|
+
import * as fs from 'node:fs';
|
|
22
|
+
import * as os from 'node:os';
|
|
23
|
+
import * as path from 'node:path';
|
|
24
|
+
import { computeQuestionSignature, inferFailureType, recordCausalFailures, retrieveCausalHints, } from './gaia-causal-memory.js';
|
|
25
|
+
// ---------------------------------------------------------------------------
|
|
26
|
+
// Mini test runner (zero external deps)
|
|
27
|
+
// ---------------------------------------------------------------------------
|
|
28
|
+
let passed = 0;
|
|
29
|
+
let failed = 0;
|
|
30
|
+
const results = [];
|
|
31
|
+
async function test(name, fn) {
|
|
32
|
+
try {
|
|
33
|
+
await fn();
|
|
34
|
+
passed++;
|
|
35
|
+
results.push({ name, ok: true });
|
|
36
|
+
console.log(` PASS ${name}`);
|
|
37
|
+
}
|
|
38
|
+
catch (err) {
|
|
39
|
+
failed++;
|
|
40
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
41
|
+
results.push({ name, ok: false, error: msg });
|
|
42
|
+
console.error(` FAIL ${name}`);
|
|
43
|
+
console.error(` ${msg}`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
function assert(condition, message) {
|
|
47
|
+
if (!condition) {
|
|
48
|
+
throw new Error(`Assertion failed: ${message}`);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
function assertEqual(actual, expected, label) {
|
|
52
|
+
if (actual !== expected) {
|
|
53
|
+
throw new Error(`${label}: expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
// ---------------------------------------------------------------------------
|
|
57
|
+
// Fixtures
|
|
58
|
+
// ---------------------------------------------------------------------------
|
|
59
|
+
function makeQuestion(text) {
|
|
60
|
+
return {
|
|
61
|
+
task_id: 'smoke-' + text.slice(0, 8),
|
|
62
|
+
level: 1,
|
|
63
|
+
question: text,
|
|
64
|
+
final_answer: 'ignored-in-smoke',
|
|
65
|
+
file_name: null,
|
|
66
|
+
file_path: null,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
function makeResult(overrides = {}) {
|
|
70
|
+
return {
|
|
71
|
+
questionId: 'smoke-q1',
|
|
72
|
+
finalAnswer: 'wrong answer',
|
|
73
|
+
turns: 3,
|
|
74
|
+
toolCallsByName: { web_search: 2 },
|
|
75
|
+
totalInputTokens: 100,
|
|
76
|
+
totalOutputTokens: 50,
|
|
77
|
+
wallMs: 1500,
|
|
78
|
+
...overrides,
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
// ---------------------------------------------------------------------------
|
|
82
|
+
// Test suite
|
|
83
|
+
// ---------------------------------------------------------------------------
|
|
84
|
+
async function runAll() {
|
|
85
|
+
console.log('\nADR-135 Track I — causal failure-avoidance edges (smoke)\n');
|
|
86
|
+
// We use a fresh temp dir per test group to isolate state.
|
|
87
|
+
const tmpBase = path.join(os.tmpdir(), 'ruflo-causal-smoke-' + process.pid);
|
|
88
|
+
// -------------------------------------------------------------------------
|
|
89
|
+
// Test 1: Record one failure → retrieve same-signature question → hint returned
|
|
90
|
+
// -------------------------------------------------------------------------
|
|
91
|
+
await test('1. record one failure → retrieve same question → hint returned', async () => {
|
|
92
|
+
const dir = path.join(tmpBase, 't1');
|
|
93
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
94
|
+
const storePath = path.join(dir, 'edges.jsonl');
|
|
95
|
+
const question = makeQuestion('What is the capital of France?');
|
|
96
|
+
const result = makeResult({ finalAnswer: 'London' });
|
|
97
|
+
const recRes = await recordCausalFailures(question, result, false, { storePath });
|
|
98
|
+
assert(recRes.edgesRecorded > 0, 'should have recorded at least one edge');
|
|
99
|
+
const retRes = await retrieveCausalHints(question, { storePath });
|
|
100
|
+
assert(retRes.edgesMatched > 0, 'should match the stored edge');
|
|
101
|
+
assert(retRes.hint.length > 0, 'hint should be non-empty');
|
|
102
|
+
assert(retRes.hint.includes('[PRIOR FAILURES]'), 'hint should contain [PRIOR FAILURES] header');
|
|
103
|
+
assert(retRes.hint.includes('web_search'), 'hint should mention the failed tool');
|
|
104
|
+
});
|
|
105
|
+
// -------------------------------------------------------------------------
|
|
106
|
+
// Test 2: Record 3 failures → retrieve unrelated question → empty hint
|
|
107
|
+
// -------------------------------------------------------------------------
|
|
108
|
+
await test('2. record 3 failures → retrieve unrelated question → empty hint', async () => {
|
|
109
|
+
const dir = path.join(tmpBase, 't2');
|
|
110
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
111
|
+
const storePath = path.join(dir, 'edges.jsonl');
|
|
112
|
+
const questions = [
|
|
113
|
+
makeQuestion('What is the capital of France?'),
|
|
114
|
+
makeQuestion('Who wrote Hamlet?'),
|
|
115
|
+
makeQuestion('What is 2 + 2?'),
|
|
116
|
+
];
|
|
117
|
+
for (const q of questions) {
|
|
118
|
+
await recordCausalFailures(q, makeResult(), false, { storePath });
|
|
119
|
+
}
|
|
120
|
+
const unrelated = makeQuestion('Completely different and unrelated question about astrophysics and quantum phenomena XYZ123');
|
|
121
|
+
const retRes = await retrieveCausalHints(unrelated, { storePath });
|
|
122
|
+
assertEqual(retRes.edgesMatched, 0, 'edgesMatched for unrelated question');
|
|
123
|
+
assertEqual(retRes.hint, '', 'hint for unrelated question');
|
|
124
|
+
});
|
|
125
|
+
// -------------------------------------------------------------------------
|
|
126
|
+
// Test 3: Same edge recorded twice → occurrenceCount=2, not duplicated
|
|
127
|
+
// -------------------------------------------------------------------------
|
|
128
|
+
await test('3. same edge recorded twice → occurrenceCount=2, not duplicated', async () => {
|
|
129
|
+
const dir = path.join(tmpBase, 't3');
|
|
130
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
131
|
+
const storePath = path.join(dir, 'edges.jsonl');
|
|
132
|
+
const question = makeQuestion('What year did World War II end?');
|
|
133
|
+
const result = makeResult({ toolCallsByName: { web_search: 1 } });
|
|
134
|
+
// Record same failure twice
|
|
135
|
+
await recordCausalFailures(question, result, false, { storePath });
|
|
136
|
+
await recordCausalFailures(question, result, false, { storePath });
|
|
137
|
+
// Read the JSONL directly and count lines / occurrence counts
|
|
138
|
+
const raw = fs.readFileSync(storePath, 'utf8');
|
|
139
|
+
const lines = raw.trim().split('\n').filter((l) => l.trim() !== '');
|
|
140
|
+
const edges = lines.map((l) => JSON.parse(l));
|
|
141
|
+
// Should have exactly one edge for this signature+tool+step
|
|
142
|
+
const sig = computeQuestionSignature(question.question);
|
|
143
|
+
const sigEdges = edges.filter((e) => e.questionSignature === sig && e.failedTool === 'web_search');
|
|
144
|
+
assertEqual(sigEdges.length, 1, 'should have exactly one edge per signature+tool+step');
|
|
145
|
+
assertEqual(sigEdges[0].occurrenceCount, 2, 'occurrenceCount after two recordings');
|
|
146
|
+
});
|
|
147
|
+
// -------------------------------------------------------------------------
|
|
148
|
+
// Test 4: File doesn't exist → graceful empty result on retrieve
|
|
149
|
+
// -------------------------------------------------------------------------
|
|
150
|
+
await test('4. file does not exist → graceful empty result on retrieve', async () => {
|
|
151
|
+
const storePath = path.join(tmpBase, 'nonexistent', 'edges.jsonl');
|
|
152
|
+
const question = makeQuestion('How many legs does a spider have?');
|
|
153
|
+
const retRes = await retrieveCausalHints(question, { storePath });
|
|
154
|
+
assertEqual(retRes.edgesMatched, 0, 'edgesMatched when file absent');
|
|
155
|
+
assertEqual(retRes.hint, '', 'hint when file absent');
|
|
156
|
+
});
|
|
157
|
+
// -------------------------------------------------------------------------
|
|
158
|
+
// Test 5: Corrupted JSONL line → skipped, doesn't crash
|
|
159
|
+
// -------------------------------------------------------------------------
|
|
160
|
+
await test('5. corrupted JSONL line → skipped, does not crash', async () => {
|
|
161
|
+
const dir = path.join(tmpBase, 't5');
|
|
162
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
163
|
+
const storePath = path.join(dir, 'edges.jsonl');
|
|
164
|
+
const goodQuestion = makeQuestion('Name the largest planet in the solar system.');
|
|
165
|
+
const goodEdge = {
|
|
166
|
+
questionSignature: computeQuestionSignature(goodQuestion.question),
|
|
167
|
+
failedTool: 'python_exec',
|
|
168
|
+
failedTrajectoryStep: 'python_exec called once; failure type=wrong_answer',
|
|
169
|
+
observedFailureType: 'wrong_answer',
|
|
170
|
+
createdAt: new Date().toISOString(),
|
|
171
|
+
occurrenceCount: 1,
|
|
172
|
+
};
|
|
173
|
+
// Write one valid edge + one corrupted line
|
|
174
|
+
fs.mkdirSync(path.dirname(storePath), { recursive: true });
|
|
175
|
+
fs.writeFileSync(storePath, JSON.stringify(goodEdge) + '\n' +
|
|
176
|
+
'{"this": is not valid JSON!!!\n' +
|
|
177
|
+
'{"partial":true\n', 'utf8');
|
|
178
|
+
// retrieve should work and not throw
|
|
179
|
+
const retRes = await retrieveCausalHints(goodQuestion, { storePath });
|
|
180
|
+
assertEqual(retRes.edgesMatched, 1, 'should still find the one valid edge');
|
|
181
|
+
assert(retRes.hint.includes('[PRIOR FAILURES]'), 'hint should be returned for valid edge');
|
|
182
|
+
assert(retRes.hint.includes('python_exec'), 'hint should reference python_exec');
|
|
183
|
+
});
|
|
184
|
+
// -------------------------------------------------------------------------
|
|
185
|
+
// Test 6: maxEdgesPerSignature cap respected
|
|
186
|
+
// -------------------------------------------------------------------------
|
|
187
|
+
await test('6. maxEdgesPerSignature cap respected', async () => {
|
|
188
|
+
const dir = path.join(tmpBase, 't6');
|
|
189
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
190
|
+
const storePath = path.join(dir, 'edges.jsonl');
|
|
191
|
+
const maxEdgesPerSignature = 2;
|
|
192
|
+
const question = makeQuestion('Explain the theory of relativity in simple terms.');
|
|
193
|
+
// Record 3 failures with different tools — only first 2 should be stored
|
|
194
|
+
for (const tool of ['web_search', 'file_read', 'python_exec']) {
|
|
195
|
+
const result = makeResult({ toolCallsByName: { [tool]: 1 } });
|
|
196
|
+
await recordCausalFailures(question, result, false, {
|
|
197
|
+
storePath,
|
|
198
|
+
maxEdgesPerSignature,
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
const raw = fs.readFileSync(storePath, 'utf8');
|
|
202
|
+
const lines = raw.trim().split('\n').filter((l) => l.trim() !== '');
|
|
203
|
+
const sig = computeQuestionSignature(question.question);
|
|
204
|
+
const sigEdges = lines
|
|
205
|
+
.map((l) => JSON.parse(l))
|
|
206
|
+
.filter((e) => e.questionSignature === sig);
|
|
207
|
+
assert(sigEdges.length <= maxEdgesPerSignature, `should have at most ${maxEdgesPerSignature} edges, got ${sigEdges.length}`);
|
|
208
|
+
});
|
|
209
|
+
// -------------------------------------------------------------------------
|
|
210
|
+
// Test 7: Signature computation is deterministic
|
|
211
|
+
// -------------------------------------------------------------------------
|
|
212
|
+
await test('7. signature computation is deterministic', async () => {
|
|
213
|
+
const text = 'What is the boiling point of water at sea level?';
|
|
214
|
+
const sig1 = computeQuestionSignature(text);
|
|
215
|
+
const sig2 = computeQuestionSignature(text);
|
|
216
|
+
const sig3 = computeQuestionSignature(text.trim()); // trim should not change result
|
|
217
|
+
const sig4 = computeQuestionSignature(text.toUpperCase().toLowerCase()); // normalisation
|
|
218
|
+
assertEqual(sig1, sig2, 'same input produces same signature (call 1 vs 2)');
|
|
219
|
+
assertEqual(sig1, sig3, 'trimmed input produces same signature');
|
|
220
|
+
assertEqual(sig1, sig4, 'case-normalised input produces same signature');
|
|
221
|
+
assertEqual(sig1.length, 16, 'signature is 16 hex characters');
|
|
222
|
+
// Different text should produce different signature (with overwhelming probability)
|
|
223
|
+
const sigOther = computeQuestionSignature('Completely different question about astronomy.');
|
|
224
|
+
assert(sig1 !== sigOther, 'different inputs produce different signatures');
|
|
225
|
+
});
|
|
226
|
+
// -------------------------------------------------------------------------
|
|
227
|
+
// Test 8: Correct answer trajectory → no edges recorded
|
|
228
|
+
// -------------------------------------------------------------------------
|
|
229
|
+
await test('8. correct answer trajectory → no edges recorded', async () => {
|
|
230
|
+
const dir = path.join(tmpBase, 't8');
|
|
231
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
232
|
+
const storePath = path.join(dir, 'edges.jsonl');
|
|
233
|
+
const question = makeQuestion('What color is the sky on a clear day?');
|
|
234
|
+
const result = makeResult({ finalAnswer: 'blue' });
|
|
235
|
+
// wasCorrect = true
|
|
236
|
+
const recRes = await recordCausalFailures(question, result, true, { storePath });
|
|
237
|
+
assertEqual(recRes.edgesRecorded, 0, 'no edges for correct trajectory');
|
|
238
|
+
assert(!fs.existsSync(storePath), 'store file should not be created for correct answer');
|
|
239
|
+
});
|
|
240
|
+
// -------------------------------------------------------------------------
|
|
241
|
+
// Bonus: inferFailureType unit checks
|
|
242
|
+
// -------------------------------------------------------------------------
|
|
243
|
+
await test('inferFailureType: null for correct answers', async () => {
|
|
244
|
+
const result = makeResult({ finalAnswer: 'blue' });
|
|
245
|
+
const ft = inferFailureType(result, true);
|
|
246
|
+
assertEqual(ft, null, 'inferFailureType returns null for wasCorrect=true');
|
|
247
|
+
});
|
|
248
|
+
await test('inferFailureType: timeout when timedOut=true', async () => {
|
|
249
|
+
const result = makeResult({ timedOut: true });
|
|
250
|
+
const ft = inferFailureType(result, false);
|
|
251
|
+
assertEqual(ft, 'timeout', 'inferFailureType=timeout when timedOut flag set');
|
|
252
|
+
});
|
|
253
|
+
await test('inferFailureType: tool_error when error present', async () => {
|
|
254
|
+
const result = makeResult({ error: 'network timeout' });
|
|
255
|
+
const ft = inferFailureType(result, false);
|
|
256
|
+
assertEqual(ft, 'tool_error', 'inferFailureType=tool_error when error field set');
|
|
257
|
+
});
|
|
258
|
+
await test('inferFailureType: empty_result when finalAnswer is null', async () => {
|
|
259
|
+
const result = makeResult({ finalAnswer: null });
|
|
260
|
+
const ft = inferFailureType(result, false);
|
|
261
|
+
assertEqual(ft, 'empty_result', 'inferFailureType=empty_result when finalAnswer=null');
|
|
262
|
+
});
|
|
263
|
+
await test('inferFailureType: wrong_answer for normal failure', async () => {
|
|
264
|
+
const result = makeResult({ finalAnswer: 'incorrect value' });
|
|
265
|
+
const ft = inferFailureType(result, false);
|
|
266
|
+
assertEqual(ft, 'wrong_answer', 'inferFailureType=wrong_answer for normal wrong answer');
|
|
267
|
+
});
|
|
268
|
+
// -------------------------------------------------------------------------
|
|
269
|
+
// Cleanup
|
|
270
|
+
// -------------------------------------------------------------------------
|
|
271
|
+
try {
|
|
272
|
+
fs.rmSync(tmpBase, { recursive: true, force: true });
|
|
273
|
+
}
|
|
274
|
+
catch {
|
|
275
|
+
// Non-fatal cleanup failure; temp files will be cleared by OS.
|
|
276
|
+
}
|
|
277
|
+
// -------------------------------------------------------------------------
|
|
278
|
+
// Summary
|
|
279
|
+
// -------------------------------------------------------------------------
|
|
280
|
+
console.log(`\n${'─'.repeat(60)}`);
|
|
281
|
+
console.log(`Results: ${passed} passed, ${failed} failed (${passed + failed} total)`);
|
|
282
|
+
if (failed > 0) {
|
|
283
|
+
console.error('\nFailed tests:');
|
|
284
|
+
for (const r of results.filter((r) => !r.ok)) {
|
|
285
|
+
console.error(` ${r.name}: ${r.error}`);
|
|
286
|
+
}
|
|
287
|
+
process.exit(1);
|
|
288
|
+
}
|
|
289
|
+
else {
|
|
290
|
+
console.log('\nAll smoke tests passed.');
|
|
291
|
+
console.log('\nNote: Track I lift is compound (+0pp first run, +2-5pp after 5+ runs).');
|
|
292
|
+
console.log('Wiring into gaia-bench.ts is a follow-up PR (in-flight conflict avoidance).');
|
|
293
|
+
process.exit(0);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
runAll().catch((err) => {
|
|
297
|
+
console.error('Unhandled error in smoke runner:', err);
|
|
298
|
+
process.exit(1);
|
|
299
|
+
});
|
|
300
|
+
//# sourceMappingURL=gaia-causal-memory.smoke.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gaia-causal-memory.smoke.js","sourceRoot":"","sources":["../../../src/benchmarks/gaia-causal-memory.smoke.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EACL,wBAAwB,EACxB,gBAAgB,EAChB,oBAAoB,EACpB,mBAAmB,GAEpB,MAAM,yBAAyB,CAAC;AAIjC,8EAA8E;AAC9E,wCAAwC;AACxC,8EAA8E;AAE9E,IAAI,MAAM,GAAG,CAAC,CAAC;AACf,IAAI,MAAM,GAAG,CAAC,CAAC;AACf,MAAM,OAAO,GAAyD,EAAE,CAAC;AAEzE,KAAK,UAAU,IAAI,CAAC,IAAY,EAAE,EAAuB;IACvD,IAAI,CAAC;QACH,MAAM,EAAE,EAAE,CAAC;QACX,MAAM,EAAE,CAAC;QACT,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,EAAE,CAAC;QACT,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7D,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QAC9C,OAAO,CAAC,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;QACjC,OAAO,CAAC,KAAK,CAAC,WAAW,GAAG,EAAE,CAAC,CAAC;IAClC,CAAC;AACH,CAAC;AAED,SAAS,MAAM,CAAC,SAAkB,EAAE,OAAe;IACjD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,qBAAqB,OAAO,EAAE,CAAC,CAAC;IAClD,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAI,MAAS,EAAE,QAAW,EAAE,KAAa;IAC3D,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,cAAc,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACnG,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,WAAW;AACX,8EAA8E;AAE9E,SAAS,YAAY,CAAC,IAAY;IAChC,OAAO;QACL,OAAO,EAAE,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;QACpC,KAAK,EAAE,CAAC;QACR,QAAQ,EAAE,IAAI;QACd,YAAY,EAAE,kBAAkB;QAChC,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,IAAI;KAChB,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,YAAsC,EAAE;IAC1D,OAAO;QACL,UAAU,EAAE,UAAU;QACtB,WAAW,EAAE,cAAc;QAC3B,KAAK,EAAE,CAAC;QACR,eAAe,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE;QAClC,gBAAgB,EAAE,GAAG;QACrB,iBAAiB,EAAE,EAAE;QACrB,MAAM,EAAE,IAAI;QACZ,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E,KAAK,UAAU,MAAM;IACnB,OAAO,CAAC,GAAG,CAAC,8DAA8D,CAAC,CAAC;IAE5E,2DAA2D;IAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,qBAAqB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAE5E,4EAA4E;IAC5E,gFAAgF;IAChF,4EAA4E;IAC5E,MAAM,IAAI,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;QACtF,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACrC,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAEhD,MAAM,QAAQ,GAAG,YAAY,CAAC,gCAAgC,CAAC,CAAC;QAChE,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;QAErD,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAClF,MAAM,CAAC,MAAM,CAAC,aAAa,GAAG,CAAC,EAAE,wCAAwC,CAAC,CAAC;QAE3E,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAClE,MAAM,CAAC,MAAM,CAAC,YAAY,GAAG,CAAC,EAAE,8BAA8B,CAAC,CAAC;QAChE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,0BAA0B,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,6CAA6C,CAAC,CAAC;QAChG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,qCAAqC,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,uEAAuE;IACvE,4EAA4E;IAC5E,MAAM,IAAI,CAAC,iEAAiE,EAAE,KAAK,IAAI,EAAE;QACvF,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACrC,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAEhD,MAAM,SAAS,GAAG;YAChB,YAAY,CAAC,gCAAgC,CAAC;YAC9C,YAAY,CAAC,mBAAmB,CAAC;YACjC,YAAY,CAAC,gBAAgB,CAAC;SAC/B,CAAC;QAEF,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;YAC1B,MAAM,oBAAoB,CAAC,CAAC,EAAE,UAAU,EAAE,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,SAAS,GAAG,YAAY,CAAC,6FAA6F,CAAC,CAAC;QAC9H,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAEnE,WAAW,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC,EAAE,qCAAqC,CAAC,CAAC;QAC3E,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,EAAE,6BAA6B,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,uEAAuE;IACvE,4EAA4E;IAC5E,MAAM,IAAI,CAAC,iEAAiE,EAAE,KAAK,IAAI,EAAE;QACvF,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACrC,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAEhD,MAAM,QAAQ,GAAG,YAAY,CAAC,iCAAiC,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,eAAe,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAElE,4BAA4B;QAC5B,MAAM,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QACnE,MAAM,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAEnE,8DAA8D;QAC9D,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACpE,MAAM,KAAK,GAAiB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAe,CAAC,CAAC;QAE1E,4DAA4D;QAC5D,MAAM,GAAG,GAAG,wBAAwB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACxD,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,KAAK,GAAG,IAAI,CAAC,CAAC,UAAU,KAAK,YAAY,CAAC,CAAC;QACnG,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,sDAAsD,CAAC,CAAC;QACxF,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,EAAE,sCAAsC,CAAC,CAAC;IACtF,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,iEAAiE;IACjE,4EAA4E;IAC5E,MAAM,IAAI,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;QAClF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,EAAE,aAAa,CAAC,CAAC;QACnE,MAAM,QAAQ,GAAG,YAAY,CAAC,mCAAmC,CAAC,CAAC;QAEnE,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAClE,WAAW,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC,EAAE,+BAA+B,CAAC,CAAC;QACrE,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,EAAE,uBAAuB,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,wDAAwD;IACxD,4EAA4E;IAC5E,MAAM,IAAI,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QACzE,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACrC,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAEhD,MAAM,YAAY,GAAG,YAAY,CAAC,8CAA8C,CAAC,CAAC;QAClF,MAAM,QAAQ,GAAe;YAC3B,iBAAiB,EAAE,wBAAwB,CAAC,YAAY,CAAC,QAAQ,CAAC;YAClE,UAAU,EAAE,aAAa;YACzB,oBAAoB,EAAE,oDAAoD;YAC1E,mBAAmB,EAAE,cAAc;YACnC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,eAAe,EAAE,CAAC;SACnB,CAAC;QAEF,4CAA4C;QAC5C,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3D,EAAE,CAAC,aAAa,CACd,SAAS,EACT,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,IAAI;YAC/B,iCAAiC;YACjC,mBAAmB,EACnB,MAAM,CACP,CAAC;QAEF,qCAAqC;QACrC,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QACtE,WAAW,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC,EAAE,sCAAsC,CAAC,CAAC;QAC5E,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,wCAAwC,CAAC,CAAC;QAC3F,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,mCAAmC,CAAC,CAAC;IACnF,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,6CAA6C;IAC7C,4EAA4E;IAC5E,MAAM,IAAI,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;QAC7D,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACrC,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAChD,MAAM,oBAAoB,GAAG,CAAC,CAAC;QAE/B,MAAM,QAAQ,GAAG,YAAY,CAAC,mDAAmD,CAAC,CAAC;QAEnF,yEAAyE;QACzE,KAAK,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,WAAW,EAAE,aAAa,CAAC,EAAE,CAAC;YAC9D,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,eAAe,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC9D,MAAM,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE;gBAClD,SAAS;gBACT,oBAAoB;aACrB,CAAC,CAAC;QACL,CAAC;QAED,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACpE,MAAM,GAAG,GAAG,wBAAwB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACxD,MAAM,QAAQ,GAAG,KAAK;aACnB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAe,CAAC;aACvC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,KAAK,GAAG,CAAC,CAAC;QAE9C,MAAM,CACJ,QAAQ,CAAC,MAAM,IAAI,oBAAoB,EACvC,uBAAuB,oBAAoB,eAAe,QAAQ,CAAC,MAAM,EAAE,CAC5E,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,iDAAiD;IACjD,4EAA4E;IAC5E,MAAM,IAAI,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACjE,MAAM,IAAI,GAAG,kDAAkD,CAAC;QAEhE,MAAM,IAAI,GAAG,wBAAwB,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,IAAI,GAAG,wBAAwB,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,IAAI,GAAG,wBAAwB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,gCAAgC;QACpF,MAAM,IAAI,GAAG,wBAAwB,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,gBAAgB;QAEzF,WAAW,CAAC,IAAI,EAAE,IAAI,EAAE,kDAAkD,CAAC,CAAC;QAC5E,WAAW,CAAC,IAAI,EAAE,IAAI,EAAE,uCAAuC,CAAC,CAAC;QACjE,WAAW,CAAC,IAAI,EAAE,IAAI,EAAE,+CAA+C,CAAC,CAAC;QACzE,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,EAAE,gCAAgC,CAAC,CAAC;QAE/D,oFAAoF;QACpF,MAAM,QAAQ,GAAG,wBAAwB,CAAC,gDAAgD,CAAC,CAAC;QAC5F,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,+CAA+C,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,wDAAwD;IACxD,4EAA4E;IAC5E,MAAM,IAAI,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;QACxE,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACrC,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAEhD,MAAM,QAAQ,GAAG,YAAY,CAAC,uCAAuC,CAAC,CAAC;QACvE,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;QAEnD,oBAAoB;QACpB,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QACjF,WAAW,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC,EAAE,iCAAiC,CAAC,CAAC;QACxE,MAAM,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,qDAAqD,CAAC,CAAC;IAC3F,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,sCAAsC;IACtC,4EAA4E;IAC5E,MAAM,IAAI,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;QAClE,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;QACnD,MAAM,EAAE,GAAG,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC1C,WAAW,CAAC,EAAE,EAAE,IAAI,EAAE,mDAAmD,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QACpE,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,MAAM,EAAE,GAAG,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC3C,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,iDAAiD,CAAC,CAAC;IAChF,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QACvE,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,CAAC;QACxD,MAAM,EAAE,GAAG,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC3C,WAAW,CAAC,EAAE,EAAE,YAAY,EAAE,kDAAkD,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;QAC/E,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;QACjD,MAAM,EAAE,GAAG,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC3C,WAAW,CAAC,EAAE,EAAE,cAAc,EAAE,qDAAqD,CAAC,CAAC;IACzF,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QACzE,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,WAAW,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAC9D,MAAM,EAAE,GAAG,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC3C,WAAW,CAAC,EAAE,EAAE,cAAc,EAAE,uDAAuD,CAAC,CAAC;IAC3F,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,UAAU;IACV,4EAA4E;IAC5E,IAAI,CAAC;QACH,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACvD,CAAC;IAAC,MAAM,CAAC;QACP,+DAA+D;IACjE,CAAC;IAED,4EAA4E;IAC5E,UAAU;IACV,4EAA4E;IAC5E,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,YAAY,MAAM,YAAY,MAAM,YAAY,MAAM,GAAG,MAAM,SAAS,CAAC,CAAC;IAEtF,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACjC,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;YAC7C,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QAC3C,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;QACzC,OAAO,CAAC,GAAG,CACT,0EAA0E,CAC3E,CAAC;QACF,OAAO,CAAC,GAAG,CACT,6EAA6E,CAC9E,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACrB,OAAO,CAAC,KAAK,CAAC,kCAAkC,EAAE,GAAG,CAAC,CAAC;IACvD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* GAIA Claude-p Wrapper — iter 54 (#2156)
|
|
3
|
+
*
|
|
4
|
+
* Delegates each GAIA question to `claude -p` (Claude Code headless mode),
|
|
5
|
+
* which gives us WebSearch, WebFetch, Read (multimodal incl. PDF/DOCX/images),
|
|
6
|
+
* and Bash (Python execution) for free — the same tools HAL uses.
|
|
7
|
+
*
|
|
8
|
+
* Why this approach over a native TS CodeAgent:
|
|
9
|
+
* - HAL gaps vs ruflo were: visit_webpage, file reading (PDF/DOCX/XLSX/images),
|
|
10
|
+
* Python execution. Claude Code's built-in tools solve ALL of these.
|
|
11
|
+
* - No wheel-reinvention: battle-tested tool infra, native multimodal, proper
|
|
12
|
+
* tool-budget management, Anthropic WebSearch API.
|
|
13
|
+
* - Baseline: 24/53 (45.3%). Target: ≥45/53 to surpass HAL's 82.07%.
|
|
14
|
+
*
|
|
15
|
+
* SECURITY NOTE on --dangerously-skip-permissions:
|
|
16
|
+
* This flag is ONLY used inside the GAIA harness context, which is a sandboxed
|
|
17
|
+
* benchmark evaluation environment. GAIA questions have no real-world
|
|
18
|
+
* side effects — they are read-only research questions. The flag lets Claude Code
|
|
19
|
+
* use its tools (WebSearch, WebFetch, Read, Bash) without per-tool permission
|
|
20
|
+
* prompts, which is required for unattended benchmark execution. It MUST NOT
|
|
21
|
+
* be used in production workflows where Claude Code could affect real systems.
|
|
22
|
+
*
|
|
23
|
+
* JSON output format from `claude -p --output-format json`:
|
|
24
|
+
* {
|
|
25
|
+
* type: "result",
|
|
26
|
+
* subtype: "success" | "error_max_budget_usd" | ...,
|
|
27
|
+
* is_error: boolean,
|
|
28
|
+
* result: string, // final assistant message text
|
|
29
|
+
* total_cost_usd: number,
|
|
30
|
+
* duration_ms: number,
|
|
31
|
+
* num_turns: number,
|
|
32
|
+
* ...
|
|
33
|
+
* }
|
|
34
|
+
*
|
|
35
|
+
* Refs: ADR-138 (reference, NOT implemented), iter 54, #2156
|
|
36
|
+
*/
|
|
37
|
+
import type { GaiaQuestion } from './gaia-loader.js';
|
|
38
|
+
/** Default model for claude -p GAIA runs. Sonnet for quality parity with HAL. */
|
|
39
|
+
export declare const CLAUDE_P_DEFAULT_MODEL = "claude-sonnet-4-6";
|
|
40
|
+
/** Per-question budget cap (USD). HAL uses Sonnet 4.5 so $0.30 headroom is safe. */
|
|
41
|
+
export declare const CLAUDE_P_PER_QUESTION_BUDGET_USD = 0.3;
|
|
42
|
+
/** Subprocess timeout: 5 minutes per question. */
|
|
43
|
+
export declare const CLAUDE_P_TIMEOUT_MS: number;
|
|
44
|
+
export interface ClaudePResult {
|
|
45
|
+
/** The extracted answer, or null if extraction failed. */
|
|
46
|
+
finalAnswer: string | null;
|
|
47
|
+
/** Raw result text from claude -p. */
|
|
48
|
+
rawResult: string;
|
|
49
|
+
/** Whether claude -p exited with an error. */
|
|
50
|
+
isError: boolean;
|
|
51
|
+
/** claude -p's reported error message (if any). */
|
|
52
|
+
errorMessage?: string;
|
|
53
|
+
/** Actual cost reported by claude -p. */
|
|
54
|
+
costUsd: number;
|
|
55
|
+
/** Wall-clock time in ms. */
|
|
56
|
+
wallMs: number;
|
|
57
|
+
/** Number of turns claude -p used. */
|
|
58
|
+
numTurns: number;
|
|
59
|
+
/** claude -p stop reason. */
|
|
60
|
+
stopReason?: string;
|
|
61
|
+
}
|
|
62
|
+
export interface ClaudePOptions {
|
|
63
|
+
/** Model ID (default: CLAUDE_P_DEFAULT_MODEL). */
|
|
64
|
+
model?: string;
|
|
65
|
+
/** Per-question budget cap in USD (default: CLAUDE_P_PER_QUESTION_BUDGET_USD). */
|
|
66
|
+
budgetUsd?: number;
|
|
67
|
+
/** Timeout in ms (default: CLAUDE_P_TIMEOUT_MS). */
|
|
68
|
+
timeoutMs?: number;
|
|
69
|
+
/** Absolute path to the claude binary (default: resolved from $PATH). */
|
|
70
|
+
claudeBin?: string;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Build the prompt sent to claude -p for a GAIA question.
|
|
74
|
+
*
|
|
75
|
+
* Includes the question text, optional attachment path, and precise instructions
|
|
76
|
+
* for using available tools and producing FINAL_ANSWER: in the expected format.
|
|
77
|
+
*/
|
|
78
|
+
export declare function buildClaudePPrompt(question: GaiaQuestion): string;
|
|
79
|
+
/**
|
|
80
|
+
* Run a single GAIA question via `claude -p` headless mode.
|
|
81
|
+
*
|
|
82
|
+
* Spawns a subprocess, captures JSON output, extracts the final answer.
|
|
83
|
+
*/
|
|
84
|
+
export declare function runGaiaQuestionViaClaudeP(question: GaiaQuestion, options?: ClaudePOptions): Promise<ClaudePResult>;
|
|
85
|
+
/**
|
|
86
|
+
* Extract the FINAL_ANSWER value from claude -p's result text.
|
|
87
|
+
*
|
|
88
|
+
* Primary: regex match on `FINAL_ANSWER: <value>`
|
|
89
|
+
* Fallback: last non-empty line if no FINAL_ANSWER marker found.
|
|
90
|
+
*/
|
|
91
|
+
export declare function extractFinalAnswer(text: string): string | null;
|
|
92
|
+
export interface ClaudePBatchOptions extends ClaudePOptions {
|
|
93
|
+
/** Max parallel questions (default: 2 — claude -p uses significant local resources). */
|
|
94
|
+
concurrency?: number;
|
|
95
|
+
/** Callback for per-question progress logging. */
|
|
96
|
+
onProgress?: (idx: number, total: number, questionId: string, answer: string | null, costUsd: number) => void;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Run a batch of GAIA questions through the claude -p wrapper.
|
|
100
|
+
*
|
|
101
|
+
* Concurrency is limited (default 2) because each claude -p subprocess
|
|
102
|
+
* is heavyweight — it starts a full Claude Code session with LSP etc.
|
|
103
|
+
*/
|
|
104
|
+
export declare function runGaiaQuestionsBatchViaClaudeP(questions: GaiaQuestion[], options?: ClaudePBatchOptions): Promise<ClaudePResult[]>;
|
|
105
|
+
//# sourceMappingURL=gaia-claude-p.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gaia-claude-p.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/gaia-claude-p.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMrD,iFAAiF;AACjF,eAAO,MAAM,sBAAsB,sBAAsB,CAAC;AAE1D,oFAAoF;AACpF,eAAO,MAAM,gCAAgC,MAAO,CAAC;AAErD,kDAAkD;AAClD,eAAO,MAAM,mBAAmB,QAAgB,CAAC;AASjD,MAAM,WAAW,aAAa;IAC5B,0DAA0D;IAC1D,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,sCAAsC;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,8CAA8C;IAC9C,OAAO,EAAE,OAAO,CAAC;IACjB,mDAAmD;IACnD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,yCAAyC;IACzC,OAAO,EAAE,MAAM,CAAC;IAChB,6BAA6B;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,6BAA6B;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,kDAAkD;IAClD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,kFAAkF;IAClF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yEAAyE;IACzE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAMD;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,YAAY,GAAG,MAAM,CAiCjE;AAMD;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,QAAQ,EAAE,YAAY,EACtB,OAAO,GAAE,cAAmB,GAC3B,OAAO,CAAC,aAAa,CAAC,CAsFxB;AA4ED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAiB9D;AAMD,MAAM,WAAW,mBAAoB,SAAQ,cAAc;IACzD,wFAAwF;IACxF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kDAAkD;IAClD,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CAC/G;AAED;;;;;GAKG;AACH,wBAAsB,+BAA+B,CACnD,SAAS,EAAE,YAAY,EAAE,EACzB,OAAO,GAAE,mBAAwB,GAChC,OAAO,CAAC,aAAa,EAAE,CAAC,CAqB1B"}
|