@claude-flow/cli 3.11.0 → 3.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. package/dist/src/commands/doctor.d.ts.map +1 -1
  2. package/dist/src/commands/doctor.js +175 -1
  3. package/dist/src/commands/doctor.js.map +1 -1
  4. package/dist/src/commands/eject.d.ts +33 -0
  5. package/dist/src/commands/eject.d.ts.map +1 -0
  6. package/dist/src/commands/eject.js +195 -0
  7. package/dist/src/commands/eject.js.map +1 -0
  8. package/dist/src/commands/hooks.d.ts.map +1 -1
  9. package/dist/src/commands/hooks.js +2 -1
  10. package/dist/src/commands/hooks.js.map +1 -1
  11. package/dist/src/commands/index.d.ts.map +1 -1
  12. package/dist/src/commands/index.js +4 -0
  13. package/dist/src/commands/index.js.map +1 -1
  14. package/dist/src/commands/init.d.ts.map +1 -1
  15. package/dist/src/commands/init.js +3 -0
  16. package/dist/src/commands/init.js.map +1 -1
  17. package/dist/src/commands/metaharness.d.ts +39 -0
  18. package/dist/src/commands/metaharness.d.ts.map +1 -0
  19. package/dist/src/commands/metaharness.js +194 -0
  20. package/dist/src/commands/metaharness.js.map +1 -0
  21. package/dist/src/mcp-client.d.ts.map +1 -1
  22. package/dist/src/mcp-client.js +4 -0
  23. package/dist/src/mcp-client.js.map +1 -1
  24. package/dist/src/mcp-tools/index.d.ts +1 -0
  25. package/dist/src/mcp-tools/index.d.ts.map +1 -1
  26. package/dist/src/mcp-tools/index.js +2 -0
  27. package/dist/src/mcp-tools/index.js.map +1 -1
  28. package/dist/src/mcp-tools/metaharness-tools.d.ts +31 -0
  29. package/dist/src/mcp-tools/metaharness-tools.d.ts.map +1 -0
  30. package/dist/src/mcp-tools/metaharness-tools.js +372 -0
  31. package/dist/src/mcp-tools/metaharness-tools.js.map +1 -0
  32. package/dist/src/plugins/store/discovery.d.ts.map +1 -1
  33. package/dist/src/plugins/store/discovery.js +3 -0
  34. package/dist/src/plugins/store/discovery.js.map +1 -1
  35. package/dist/src/plugins/store/types.d.ts +6 -1
  36. package/dist/src/plugins/store/types.d.ts.map +1 -1
  37. package/dist/src/ruvector/model-router.d.ts.map +1 -1
  38. package/dist/src/ruvector/model-router.js +37 -0
  39. package/dist/src/ruvector/model-router.js.map +1 -1
  40. package/dist/src/ruvector/router-parallel-recorder.d.ts +127 -0
  41. package/dist/src/ruvector/router-parallel-recorder.d.ts.map +1 -0
  42. package/dist/src/ruvector/router-parallel-recorder.js +183 -0
  43. package/dist/src/ruvector/router-parallel-recorder.js.map +1 -0
  44. package/dist/tsconfig.tsbuildinfo +1 -1
  45. package/package.json +11 -9
  46. package/dist/src/benchmarks/capability-tasks.d.ts +0 -126
  47. package/dist/src/benchmarks/capability-tasks.d.ts.map +0 -1
  48. package/dist/src/benchmarks/capability-tasks.js +0 -147
  49. package/dist/src/benchmarks/capability-tasks.js.map +0 -1
  50. package/dist/src/benchmarks/gaia-agent-contrastive.d.ts +0 -53
  51. package/dist/src/benchmarks/gaia-agent-contrastive.d.ts.map +0 -1
  52. package/dist/src/benchmarks/gaia-agent-contrastive.js +0 -177
  53. package/dist/src/benchmarks/gaia-agent-contrastive.js.map +0 -1
  54. package/dist/src/benchmarks/gaia-agent-gemini.d.ts +0 -51
  55. package/dist/src/benchmarks/gaia-agent-gemini.d.ts.map +0 -1
  56. package/dist/src/benchmarks/gaia-agent-gemini.js +0 -352
  57. package/dist/src/benchmarks/gaia-agent-gemini.js.map +0 -1
  58. package/dist/src/benchmarks/gaia-claude-p.d.ts +0 -105
  59. package/dist/src/benchmarks/gaia-claude-p.d.ts.map +0 -1
  60. package/dist/src/benchmarks/gaia-claude-p.js +0 -269
  61. package/dist/src/benchmarks/gaia-claude-p.js.map +0 -1
  62. package/dist/src/benchmarks/gaia-claude-p.smoke.d.ts +0 -20
  63. package/dist/src/benchmarks/gaia-claude-p.smoke.d.ts.map +0 -1
  64. package/dist/src/benchmarks/gaia-claude-p.smoke.js +0 -168
  65. package/dist/src/benchmarks/gaia-claude-p.smoke.js.map +0 -1
  66. package/dist/src/benchmarks/gaia-codeagent-runner.py +0 -556
  67. package/dist/src/benchmarks/gaia-codeagent.d.ts +0 -129
  68. package/dist/src/benchmarks/gaia-codeagent.d.ts.map +0 -1
  69. package/dist/src/benchmarks/gaia-codeagent.js +0 -590
  70. package/dist/src/benchmarks/gaia-codeagent.js.map +0 -1
  71. package/dist/src/benchmarks/gaia-codeagent.smoke.d.ts +0 -33
  72. package/dist/src/benchmarks/gaia-codeagent.smoke.d.ts.map +0 -1
  73. package/dist/src/benchmarks/gaia-codeagent.smoke.js +0 -241
  74. package/dist/src/benchmarks/gaia-codeagent.smoke.js.map +0 -1
  75. package/dist/src/benchmarks/gaia-dag-pilot.d.ts +0 -22
  76. package/dist/src/benchmarks/gaia-dag-pilot.d.ts.map +0 -1
  77. package/dist/src/benchmarks/gaia-dag-pilot.js +0 -158
  78. package/dist/src/benchmarks/gaia-dag-pilot.js.map +0 -1
  79. package/dist/src/benchmarks/gaia-dag.d.ts +0 -110
  80. package/dist/src/benchmarks/gaia-dag.d.ts.map +0 -1
  81. package/dist/src/benchmarks/gaia-dag.js +0 -658
  82. package/dist/src/benchmarks/gaia-dag.js.map +0 -1
  83. package/dist/src/benchmarks/gaia-dag.smoke.d.ts +0 -21
  84. package/dist/src/benchmarks/gaia-dag.smoke.d.ts.map +0 -1
  85. package/dist/src/benchmarks/gaia-dag.smoke.js +0 -290
  86. package/dist/src/benchmarks/gaia-dag.smoke.js.map +0 -1
  87. package/dist/src/benchmarks/gaia-ensemble-pilot.d.ts +0 -17
  88. package/dist/src/benchmarks/gaia-ensemble-pilot.d.ts.map +0 -1
  89. package/dist/src/benchmarks/gaia-ensemble-pilot.js +0 -106
  90. package/dist/src/benchmarks/gaia-ensemble-pilot.js.map +0 -1
  91. package/dist/src/benchmarks/gaia-ensemble.d.ts +0 -103
  92. package/dist/src/benchmarks/gaia-ensemble.d.ts.map +0 -1
  93. package/dist/src/benchmarks/gaia-ensemble.js +0 -462
  94. package/dist/src/benchmarks/gaia-ensemble.js.map +0 -1
  95. package/dist/src/benchmarks/gaia-gemini-pilot.d.ts +0 -17
  96. package/dist/src/benchmarks/gaia-gemini-pilot.d.ts.map +0 -1
  97. package/dist/src/benchmarks/gaia-gemini-pilot.js +0 -168
  98. package/dist/src/benchmarks/gaia-gemini-pilot.js.map +0 -1
  99. package/dist/src/benchmarks/gaia-mode-router.d.ts +0 -67
  100. package/dist/src/benchmarks/gaia-mode-router.d.ts.map +0 -1
  101. package/dist/src/benchmarks/gaia-mode-router.js +0 -126
  102. package/dist/src/benchmarks/gaia-mode-router.js.map +0 -1
  103. package/dist/src/benchmarks/gaia-mode-router.smoke.d.ts +0 -15
  104. package/dist/src/benchmarks/gaia-mode-router.smoke.d.ts.map +0 -1
  105. package/dist/src/benchmarks/gaia-mode-router.smoke.js +0 -131
  106. package/dist/src/benchmarks/gaia-mode-router.smoke.js.map +0 -1
  107. package/dist/src/benchmarks/gaia-tools/grounded_query.smoke.d.ts +0 -24
  108. package/dist/src/benchmarks/gaia-tools/grounded_query.smoke.d.ts.map +0 -1
  109. package/dist/src/benchmarks/gaia-tools/grounded_query.smoke.js +0 -276
  110. package/dist/src/benchmarks/gaia-tools/grounded_query.smoke.js.map +0 -1
  111. package/dist/src/benchmarks/gaia-tools/image_describe.d.ts +0 -45
  112. package/dist/src/benchmarks/gaia-tools/image_describe.d.ts.map +0 -1
  113. package/dist/src/benchmarks/gaia-tools/image_describe.js +0 -222
  114. package/dist/src/benchmarks/gaia-tools/image_describe.js.map +0 -1
  115. package/dist/src/benchmarks/gaia-tools/image_describe.smoke.d.ts +0 -24
  116. package/dist/src/benchmarks/gaia-tools/image_describe.smoke.d.ts.map +0 -1
  117. package/dist/src/benchmarks/gaia-tools/image_describe.smoke.js +0 -169
  118. package/dist/src/benchmarks/gaia-tools/image_describe.smoke.js.map +0 -1
  119. package/dist/src/benchmarks/gaia-tools/pdf_read.d.ts +0 -27
  120. package/dist/src/benchmarks/gaia-tools/pdf_read.d.ts.map +0 -1
  121. package/dist/src/benchmarks/gaia-tools/pdf_read.js +0 -226
  122. package/dist/src/benchmarks/gaia-tools/pdf_read.js.map +0 -1
  123. package/dist/src/benchmarks/gaia-tools/python_exec.d.ts +0 -31
  124. package/dist/src/benchmarks/gaia-tools/python_exec.d.ts.map +0 -1
  125. package/dist/src/benchmarks/gaia-tools/python_exec.js +0 -94
  126. package/dist/src/benchmarks/gaia-tools/python_exec.js.map +0 -1
  127. package/dist/src/benchmarks/gaia-tools/python_exec.smoke.d.ts +0 -13
  128. package/dist/src/benchmarks/gaia-tools/python_exec.smoke.d.ts.map +0 -1
  129. package/dist/src/benchmarks/gaia-tools/python_exec.smoke.js +0 -98
  130. package/dist/src/benchmarks/gaia-tools/python_exec.smoke.js.map +0 -1
  131. package/dist/src/benchmarks/gaia-tools/visit_webpage.d.ts +0 -52
  132. package/dist/src/benchmarks/gaia-tools/visit_webpage.d.ts.map +0 -1
  133. package/dist/src/benchmarks/gaia-tools/visit_webpage.js +0 -304
  134. package/dist/src/benchmarks/gaia-tools/visit_webpage.js.map +0 -1
  135. package/dist/src/benchmarks/gaia-tools/web_browse.d.ts +0 -61
  136. package/dist/src/benchmarks/gaia-tools/web_browse.d.ts.map +0 -1
  137. package/dist/src/benchmarks/gaia-tools/web_browse.js +0 -217
  138. package/dist/src/benchmarks/gaia-tools/web_browse.js.map +0 -1
  139. package/dist/src/benchmarks/gaia-tools/web_browse.smoke.d.ts +0 -26
  140. package/dist/src/benchmarks/gaia-tools/web_browse.smoke.d.ts.map +0 -1
  141. package/dist/src/benchmarks/gaia-tools/web_browse.smoke.js +0 -152
  142. package/dist/src/benchmarks/gaia-tools/web_browse.smoke.js.map +0 -1
  143. package/dist/src/benchmarks/gaia-tools/web_search.smoke.d.ts +0 -24
  144. package/dist/src/benchmarks/gaia-tools/web_search.smoke.d.ts.map +0 -1
  145. package/dist/src/benchmarks/gaia-tools/web_search.smoke.js +0 -311
  146. package/dist/src/benchmarks/gaia-tools/web_search.smoke.js.map +0 -1
  147. package/dist/src/benchmarks/gaia-visit-webpage.smoke.d.ts +0 -25
  148. package/dist/src/benchmarks/gaia-visit-webpage.smoke.d.ts.map +0 -1
  149. package/dist/src/benchmarks/gaia-visit-webpage.smoke.js +0 -304
  150. package/dist/src/benchmarks/gaia-visit-webpage.smoke.js.map +0 -1
  151. package/dist/src/commands/performance-capability.d.ts +0 -30
  152. package/dist/src/commands/performance-capability.d.ts.map +0 -1
  153. package/dist/src/commands/performance-capability.js +0 -349
  154. package/dist/src/commands/performance-capability.js.map +0 -1
@@ -1,462 +0,0 @@
1
- /**
2
- * GAIA Ensemble Runner — ADR-139
3
- *
4
- * Runs a GAIA question through N models in parallel, aggregates answers via
5
- * majority vote, with a judge-model tiebreak when no consensus is reached.
6
- *
7
- * Architecture:
8
- * 1. Each model runs independently using the full tool harness.
9
- * 2. Answers are normalised (via normaliseAnswer from gaia-judge.ts).
10
- * 3. Majority vote: if ≥2 models agree on normalised answer → that wins.
11
- * 4. Tiebreak: when all answers differ (or N=2 with disagreement), the judge
12
- * model picks the best answer with a brief rationale.
13
- * 5. Abstain: if all models return null/timedOut → failed question.
14
- *
15
- * Supported models:
16
- * - Claude (claude-sonnet-4-6, etc.) via Anthropic API (gaia-agent.ts)
17
- * - Gemini (gemini-2.5-pro, etc.) via Google AI API (gaia-agent-gemini.js compiled)
18
- * - OpenRouter (gpt-5, deepseek-v3.2, kimi-k2, etc.) via OpenAI-compatible API
19
- * NOTE: OpenRouter requires funded account — returns 402 when credits exhausted.
20
- *
21
- * CLI integration:
22
- * gaia-bench run --mode=ensemble --models=claude-sonnet-4-6,gemini-2.5-pro,openai/gpt-5
23
- *
24
- * Cost model (per question, typical L1 ~10k input + ~3k output tokens):
25
- * claude-sonnet-4-6: ~$0.075
26
- * gemini-2.5-pro: ~$0.043
27
- * openai/gpt-5 (OR): ~$0.043
28
- * 3-model total: ~$0.161 (53-Q: ~$8.54, 300-Q: ~$48.30)
29
- *
30
- * Refs: ADR-139, ADR-133, ADR-135, #2156
31
- */
32
- import { execSync } from 'node:child_process';
33
- import { runGaiaAgent, resolveAnthropicApiKey } from './gaia-agent.js';
34
- import { normaliseAnswer } from './gaia-judge.js';
35
- // ---------------------------------------------------------------------------
36
- // Constants
37
- // ---------------------------------------------------------------------------
38
- const OPENROUTER_API_BASE = 'https://openrouter.ai/api/v1';
39
- const ANTHROPIC_API_URL = 'https://api.anthropic.com/v1/messages';
40
- const ANTHROPIC_API_VERSION = '2023-06-01';
41
- const DEFAULT_JUDGE_MODEL = 'claude-sonnet-4-6';
42
- const DEFAULT_TIEBREAK_MAX_TOKENS = 300;
43
- const FINAL_ANSWER_RE = /FINAL_ANSWER:\s*(.+)/i;
44
- /** Prefix for OpenRouter model IDs — any model containing "/" is treated as OR. */
45
- const OPENROUTER_MODEL_MARKER = '/';
46
- /** Prefix for Gemini model IDs. */
47
- const GEMINI_MODEL_PREFIX = 'gemini-';
48
- // ---------------------------------------------------------------------------
49
- // Pricing constants (USD per million tokens)
50
- // ---------------------------------------------------------------------------
51
- const MODEL_PRICING = {
52
- 'claude-haiku-4-5': { inputPerM: 0.25, outputPerM: 1.25 },
53
- 'claude-sonnet-4-5': { inputPerM: 3.0, outputPerM: 15.0 },
54
- 'claude-sonnet-4-6': { inputPerM: 3.0, outputPerM: 15.0 },
55
- 'claude-opus-4-7': { inputPerM: 5.0, outputPerM: 25.0 },
56
- 'gemini-2.5-pro': { inputPerM: 1.25, outputPerM: 10.0 },
57
- 'gemini-2.5-flash': { inputPerM: 0.30, outputPerM: 2.50 },
58
- // OpenRouter models — verified available 2026-05-28
59
- 'openai/gpt-5': { inputPerM: 1.25, outputPerM: 10.0 },
60
- 'openai/gpt-5.4': { inputPerM: 2.50, outputPerM: 15.0 },
61
- 'openai/o3': { inputPerM: 2.00, outputPerM: 8.0 },
62
- 'deepseek/deepseek-v3.2': { inputPerM: 0.252, outputPerM: 0.378 },
63
- 'moonshotai/kimi-k2': { inputPerM: 0.57, outputPerM: 2.30 },
64
- 'moonshotai/kimi-k2.6': { inputPerM: 0.73, outputPerM: 3.49 },
65
- 'qwen/qwen3.7-max': { inputPerM: 1.25, outputPerM: 3.75 },
66
- };
67
- function estimateCostUsd(model, inputTokens, outputTokens) {
68
- const pricing = MODEL_PRICING[model] ?? { inputPerM: 3.0, outputPerM: 15.0 };
69
- return (inputTokens / 1_000_000) * pricing.inputPerM +
70
- (outputTokens / 1_000_000) * pricing.outputPerM;
71
- }
72
- // ---------------------------------------------------------------------------
73
- // API key resolution
74
- // ---------------------------------------------------------------------------
75
- function resolveGeminiApiKey(supplied) {
76
- if (supplied?.trim())
77
- return supplied.trim();
78
- const env = process.env['GOOGLE_AI_API_KEY'] || process.env['GEMINI_API_KEY'];
79
- if (env?.trim())
80
- return env.trim();
81
- try {
82
- const out = execSync('gcloud secrets versions access latest --secret=GOOGLE_AI_API_KEY 2>/dev/null', { encoding: 'utf-8', timeout: 10_000 }).trim();
83
- if (out)
84
- return out;
85
- }
86
- catch { /* fall through */ }
87
- try {
88
- const out = execSync('gcloud secrets versions access latest --secret=GEMINI_API_KEY 2>/dev/null', { encoding: 'utf-8', timeout: 10_000 }).trim();
89
- if (out)
90
- return out;
91
- }
92
- catch { /* fall through */ }
93
- throw new Error('GOOGLE_AI_API_KEY not found in env or GCP Secret Manager.');
94
- }
95
- function resolveOpenRouterApiKey(supplied) {
96
- if (supplied?.trim())
97
- return supplied.trim();
98
- const env = process.env['OPENROUTER_API_KEY'];
99
- if (env?.trim())
100
- return env.trim();
101
- try {
102
- const out = execSync('gcloud secrets versions access latest --secret=OPENROUTER_API_KEY 2>/dev/null', { encoding: 'utf-8', timeout: 10_000 }).trim();
103
- if (out)
104
- return out;
105
- }
106
- catch { /* fall through */ }
107
- return null;
108
- }
109
- // ---------------------------------------------------------------------------
110
- // Model dispatch helpers
111
- // ---------------------------------------------------------------------------
112
- function isGeminiModel(modelId) {
113
- return modelId.startsWith(GEMINI_MODEL_PREFIX);
114
- }
115
- function isOpenRouterModel(modelId) {
116
- return modelId.includes(OPENROUTER_MODEL_MARKER);
117
- }
118
- /** Run a single Gemini model via REST API (mirrors gaia-agent-gemini compiled logic). */
119
- async function runGeminiModel(question, model, apiKey, maxTurns, maxTokensPerTurn, perTurnTimeoutMs) {
120
- const wallStart = Date.now();
121
- // The compiled Gemini agent has no .ts source in src/ — load via dynamic import.
122
- // new Function trick bypasses the static import() type-check for a dist-only module.
123
- // eslint-disable-next-line @typescript-eslint/no-implied-eval, no-new-func
124
- const dynamicImport = new Function('m', 'return import(m)');
125
- const geminiMod = await dynamicImport('./gaia-agent-gemini.js');
126
- const runGeminiAgent = geminiMod['runGeminiAgent'];
127
- const raw = await runGeminiAgent(question, {
128
- model,
129
- maxTurns,
130
- maxTokensPerTurn,
131
- perTurnTimeoutMs,
132
- apiKey,
133
- });
134
- return {
135
- model,
136
- finalAnswer: raw.finalAnswer,
137
- normalisedAnswer: normaliseAnswer(raw.finalAnswer),
138
- turns: raw.turns,
139
- totalInputTokens: raw.totalInputTokens,
140
- totalOutputTokens: raw.totalOutputTokens + (raw.totalThinkingTokens ?? 0),
141
- wallMs: raw.wallMs,
142
- estimatedCostUsd: raw.estimatedCostUsd,
143
- timedOut: raw.timedOut,
144
- error: raw.error,
145
- };
146
- }
147
- /** Run a model via OpenRouter's OpenAI-compatible chat API. */
148
- async function runOpenRouterModel(question, model, apiKey, maxTurns, maxTokensPerTurn, perTurnTimeoutMs) {
149
- const wallStart = Date.now();
150
- const systemPrompt = [
151
- 'You are a precise question-answering agent.',
152
- 'RULES:',
153
- '1. Answer using your best knowledge and reasoning.',
154
- '2. When you have a final answer, output it on this EXACT format: FINAL_ANSWER: <answer>',
155
- '3. Keep answers concise. For numbers, just the number. For names, just the name.',
156
- '4. Do not include units unless specifically asked.',
157
- '5. MANDATORY: Always end with a FINAL_ANSWER line.',
158
- ].join('\n');
159
- const messages = [
160
- { role: 'system', content: systemPrompt },
161
- { role: 'user', content: question.question },
162
- ];
163
- let finalAnswer = null;
164
- let totalInputTokens = 0;
165
- let totalOutputTokens = 0;
166
- let turns = 0;
167
- let error;
168
- // OpenRouter does not support tool use in this thin adapter — runs non-agentic.
169
- // For GAIA L1 (mostly factual/reasoning), direct completion is sufficient.
170
- for (let turn = 0; turn < Math.min(maxTurns, 3); turn++) {
171
- turns = turn + 1;
172
- const controller = new AbortController();
173
- const timer = setTimeout(() => controller.abort(), perTurnTimeoutMs);
174
- let resp;
175
- try {
176
- resp = await fetch(`${OPENROUTER_API_BASE}/chat/completions`, {
177
- method: 'POST',
178
- headers: {
179
- 'Authorization': `Bearer ${apiKey}`,
180
- 'Content-Type': 'application/json',
181
- 'HTTP-Referer': 'https://github.com/ruvnet/ruflo',
182
- 'X-Title': 'Ruflo GAIA Benchmark',
183
- },
184
- body: JSON.stringify({
185
- model,
186
- messages,
187
- max_tokens: maxTokensPerTurn,
188
- temperature: 0,
189
- }),
190
- signal: controller.signal,
191
- });
192
- }
193
- finally {
194
- clearTimeout(timer);
195
- }
196
- if (!resp.ok) {
197
- const errText = await resp.text().catch(() => '<unreadable>');
198
- error = `OpenRouter API error ${resp.status}: ${errText.slice(0, 300)}`;
199
- break;
200
- }
201
- const data = await resp.json();
202
- totalInputTokens += data.usage?.prompt_tokens ?? 0;
203
- totalOutputTokens += data.usage?.completion_tokens ?? 0;
204
- const content = data.choices[0]?.message.content ?? '';
205
- const match = FINAL_ANSWER_RE.exec(content);
206
- if (match?.[1]) {
207
- finalAnswer = match[1].trim();
208
- break;
209
- }
210
- // No answer yet — append assistant reply and ask to commit
211
- messages.push({ role: 'assistant', content });
212
- messages.push({ role: 'user', content: 'Please commit to a final answer: FINAL_ANSWER: <your answer>' });
213
- }
214
- const wallMs = Date.now() - wallStart;
215
- return {
216
- model,
217
- finalAnswer,
218
- normalisedAnswer: normaliseAnswer(finalAnswer),
219
- turns,
220
- totalInputTokens,
221
- totalOutputTokens,
222
- wallMs,
223
- estimatedCostUsd: estimateCostUsd(model, totalInputTokens, totalOutputTokens),
224
- error,
225
- };
226
- }
227
- /** Run a Claude model via Anthropic API (full agentic harness). */
228
- async function runClaudeModel(question, model, apiKey, opts) {
229
- const raw = await runGaiaAgent(question, { ...opts, model, apiKey });
230
- return {
231
- model,
232
- finalAnswer: raw.finalAnswer,
233
- normalisedAnswer: normaliseAnswer(raw.finalAnswer),
234
- turns: raw.turns,
235
- totalInputTokens: raw.totalInputTokens,
236
- totalOutputTokens: raw.totalOutputTokens,
237
- wallMs: raw.wallMs,
238
- estimatedCostUsd: estimateCostUsd(model, raw.totalInputTokens, raw.totalOutputTokens),
239
- timedOut: raw.timedOut,
240
- error: raw.error,
241
- };
242
- }
243
- // ---------------------------------------------------------------------------
244
- // Tiebreak judge
245
- // ---------------------------------------------------------------------------
246
- async function judgePickBest(question, candidates, judgeModel, apiKey) {
247
- const candidateLines = candidates
248
- .map((c, i) => ` Option ${i + 1} (${c.model}): "${c.answer}"`)
249
- .join('\n');
250
- const prompt = [
251
- `Question: ${question.question}`,
252
- '',
253
- 'The following models gave different answers. Pick the most likely correct answer.',
254
- 'Consider factual accuracy, question phrasing, and common GAIA evaluation rules',
255
- '(exact match, no trailing units unless asked).',
256
- '',
257
- 'Candidates:',
258
- candidateLines,
259
- '',
260
- 'Respond in this EXACT format:',
261
- 'BEST_ANSWER: <the answer text, exactly as it would appear>',
262
- 'RATIONALE: <brief 1-sentence justification>',
263
- ].join('\n');
264
- const controller = new AbortController();
265
- const timer = setTimeout(() => controller.abort(), 30_000);
266
- let resp;
267
- try {
268
- resp = await fetch(ANTHROPIC_API_URL, {
269
- method: 'POST',
270
- headers: {
271
- 'x-api-key': apiKey,
272
- 'anthropic-version': ANTHROPIC_API_VERSION,
273
- 'content-type': 'application/json',
274
- },
275
- body: JSON.stringify({
276
- model: judgeModel,
277
- max_tokens: DEFAULT_TIEBREAK_MAX_TOKENS,
278
- messages: [{ role: 'user', content: prompt }],
279
- }),
280
- signal: controller.signal,
281
- });
282
- }
283
- finally {
284
- clearTimeout(timer);
285
- }
286
- if (!resp.ok) {
287
- const errText = await resp.text().catch(() => '<unreadable>');
288
- throw new Error(`Judge API error ${resp.status}: ${errText.slice(0, 200)}`);
289
- }
290
- const data = await resp.json();
291
- const text = data.content.filter((b) => b.type === 'text').map((b) => b.text).join('\n');
292
- const answerMatch = /BEST_ANSWER:\s*(.+)/i.exec(text);
293
- const rationaleMatch = /RATIONALE:\s*(.+)/i.exec(text);
294
- return {
295
- answer: answerMatch?.[1]?.trim() ?? candidates[0].answer,
296
- rationale: rationaleMatch?.[1]?.trim() ?? '',
297
- inputTokens: data.usage.input_tokens,
298
- outputTokens: data.usage.output_tokens,
299
- };
300
- }
301
- // ---------------------------------------------------------------------------
302
- // Aggregation
303
- // ---------------------------------------------------------------------------
304
- function aggregateByMajority(results) {
305
- const answered = results.filter((r) => r.normalisedAnswer !== '');
306
- if (answered.length === 0)
307
- return { answer: null, method: 'abstain' };
308
- // Count occurrences of each normalised answer
309
- const counts = new Map();
310
- for (const r of answered) {
311
- counts.set(r.normalisedAnswer, (counts.get(r.normalisedAnswer) ?? 0) + 1);
312
- }
313
- // Check for majority (> 50% of total models)
314
- const threshold = results.length / 2;
315
- for (const [norm, count] of counts) {
316
- if (count > threshold) {
317
- // Find the original (un-normalised) answer from any matching result
318
- const match = answered.find((r) => r.normalisedAnswer === norm);
319
- return { answer: match.finalAnswer, method: 'majority' };
320
- }
321
- }
322
- // No majority — needs tiebreak
323
- return { answer: null, method: 'judge-tiebreak' };
324
- }
325
- // ---------------------------------------------------------------------------
326
- // Public: run a single question through the ensemble
327
- // ---------------------------------------------------------------------------
328
- export async function runEnsembleQuestion(question, options = {}) {
329
- const { models = ['claude-sonnet-4-6', 'gemini-2.5-pro'], judgeModel = DEFAULT_JUDGE_MODEL, anthropicApiKey: suppliedAnthropicKey, geminiApiKey: suppliedGeminiKey, openrouterApiKey: suppliedOrKey, maxTurns = 8, maxTokensPerTurn = 2048, perTurnTimeoutMs = 60_000, } = options;
330
- const wallStart = Date.now();
331
- // Resolve API keys once
332
- const anthropicKey = resolveAnthropicApiKey(suppliedAnthropicKey);
333
- const geminiKey = isGeminiModelNeeded(models) ? resolveGeminiApiKey(suppliedGeminiKey) : '';
334
- const orKey = isORModelNeeded(models) ? (resolveOpenRouterApiKey(suppliedOrKey) ?? '') : '';
335
- // Run all models in parallel
336
- const modelPromises = models.map((modelId) => {
337
- if (isOpenRouterModel(modelId)) {
338
- if (!orKey) {
339
- return Promise.resolve({
340
- model: modelId,
341
- finalAnswer: null,
342
- normalisedAnswer: '',
343
- turns: 0,
344
- totalInputTokens: 0,
345
- totalOutputTokens: 0,
346
- wallMs: 0,
347
- estimatedCostUsd: 0,
348
- error: 'OPENROUTER_API_KEY not available or account exhausted',
349
- });
350
- }
351
- return runOpenRouterModel(question, modelId, orKey, maxTurns, maxTokensPerTurn, perTurnTimeoutMs);
352
- }
353
- if (isGeminiModel(modelId)) {
354
- return runGeminiModel(question, modelId, geminiKey, maxTurns, maxTokensPerTurn, perTurnTimeoutMs);
355
- }
356
- // Default: Claude via Anthropic API
357
- return runClaudeModel(question, modelId, anthropicKey, {
358
- maxTurns,
359
- maxTokensPerTurn,
360
- perTurnTimeoutMs,
361
- });
362
- });
363
- const modelResults = await Promise.all(modelPromises);
364
- // Aggregate
365
- const { answer: majorityAnswer, method } = aggregateByMajority(modelResults);
366
- let finalAnswer = majorityAnswer;
367
- let aggregationMethod = method;
368
- let judgeRationale;
369
- let judgeInputTokens = 0;
370
- let judgeOutputTokens = 0;
371
- if (method === 'judge-tiebreak') {
372
- const candidates = modelResults
373
- .filter((r) => r.normalisedAnswer !== '')
374
- .map((r) => ({ model: r.model, answer: r.finalAnswer ?? '' }));
375
- if (candidates.length > 0) {
376
- try {
377
- const judgeResult = await judgePickBest(question, candidates, judgeModel, anthropicKey);
378
- finalAnswer = judgeResult.answer;
379
- judgeRationale = judgeResult.rationale;
380
- judgeInputTokens = judgeResult.inputTokens;
381
- judgeOutputTokens = judgeResult.outputTokens;
382
- }
383
- catch (e) {
384
- // Judge failed — fall back to first non-null answer
385
- finalAnswer = candidates[0].answer;
386
- judgeRationale = `Judge failed: ${e instanceof Error ? e.message : String(e)}`;
387
- }
388
- }
389
- else {
390
- aggregationMethod = 'abstain';
391
- }
392
- }
393
- const totalInputTokens = modelResults.reduce((s, r) => s + r.totalInputTokens, 0) + judgeInputTokens;
394
- const totalOutputTokens = modelResults.reduce((s, r) => s + r.totalOutputTokens, 0) + judgeOutputTokens;
395
- const modelCost = modelResults.reduce((s, r) => s + r.estimatedCostUsd, 0);
396
- const judgeCost = estimateCostUsd(judgeModel, judgeInputTokens, judgeOutputTokens);
397
- return {
398
- questionId: question.task_id,
399
- finalAnswer,
400
- aggregationMethod,
401
- judgeRationale,
402
- models: modelResults,
403
- totalInputTokens,
404
- totalOutputTokens,
405
- estimatedCostUsd: modelCost + judgeCost,
406
- wallMs: Date.now() - wallStart,
407
- };
408
- }
409
- // ---------------------------------------------------------------------------
410
- // Helpers
411
- // ---------------------------------------------------------------------------
412
- function isGeminiModelNeeded(models) {
413
- return models.some(isGeminiModel);
414
- }
415
- function isORModelNeeded(models) {
416
- return models.some(isOpenRouterModel);
417
- }
418
- export async function runEnsemblePilot(questions, options = {}) {
419
- const perQuestion = [];
420
- let correct = 0;
421
- for (const q of questions) {
422
- const result = await runEnsembleQuestion(q, options);
423
- const norm = normaliseAnswer(result.finalAnswer);
424
- const expectedNorm = normaliseAnswer(q.final_answer);
425
- const isCorrect = norm !== '' && norm === expectedNorm;
426
- if (isCorrect)
427
- correct++;
428
- perQuestion.push({
429
- taskId: q.task_id,
430
- question: q.question.slice(0, 80),
431
- expected: q.final_answer ?? '',
432
- got: result.finalAnswer,
433
- correct: isCorrect,
434
- aggregationMethod: result.aggregationMethod,
435
- judgeRationale: result.judgeRationale,
436
- costUsd: result.estimatedCostUsd,
437
- wallMs: result.wallMs,
438
- perModel: result.models.map((m) => ({
439
- model: m.model,
440
- answer: m.finalAnswer,
441
- costUsd: m.estimatedCostUsd,
442
- })),
443
- });
444
- // Progress log to stderr to avoid polluting stdout JSON
445
- process.stderr.write(`[ensemble] ${q.task_id} → ${result.finalAnswer ?? 'null'} ` +
446
- `(${result.aggregationMethod}, ${isCorrect ? 'CORRECT' : 'WRONG'}, $${result.estimatedCostUsd.toFixed(4)})\n`);
447
- }
448
- const totalCostUsd = perQuestion.reduce((s, r) => s + r.costUsd, 0);
449
- const avgCostPerQ = totalCostUsd / Math.max(perQuestion.length, 1);
450
- const meanWallMs = perQuestion.reduce((s, r) => s + r.wallMs, 0) / Math.max(perQuestion.length, 1);
451
- return {
452
- correct,
453
- total: perQuestion.length,
454
- accuracy: correct / Math.max(perQuestion.length, 1),
455
- perQuestion,
456
- totalCostUsd,
457
- projectedCost53Q: avgCostPerQ * 53,
458
- projectedCost300Q: avgCostPerQ * 300,
459
- meanWallMs,
460
- };
461
- }
462
- //# sourceMappingURL=gaia-ensemble.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"gaia-ensemble.js","sourceRoot":"","sources":["../../../src/benchmarks/gaia-ensemble.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAE9C,OAAO,EAAqC,YAAY,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AAC1G,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAElD,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,MAAM,mBAAmB,GAAG,8BAA8B,CAAC;AAC3D,MAAM,iBAAiB,GAAG,uCAAuC,CAAC;AAClE,MAAM,qBAAqB,GAAG,YAAY,CAAC;AAC3C,MAAM,mBAAmB,GAAG,mBAAmB,CAAC;AAChD,MAAM,2BAA2B,GAAG,GAAG,CAAC;AACxC,MAAM,eAAe,GAAG,uBAAuB,CAAC;AAEhD,mFAAmF;AACnF,MAAM,uBAAuB,GAAG,GAAG,CAAC;AACpC,mCAAmC;AACnC,MAAM,mBAAmB,GAAG,SAAS,CAAC;AAEtC,8EAA8E;AAC9E,6CAA6C;AAC7C,8EAA8E;AAE9E,MAAM,aAAa,GAA8D;IAC/E,kBAAkB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE;IACzD,mBAAmB,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE;IACzD,mBAAmB,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE;IACzD,iBAAiB,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE;IACvD,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE;IACvD,kBAAkB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE;IACzD,oDAAoD;IACpD,cAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE;IACrD,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE;IACvD,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE;IACjD,wBAAwB,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE;IACjE,oBAAoB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE;IAC3D,sBAAsB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE;IAC7D,kBAAkB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE;CAC1D,CAAC;AAEF,SAAS,eAAe,CAAC,KAAa,EAAE,WAAmB,EAAE,YAAoB;IAC/E,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IAC7E,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,OAAO,CAAC,SAAS;QAC7C,CAAC,YAAY,GAAG,SAAS,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC;AACzD,CAAC;AAqDD,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,SAAS,mBAAmB,CAAC,QAAiB;IAC5C,IAAI,QAAQ,EAAE,IAAI,EAAE;QAAE,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC9E,IAAI,GAAG,EAAE,IAAI,EAAE;QAAE,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC;IACnC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,QAAQ,CAClB,8EAA8E,EAC9E,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CACvC,CAAC,IAAI,EAAE,CAAC;QACT,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC;IAC9B,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,QAAQ,CAClB,2EAA2E,EAC3E,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CACvC,CAAC,IAAI,EAAE,CAAC;QACT,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC;IAC9B,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;AAC/E,CAAC;AAED,SAAS,uBAAuB,CAAC,QAAiB;IAChD,IAAI,QAAQ,EAAE,IAAI,EAAE;QAAE,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;IAC9C,IAAI,GAAG,EAAE,IAAI,EAAE;QAAE,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC;IACnC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,QAAQ,CAClB,+EAA+E,EAC/E,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CACvC,CAAC,IAAI,EAAE,CAAC;QACT,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC;IAC9B,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,yBAAyB;AACzB,8EAA8E;AAE9E,SAAS,aAAa,CAAC,OAAe;IACpC,OAAO,OAAO,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,iBAAiB,CAAC,OAAe;IACxC,OAAO,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAC;AACnD,CAAC;AAED,yFAAyF;AACzF,KAAK,UAAU,cAAc,CAC3B,QAAsB,EACtB,KAAa,EACb,MAAc,EACd,QAAgB,EAChB,gBAAwB,EACxB,gBAAwB;IAExB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAgB7B,iFAAiF;IACjF,qFAAqF;IACrF,2EAA2E;IAC3E,MAAM,aAAa,GAAG,IAAI,QAAQ,CAAC,GAAG,EAAE,kBAAkB,CAAoD,CAAC;IAC/G,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,wBAAwB,CAAC,CAAC;IAChE,MAAM,cAAc,GAAG,SAAS,CAAC,gBAAgB,CAAkB,CAAC;IAEpE,MAAM,GAAG,GAAG,MAAM,cAAc,CAAC,QAAQ,EAAE;QACzC,KAAK;QACL,QAAQ;QACR,gBAAgB;QAChB,gBAAgB;QAChB,MAAM;KACP,CAAC,CAAC;IAEH,OAAO;QACL,KAAK;QACL,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,gBAAgB,EAAE,eAAe,CAAC,GAAG,CAAC,WAAW,CAAC;QAClD,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;QACtC,iBAAiB,EAAE,GAAG,CAAC,iBAAiB,GAAG,CAAC,GAAG,CAAC,mBAAmB,IAAI,CAAC,CAAC;QACzE,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;QACtC,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,KAAK,EAAE,GAAG,CAAC,KAAK;KACjB,CAAC;AACJ,CAAC;AAED,+DAA+D;AAC/D,KAAK,UAAU,kBAAkB,CAC/B,QAAsB,EACtB,KAAa,EACb,MAAc,EACd,QAAgB,EAChB,gBAAwB,EACxB,gBAAwB;IAExB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,MAAM,YAAY,GAAG;QACnB,6CAA6C;QAC7C,QAAQ;QACR,oDAAoD;QACpD,yFAAyF;QACzF,kFAAkF;QAClF,oDAAoD;QACpD,oDAAoD;KACrD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAGb,MAAM,QAAQ,GAAc;QAC1B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE;QACzC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,QAAQ,EAAE;KAC7C,CAAC;IAEF,IAAI,WAAW,GAAkB,IAAI,CAAC;IACtC,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAyB,CAAC;IAE9B,gFAAgF;IAChF,2EAA2E;IAC3E,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC;QACxD,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC;QACjB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,gBAAgB,CAAC,CAAC;QAErE,IAAI,IAAc,CAAC;QACnB,IAAI,CAAC;YACH,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,mBAAmB,mBAAmB,EAAE;gBAC5D,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,eAAe,EAAE,UAAU,MAAM,EAAE;oBACnC,cAAc,EAAE,kBAAkB;oBAClC,cAAc,EAAE,iCAAiC;oBACjD,SAAS,EAAE,sBAAsB;iBAClC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK;oBACL,QAAQ;oBACR,UAAU,EAAE,gBAAgB;oBAC5B,WAAW,EAAE,CAAC;iBACf,CAAC;gBACF,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CAAC;QACL,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,cAAc,CAAC,CAAC;YAC9D,KAAK,GAAG,wBAAwB,IAAI,CAAC,MAAM,KAAK,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;YACxE,MAAM;QACR,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAG3B,CAAC;QAEF,gBAAgB,IAAI,IAAI,CAAC,KAAK,EAAE,aAAa,IAAI,CAAC,CAAC;QACnD,iBAAiB,IAAI,IAAI,CAAC,KAAK,EAAE,iBAAiB,IAAI,CAAC,CAAC;QAExD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QACvD,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC5C,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACf,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC9B,MAAM;QACR,CAAC;QAED,2DAA2D;QAC3D,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;QAC9C,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,8DAA8D,EAAE,CAAC,CAAC;IAC3G,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IACtC,OAAO;QACL,KAAK;QACL,WAAW;QACX,gBAAgB,EAAE,eAAe,CAAC,WAAW,CAAC;QAC9C,KAAK;QACL,gBAAgB;QAChB,iBAAiB;QACjB,MAAM;QACN,gBAAgB,EAAE,eAAe,CAAC,KAAK,EAAE,gBAAgB,EAAE,iBAAiB,CAAC;QAC7E,KAAK;KACN,CAAC;AACJ,CAAC;AAED,mEAAmE;AACnE,KAAK,UAAU,cAAc,CAC3B,QAAsB,EACtB,KAAa,EACb,MAAc,EACd,IAAsB;IAEtB,MAAM,GAAG,GAAoB,MAAM,YAAY,CAAC,QAAQ,EAAE,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IACtF,OAAO;QACL,KAAK;QACL,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,gBAAgB,EAAE,eAAe,CAAC,GAAG,CAAC,WAAW,CAAC;QAClD,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;QACtC,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;QACxC,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,gBAAgB,EAAE,eAAe,CAAC,KAAK,EAAE,GAAG,CAAC,gBAAgB,EAAE,GAAG,CAAC,iBAAiB,CAAC;QACrF,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,KAAK,EAAE,GAAG,CAAC,KAAK;KACjB,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E,KAAK,UAAU,aAAa,CAC1B,QAAsB,EACtB,UAAoD,EACpD,UAAkB,EAClB,MAAc;IAEd,MAAM,cAAc,GAAG,UAAU;SAC9B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC;SAC9D,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,MAAM,MAAM,GAAG;QACb,aAAa,QAAQ,CAAC,QAAQ,EAAE;QAChC,EAAE;QACF,mFAAmF;QACnF,gFAAgF;QAChF,gDAAgD;QAChD,EAAE;QACF,aAAa;QACb,cAAc;QACd,EAAE;QACF,+BAA+B;QAC/B,4DAA4D;QAC5D,6CAA6C;KAC9C,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEb,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,MAAM,CAAC,CAAC;IAC3D,IAAI,IAAc,CAAC;IACnB,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,KAAK,CAAC,iBAAiB,EAAE;YACpC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,WAAW,EAAE,MAAM;gBACnB,mBAAmB,EAAE,qBAAqB;gBAC1C,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK,EAAE,UAAU;gBACjB,UAAU,EAAE,2BAA2B;gBACvC,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;aAC9C,CAAC;YACF,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,cAAc,CAAC,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,mBAAmB,IAAI,CAAC,MAAM,KAAK,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAG3B,CAAC;IAEF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzF,MAAM,WAAW,GAAG,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtD,MAAM,cAAc,GAAG,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEvD,OAAO;QACL,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM;QACxD,SAAS,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE;QAC5C,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY;QACpC,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;KACvC,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,cAAc;AACd,8EAA8E;AAE9E,SAAS,mBAAmB,CAAC,OAAyB;IAIpD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,KAAK,EAAE,CAAC,CAAC;IAClE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAEtE,8CAA8C;IAC9C,MAAM,MAAM,GAAwB,IAAI,GAAG,EAAE,CAAC;IAC9C,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,gBAAgB,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED,6CAA6C;IAC7C,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;IACrC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QACnC,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACtB,oEAAoE;YACpE,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,KAAK,IAAI,CAAC,CAAC;YAChE,OAAO,EAAE,MAAM,EAAE,KAAM,CAAC,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;QAC5D,CAAC;IACH,CAAC;IAED,+BAA+B;IAC/B,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;AACpD,CAAC;AAED,8EAA8E;AAC9E,qDAAqD;AACrD,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,QAAsB,EACtB,UAA2B,EAAE;IAE7B,MAAM,EACJ,MAAM,GAAG,CAAC,mBAAmB,EAAE,gBAAgB,CAAC,EAChD,UAAU,GAAG,mBAAmB,EAChC,eAAe,EAAE,oBAAoB,EACrC,YAAY,EAAE,iBAAiB,EAC/B,gBAAgB,EAAE,aAAa,EAC/B,QAAQ,GAAG,CAAC,EACZ,gBAAgB,GAAG,IAAI,EACvB,gBAAgB,GAAG,MAAM,GAC1B,GAAG,OAAO,CAAC;IAEZ,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,wBAAwB;IACxB,MAAM,YAAY,GAAG,sBAAsB,CAAC,oBAAoB,CAAC,CAAC;IAClE,MAAM,SAAS,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5F,MAAM,KAAK,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAE5F,6BAA6B;IAC7B,MAAM,aAAa,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QAC3C,IAAI,iBAAiB,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO,OAAO,CAAC,OAAO,CAAiB;oBACrC,KAAK,EAAE,OAAO;oBACd,WAAW,EAAE,IAAI;oBACjB,gBAAgB,EAAE,EAAE;oBACpB,KAAK,EAAE,CAAC;oBACR,gBAAgB,EAAE,CAAC;oBACnB,iBAAiB,EAAE,CAAC;oBACpB,MAAM,EAAE,CAAC;oBACT,gBAAgB,EAAE,CAAC;oBACnB,KAAK,EAAE,uDAAuD;iBAC/D,CAAC,CAAC;YACL,CAAC;YACD,OAAO,kBAAkB,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,gBAAgB,CAAC,CAAC;QACpG,CAAC;QACD,IAAI,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3B,OAAO,cAAc,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,gBAAgB,EAAE,gBAAgB,CAAC,CAAC;QACpG,CAAC;QACD,oCAAoC;QACpC,OAAO,cAAc,CAAC,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE;YACrD,QAAQ;YACR,gBAAgB;YAChB,gBAAgB;SACjB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAEtD,YAAY;IACZ,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAC;IAE7E,IAAI,WAAW,GAAG,cAAc,CAAC;IACjC,IAAI,iBAAiB,GAAsB,MAAM,CAAC;IAClD,IAAI,cAAkC,CAAC;IACvC,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAE1B,IAAI,MAAM,KAAK,gBAAgB,EAAE,CAAC;QAChC,MAAM,UAAU,GAAG,YAAY;aAC5B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,KAAK,EAAE,CAAC;aACxC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;QAEjE,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,MAAM,aAAa,CAAC,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;gBACxF,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC;gBACjC,cAAc,GAAG,WAAW,CAAC,SAAS,CAAC;gBACvC,gBAAgB,GAAG,WAAW,CAAC,WAAW,CAAC;gBAC3C,iBAAiB,GAAG,WAAW,CAAC,YAAY,CAAC;YAC/C,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,oDAAoD;gBACpD,WAAW,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;gBACnC,cAAc,GAAG,iBAAiB,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YACjF,CAAC;QACH,CAAC;aAAM,CAAC;YACN,iBAAiB,GAAG,SAAS,CAAC;QAChC,CAAC;IACH,CAAC;IAED,MAAM,gBAAgB,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,gBAAgB,EAAE,CAAC,CAAC,GAAG,gBAAgB,CAAC;IACrG,MAAM,iBAAiB,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,iBAAiB,EAAE,CAAC,CAAC,GAAG,iBAAiB,CAAC;IACxG,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC;IAC3E,MAAM,SAAS,GAAG,eAAe,CAAC,UAAU,EAAE,gBAAgB,EAAE,iBAAiB,CAAC,CAAC;IAEnF,OAAO;QACL,UAAU,EAAE,QAAQ,CAAC,OAAO;QAC5B,WAAW;QACX,iBAAiB;QACjB,cAAc;QACd,MAAM,EAAE,YAAY;QACpB,gBAAgB;QAChB,iBAAiB;QACjB,gBAAgB,EAAE,SAAS,GAAG,SAAS;QACvC,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;KAC/B,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,SAAS,mBAAmB,CAAC,MAAgB;IAC3C,OAAO,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,eAAe,CAAC,MAAgB;IACvC,OAAO,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;AACxC,CAAC;AA4BD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,SAAyB,EACzB,UAA2B,EAAE;IAE7B,MAAM,WAAW,GAAuC,EAAE,CAAC;IAC3D,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QACrD,MAAM,IAAI,GAAG,eAAe,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACjD,MAAM,YAAY,GAAG,eAAe,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,IAAI,KAAK,EAAE,IAAI,IAAI,KAAK,YAAY,CAAC;QACvD,IAAI,SAAS;YAAE,OAAO,EAAE,CAAC;QAEzB,WAAW,CAAC,IAAI,CAAC;YACf,MAAM,EAAE,CAAC,CAAC,OAAO;YACjB,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;YACjC,QAAQ,EAAE,CAAC,CAAC,YAAY,IAAI,EAAE;YAC9B,GAAG,EAAE,MAAM,CAAC,WAAW;YACvB,OAAO,EAAE,SAAS;YAClB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,OAAO,EAAE,MAAM,CAAC,gBAAgB;YAChC,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAClC,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,MAAM,EAAE,CAAC,CAAC,WAAW;gBACrB,OAAO,EAAE,CAAC,CAAC,gBAAgB;aAC5B,CAAC,CAAC;SACJ,CAAC,CAAC;QAEH,wDAAwD;QACxD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,cAAc,CAAC,CAAC,OAAO,MAAM,MAAM,CAAC,WAAW,IAAI,MAAM,GAAG;YAC5D,IAAI,MAAM,CAAC,iBAAiB,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,MAAM,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAC9G,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IACpE,MAAM,WAAW,GAAG,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACnE,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAEnG,OAAO;QACL,OAAO;QACP,KAAK,EAAE,WAAW,CAAC,MAAM;QACzB,QAAQ,EAAE,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;QACnD,WAAW;QACX,YAAY;QACZ,gBAAgB,EAAE,WAAW,GAAG,EAAE;QAClC,iBAAiB,EAAE,WAAW,GAAG,GAAG;QACpC,UAAU;KACX,CAAC;AACJ,CAAC"}
@@ -1,17 +0,0 @@
1
- /**
2
- * GAIA Phase 2 — Gemini 2.5 Pro Thinking 5-Question Pilot
3
- *
4
- * Runs 5 diverse questions through the Gemini adapter to evaluate:
5
- * - Per-question cost vs $0.12 gate
6
- * - Pass rate vs 3/5 gate
7
- *
8
- * If both gates pass, authorizes full 53Q run.
9
- *
10
- * Usage:
11
- * npx tsx src/benchmarks/gaia-gemini-pilot.ts
12
- * node dist/benchmarks/gaia-gemini-pilot.js
13
- *
14
- * Refs: ADR-133, #2156 Phase 2
15
- */
16
- export {};
17
- //# sourceMappingURL=gaia-gemini-pilot.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"gaia-gemini-pilot.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/gaia-gemini-pilot.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG"}