@claude-flow/cli 3.10.4 → 3.10.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (242) hide show
  1. package/dist/src/benchmarks/capability-tasks.d.ts +126 -0
  2. package/dist/src/benchmarks/capability-tasks.d.ts.map +1 -0
  3. package/dist/src/benchmarks/capability-tasks.js +147 -0
  4. package/dist/src/benchmarks/capability-tasks.js.map +1 -0
  5. package/dist/src/benchmarks/gaia-agent-contrastive.d.ts +53 -0
  6. package/dist/src/benchmarks/gaia-agent-contrastive.d.ts.map +1 -0
  7. package/dist/src/benchmarks/gaia-agent-contrastive.js +177 -0
  8. package/dist/src/benchmarks/gaia-agent-contrastive.js.map +1 -0
  9. package/dist/src/benchmarks/gaia-agent-gemini.d.ts +51 -0
  10. package/dist/src/benchmarks/gaia-agent-gemini.d.ts.map +1 -0
  11. package/dist/src/benchmarks/gaia-agent-gemini.js +352 -0
  12. package/dist/src/benchmarks/gaia-agent-gemini.js.map +1 -0
  13. package/dist/src/benchmarks/gaia-agent-planning.smoke.d.ts +18 -0
  14. package/dist/src/benchmarks/gaia-agent-planning.smoke.d.ts.map +1 -0
  15. package/dist/src/benchmarks/gaia-agent-planning.smoke.js +253 -0
  16. package/dist/src/benchmarks/gaia-agent-planning.smoke.js.map +1 -0
  17. package/dist/src/benchmarks/gaia-agent.d.ts +198 -0
  18. package/dist/src/benchmarks/gaia-agent.d.ts.map +1 -0
  19. package/dist/src/benchmarks/gaia-agent.js +651 -0
  20. package/dist/src/benchmarks/gaia-agent.js.map +1 -0
  21. package/dist/src/benchmarks/gaia-causal-memory.d.ts +133 -0
  22. package/dist/src/benchmarks/gaia-causal-memory.d.ts.map +1 -0
  23. package/dist/src/benchmarks/gaia-causal-memory.js +281 -0
  24. package/dist/src/benchmarks/gaia-causal-memory.js.map +1 -0
  25. package/dist/src/benchmarks/gaia-causal-memory.smoke.d.ts +22 -0
  26. package/dist/src/benchmarks/gaia-causal-memory.smoke.d.ts.map +1 -0
  27. package/dist/src/benchmarks/gaia-causal-memory.smoke.js +300 -0
  28. package/dist/src/benchmarks/gaia-causal-memory.smoke.js.map +1 -0
  29. package/dist/src/benchmarks/gaia-claude-p.d.ts +105 -0
  30. package/dist/src/benchmarks/gaia-claude-p.d.ts.map +1 -0
  31. package/dist/src/benchmarks/gaia-claude-p.js +269 -0
  32. package/dist/src/benchmarks/gaia-claude-p.js.map +1 -0
  33. package/dist/src/benchmarks/gaia-claude-p.smoke.d.ts +20 -0
  34. package/dist/src/benchmarks/gaia-claude-p.smoke.d.ts.map +1 -0
  35. package/dist/src/benchmarks/gaia-claude-p.smoke.js +168 -0
  36. package/dist/src/benchmarks/gaia-claude-p.smoke.js.map +1 -0
  37. package/dist/src/benchmarks/gaia-codeagent-runner.py +556 -0
  38. package/dist/src/benchmarks/gaia-codeagent.d.ts +129 -0
  39. package/dist/src/benchmarks/gaia-codeagent.d.ts.map +1 -0
  40. package/dist/src/benchmarks/gaia-codeagent.js +590 -0
  41. package/dist/src/benchmarks/gaia-codeagent.js.map +1 -0
  42. package/dist/src/benchmarks/gaia-codeagent.smoke.d.ts +33 -0
  43. package/dist/src/benchmarks/gaia-codeagent.smoke.d.ts.map +1 -0
  44. package/dist/src/benchmarks/gaia-codeagent.smoke.js +241 -0
  45. package/dist/src/benchmarks/gaia-codeagent.smoke.js.map +1 -0
  46. package/dist/src/benchmarks/gaia-convergence.d.ts +138 -0
  47. package/dist/src/benchmarks/gaia-convergence.d.ts.map +1 -0
  48. package/dist/src/benchmarks/gaia-convergence.js +260 -0
  49. package/dist/src/benchmarks/gaia-convergence.js.map +1 -0
  50. package/dist/src/benchmarks/gaia-convergence.smoke.d.ts +19 -0
  51. package/dist/src/benchmarks/gaia-convergence.smoke.d.ts.map +1 -0
  52. package/dist/src/benchmarks/gaia-convergence.smoke.js +246 -0
  53. package/dist/src/benchmarks/gaia-convergence.smoke.js.map +1 -0
  54. package/dist/src/benchmarks/gaia-critic.d.ts +123 -0
  55. package/dist/src/benchmarks/gaia-critic.d.ts.map +1 -0
  56. package/dist/src/benchmarks/gaia-critic.js +312 -0
  57. package/dist/src/benchmarks/gaia-critic.js.map +1 -0
  58. package/dist/src/benchmarks/gaia-critic.smoke.d.ts +21 -0
  59. package/dist/src/benchmarks/gaia-critic.smoke.d.ts.map +1 -0
  60. package/dist/src/benchmarks/gaia-critic.smoke.js +327 -0
  61. package/dist/src/benchmarks/gaia-critic.smoke.js.map +1 -0
  62. package/dist/src/benchmarks/gaia-dag-pilot.d.ts +22 -0
  63. package/dist/src/benchmarks/gaia-dag-pilot.d.ts.map +1 -0
  64. package/dist/src/benchmarks/gaia-dag-pilot.js +158 -0
  65. package/dist/src/benchmarks/gaia-dag-pilot.js.map +1 -0
  66. package/dist/src/benchmarks/gaia-dag.d.ts +110 -0
  67. package/dist/src/benchmarks/gaia-dag.d.ts.map +1 -0
  68. package/dist/src/benchmarks/gaia-dag.js +658 -0
  69. package/dist/src/benchmarks/gaia-dag.js.map +1 -0
  70. package/dist/src/benchmarks/gaia-dag.smoke.d.ts +21 -0
  71. package/dist/src/benchmarks/gaia-dag.smoke.d.ts.map +1 -0
  72. package/dist/src/benchmarks/gaia-dag.smoke.js +290 -0
  73. package/dist/src/benchmarks/gaia-dag.smoke.js.map +1 -0
  74. package/dist/src/benchmarks/gaia-decomposer.d.ts +125 -0
  75. package/dist/src/benchmarks/gaia-decomposer.d.ts.map +1 -0
  76. package/dist/src/benchmarks/gaia-decomposer.js +350 -0
  77. package/dist/src/benchmarks/gaia-decomposer.js.map +1 -0
  78. package/dist/src/benchmarks/gaia-decomposer.smoke.d.ts +21 -0
  79. package/dist/src/benchmarks/gaia-decomposer.smoke.d.ts.map +1 -0
  80. package/dist/src/benchmarks/gaia-decomposer.smoke.js +228 -0
  81. package/dist/src/benchmarks/gaia-decomposer.smoke.js.map +1 -0
  82. package/dist/src/benchmarks/gaia-e2e-smoke.d.ts +27 -0
  83. package/dist/src/benchmarks/gaia-e2e-smoke.d.ts.map +1 -0
  84. package/dist/src/benchmarks/gaia-e2e-smoke.js +136 -0
  85. package/dist/src/benchmarks/gaia-e2e-smoke.js.map +1 -0
  86. package/dist/src/benchmarks/gaia-ensemble-pilot.d.ts +17 -0
  87. package/dist/src/benchmarks/gaia-ensemble-pilot.d.ts.map +1 -0
  88. package/dist/src/benchmarks/gaia-ensemble-pilot.js +106 -0
  89. package/dist/src/benchmarks/gaia-ensemble-pilot.js.map +1 -0
  90. package/dist/src/benchmarks/gaia-ensemble.d.ts +103 -0
  91. package/dist/src/benchmarks/gaia-ensemble.d.ts.map +1 -0
  92. package/dist/src/benchmarks/gaia-ensemble.js +462 -0
  93. package/dist/src/benchmarks/gaia-ensemble.js.map +1 -0
  94. package/dist/src/benchmarks/gaia-extract.smoke.d.ts +45 -0
  95. package/dist/src/benchmarks/gaia-extract.smoke.d.ts.map +1 -0
  96. package/dist/src/benchmarks/gaia-extract.smoke.js +242 -0
  97. package/dist/src/benchmarks/gaia-extract.smoke.js.map +1 -0
  98. package/dist/src/benchmarks/gaia-gemini-pilot.d.ts +17 -0
  99. package/dist/src/benchmarks/gaia-gemini-pilot.d.ts.map +1 -0
  100. package/dist/src/benchmarks/gaia-gemini-pilot.js +168 -0
  101. package/dist/src/benchmarks/gaia-gemini-pilot.js.map +1 -0
  102. package/dist/src/benchmarks/gaia-hardness/features.d.ts +46 -0
  103. package/dist/src/benchmarks/gaia-hardness/features.d.ts.map +1 -0
  104. package/dist/src/benchmarks/gaia-hardness/features.js +170 -0
  105. package/dist/src/benchmarks/gaia-hardness/features.js.map +1 -0
  106. package/dist/src/benchmarks/gaia-hardness/predictor.d.ts +105 -0
  107. package/dist/src/benchmarks/gaia-hardness/predictor.d.ts.map +1 -0
  108. package/dist/src/benchmarks/gaia-hardness/predictor.js +260 -0
  109. package/dist/src/benchmarks/gaia-hardness/predictor.js.map +1 -0
  110. package/dist/src/benchmarks/gaia-hardness/predictor.smoke.d.ts +20 -0
  111. package/dist/src/benchmarks/gaia-hardness/predictor.smoke.d.ts.map +1 -0
  112. package/dist/src/benchmarks/gaia-hardness/predictor.smoke.js +235 -0
  113. package/dist/src/benchmarks/gaia-hardness/predictor.smoke.js.map +1 -0
  114. package/dist/src/benchmarks/gaia-hardness/train-data-loader.d.ts +51 -0
  115. package/dist/src/benchmarks/gaia-hardness/train-data-loader.d.ts.map +1 -0
  116. package/dist/src/benchmarks/gaia-hardness/train-data-loader.js +179 -0
  117. package/dist/src/benchmarks/gaia-hardness/train-data-loader.js.map +1 -0
  118. package/dist/src/benchmarks/gaia-judge.d.ts +88 -0
  119. package/dist/src/benchmarks/gaia-judge.d.ts.map +1 -0
  120. package/dist/src/benchmarks/gaia-judge.js +437 -0
  121. package/dist/src/benchmarks/gaia-judge.js.map +1 -0
  122. package/dist/src/benchmarks/gaia-loader.d.ts +87 -0
  123. package/dist/src/benchmarks/gaia-loader.d.ts.map +1 -0
  124. package/dist/src/benchmarks/gaia-loader.js +326 -0
  125. package/dist/src/benchmarks/gaia-loader.js.map +1 -0
  126. package/dist/src/benchmarks/gaia-mode-router.d.ts +67 -0
  127. package/dist/src/benchmarks/gaia-mode-router.d.ts.map +1 -0
  128. package/dist/src/benchmarks/gaia-mode-router.js +126 -0
  129. package/dist/src/benchmarks/gaia-mode-router.js.map +1 -0
  130. package/dist/src/benchmarks/gaia-mode-router.smoke.d.ts +15 -0
  131. package/dist/src/benchmarks/gaia-mode-router.smoke.d.ts.map +1 -0
  132. package/dist/src/benchmarks/gaia-mode-router.smoke.js +131 -0
  133. package/dist/src/benchmarks/gaia-mode-router.smoke.js.map +1 -0
  134. package/dist/src/benchmarks/gaia-tools/file_read.d.ts +35 -0
  135. package/dist/src/benchmarks/gaia-tools/file_read.d.ts.map +1 -0
  136. package/dist/src/benchmarks/gaia-tools/file_read.js +403 -0
  137. package/dist/src/benchmarks/gaia-tools/file_read.js.map +1 -0
  138. package/dist/src/benchmarks/gaia-tools/grounded_query.d.ts +126 -0
  139. package/dist/src/benchmarks/gaia-tools/grounded_query.d.ts.map +1 -0
  140. package/dist/src/benchmarks/gaia-tools/grounded_query.js +225 -0
  141. package/dist/src/benchmarks/gaia-tools/grounded_query.js.map +1 -0
  142. package/dist/src/benchmarks/gaia-tools/grounded_query.smoke.d.ts +24 -0
  143. package/dist/src/benchmarks/gaia-tools/grounded_query.smoke.d.ts.map +1 -0
  144. package/dist/src/benchmarks/gaia-tools/grounded_query.smoke.js +276 -0
  145. package/dist/src/benchmarks/gaia-tools/grounded_query.smoke.js.map +1 -0
  146. package/dist/src/benchmarks/gaia-tools/image_describe.d.ts +45 -0
  147. package/dist/src/benchmarks/gaia-tools/image_describe.d.ts.map +1 -0
  148. package/dist/src/benchmarks/gaia-tools/image_describe.js +222 -0
  149. package/dist/src/benchmarks/gaia-tools/image_describe.js.map +1 -0
  150. package/dist/src/benchmarks/gaia-tools/image_describe.smoke.d.ts +24 -0
  151. package/dist/src/benchmarks/gaia-tools/image_describe.smoke.d.ts.map +1 -0
  152. package/dist/src/benchmarks/gaia-tools/image_describe.smoke.js +169 -0
  153. package/dist/src/benchmarks/gaia-tools/image_describe.smoke.js.map +1 -0
  154. package/dist/src/benchmarks/gaia-tools/index.d.ts +32 -0
  155. package/dist/src/benchmarks/gaia-tools/index.d.ts.map +1 -0
  156. package/dist/src/benchmarks/gaia-tools/index.js +36 -0
  157. package/dist/src/benchmarks/gaia-tools/index.js.map +1 -0
  158. package/dist/src/benchmarks/gaia-tools/pdf_read.d.ts +27 -0
  159. package/dist/src/benchmarks/gaia-tools/pdf_read.d.ts.map +1 -0
  160. package/dist/src/benchmarks/gaia-tools/pdf_read.js +226 -0
  161. package/dist/src/benchmarks/gaia-tools/pdf_read.js.map +1 -0
  162. package/dist/src/benchmarks/gaia-tools/python_exec.d.ts +31 -0
  163. package/dist/src/benchmarks/gaia-tools/python_exec.d.ts.map +1 -0
  164. package/dist/src/benchmarks/gaia-tools/python_exec.js +94 -0
  165. package/dist/src/benchmarks/gaia-tools/python_exec.js.map +1 -0
  166. package/dist/src/benchmarks/gaia-tools/python_exec.smoke.d.ts +13 -0
  167. package/dist/src/benchmarks/gaia-tools/python_exec.smoke.d.ts.map +1 -0
  168. package/dist/src/benchmarks/gaia-tools/python_exec.smoke.js +98 -0
  169. package/dist/src/benchmarks/gaia-tools/python_exec.smoke.js.map +1 -0
  170. package/dist/src/benchmarks/gaia-tools/types.d.ts +62 -0
  171. package/dist/src/benchmarks/gaia-tools/types.d.ts.map +1 -0
  172. package/dist/src/benchmarks/gaia-tools/types.js +12 -0
  173. package/dist/src/benchmarks/gaia-tools/types.js.map +1 -0
  174. package/dist/src/benchmarks/gaia-tools/visit_webpage.d.ts +52 -0
  175. package/dist/src/benchmarks/gaia-tools/visit_webpage.d.ts.map +1 -0
  176. package/dist/src/benchmarks/gaia-tools/visit_webpage.js +304 -0
  177. package/dist/src/benchmarks/gaia-tools/visit_webpage.js.map +1 -0
  178. package/dist/src/benchmarks/gaia-tools/web_browse.d.ts +61 -0
  179. package/dist/src/benchmarks/gaia-tools/web_browse.d.ts.map +1 -0
  180. package/dist/src/benchmarks/gaia-tools/web_browse.js +217 -0
  181. package/dist/src/benchmarks/gaia-tools/web_browse.js.map +1 -0
  182. package/dist/src/benchmarks/gaia-tools/web_browse.smoke.d.ts +26 -0
  183. package/dist/src/benchmarks/gaia-tools/web_browse.smoke.d.ts.map +1 -0
  184. package/dist/src/benchmarks/gaia-tools/web_browse.smoke.js +152 -0
  185. package/dist/src/benchmarks/gaia-tools/web_browse.smoke.js.map +1 -0
  186. package/dist/src/benchmarks/gaia-tools/web_search.d.ts +30 -0
  187. package/dist/src/benchmarks/gaia-tools/web_search.d.ts.map +1 -0
  188. package/dist/src/benchmarks/gaia-tools/web_search.js +210 -0
  189. package/dist/src/benchmarks/gaia-tools/web_search.js.map +1 -0
  190. package/dist/src/benchmarks/gaia-tools/web_search.smoke.d.ts +24 -0
  191. package/dist/src/benchmarks/gaia-tools/web_search.smoke.d.ts.map +1 -0
  192. package/dist/src/benchmarks/gaia-tools/web_search.smoke.js +311 -0
  193. package/dist/src/benchmarks/gaia-tools/web_search.smoke.js.map +1 -0
  194. package/dist/src/benchmarks/gaia-visit-webpage.smoke.d.ts +25 -0
  195. package/dist/src/benchmarks/gaia-visit-webpage.smoke.d.ts.map +1 -0
  196. package/dist/src/benchmarks/gaia-visit-webpage.smoke.js +304 -0
  197. package/dist/src/benchmarks/gaia-visit-webpage.smoke.js.map +1 -0
  198. package/dist/src/benchmarks/gaia-voting.d.ts +88 -0
  199. package/dist/src/benchmarks/gaia-voting.d.ts.map +1 -0
  200. package/dist/src/benchmarks/gaia-voting.js +297 -0
  201. package/dist/src/benchmarks/gaia-voting.js.map +1 -0
  202. package/dist/src/benchmarks/gaia-voting.smoke.d.ts +20 -0
  203. package/dist/src/benchmarks/gaia-voting.smoke.d.ts.map +1 -0
  204. package/dist/src/benchmarks/gaia-voting.smoke.js +332 -0
  205. package/dist/src/benchmarks/gaia-voting.smoke.js.map +1 -0
  206. package/dist/src/commands/gaia-bench.d.ts +40 -0
  207. package/dist/src/commands/gaia-bench.d.ts.map +1 -0
  208. package/dist/src/commands/gaia-bench.js +597 -0
  209. package/dist/src/commands/gaia-bench.js.map +1 -0
  210. package/dist/src/commands/index.d.ts.map +1 -1
  211. package/dist/src/commands/index.js +2 -0
  212. package/dist/src/commands/index.js.map +1 -1
  213. package/dist/src/commands/init.d.ts.map +1 -1
  214. package/dist/src/commands/init.js +41 -4
  215. package/dist/src/commands/init.js.map +1 -1
  216. package/dist/src/commands/performance-capability.d.ts +30 -0
  217. package/dist/src/commands/performance-capability.d.ts.map +1 -0
  218. package/dist/src/commands/performance-capability.js +349 -0
  219. package/dist/src/commands/performance-capability.js.map +1 -0
  220. package/dist/src/commands/route.d.ts.map +1 -1
  221. package/dist/src/commands/route.js +9 -0
  222. package/dist/src/commands/route.js.map +1 -1
  223. package/dist/src/init/executor.d.ts.map +1 -1
  224. package/dist/src/init/executor.js +25 -6
  225. package/dist/src/init/executor.js.map +1 -1
  226. package/dist/src/init/mcp-generator.d.ts.map +1 -1
  227. package/dist/src/init/mcp-generator.js +9 -4
  228. package/dist/src/init/mcp-generator.js.map +1 -1
  229. package/dist/src/init/statusline-generator.d.ts.map +1 -1
  230. package/dist/src/init/statusline-generator.js +25 -2
  231. package/dist/src/init/statusline-generator.js.map +1 -1
  232. package/dist/src/mcp-tools/agentdb-tools.d.ts.map +1 -1
  233. package/dist/src/mcp-tools/agentdb-tools.js +20 -3
  234. package/dist/src/mcp-tools/agentdb-tools.js.map +1 -1
  235. package/dist/src/mcp-tools/system-tools.d.ts.map +1 -1
  236. package/dist/src/mcp-tools/system-tools.js +12 -1
  237. package/dist/src/mcp-tools/system-tools.js.map +1 -1
  238. package/dist/src/memory/memory-bridge.d.ts.map +1 -1
  239. package/dist/src/memory/memory-bridge.js +34 -0
  240. package/dist/src/memory/memory-bridge.js.map +1 -1
  241. package/dist/tsconfig.tsbuildinfo +1 -1
  242. package/package.json +1 -1
@@ -0,0 +1,326 @@
1
+ /**
2
+ * GAIA Dataset Loader — ADR-133-PR1
3
+ *
4
+ * Authenticates to Hugging Face, downloads the `gaia-benchmark/GAIA`
5
+ * validation split, caches it under ~/.cache/ruflo/gaia/, and exposes
6
+ * a typed `loadGaia()` API consumed by the capability-gaia subcommand.
7
+ *
8
+ * Token resolution order (mirrors performance-capability.ts ANTHROPIC_API_KEY pattern):
9
+ * 1. $HF_TOKEN env var
10
+ * 2. gcloud secrets versions access latest --secret=huggingface-token
11
+ * 3. Fail with a clear error message
12
+ *
13
+ * This file is deliberately a skeleton / PR-1 checkpoint. The full
14
+ * dataset download is gated behind a real HF_TOKEN; a 5-question smoke
15
+ * fixture is provided for offline / CI-without-HF testing.
16
+ *
17
+ * Refs: ADR-133, #2156
18
+ */
19
+ import { execSync } from 'node:child_process';
20
+ import * as fs from 'node:fs';
21
+ import * as path from 'node:path';
22
+ import * as os from 'node:os';
23
+ import * as https from 'node:https';
24
+ import * as url from 'node:url';
25
+ // ---------------------------------------------------------------------------
26
+ // HF Token resolution
27
+ // ---------------------------------------------------------------------------
28
+ /**
29
+ * Resolve a Hugging Face API token using two fallbacks:
30
+ * 1. $HF_TOKEN env var
31
+ * 2. `gcloud secrets versions access latest --secret=huggingface-token`
32
+ *
33
+ * Throws with a clear error if neither is available.
34
+ */
35
+ export function resolveHfToken() {
36
+ const envToken = process.env.HF_TOKEN;
37
+ if (envToken && envToken.trim())
38
+ return envToken.trim();
39
+ try {
40
+ const out = execSync('gcloud secrets versions access latest --secret=huggingface-token 2>/dev/null', { encoding: 'utf-8', timeout: 10_000 }).trim();
41
+ if (out)
42
+ return out;
43
+ }
44
+ catch {
45
+ /* fall through */
46
+ }
47
+ throw new Error('HF_TOKEN not found. Set the env var or store it in GCP Secret Manager under the name ' +
48
+ '"huggingface-token" (e.g. `echo -n "$TOKEN" | gcloud secrets versions add huggingface-token --data-file=-`).');
49
+ }
50
+ // ---------------------------------------------------------------------------
51
+ // Cache helpers
52
+ // ---------------------------------------------------------------------------
53
+ function defaultCacheDir() {
54
+ return path.join(os.homedir(), '.cache', 'ruflo', 'gaia');
55
+ }
56
+ /**
57
+ * Export the default cache directory path (no side effects).
58
+ */
59
+ export function getDefaultCacheDir() {
60
+ return defaultCacheDir();
61
+ }
62
+ function ensureDir(dir) {
63
+ fs.mkdirSync(dir, { recursive: true });
64
+ }
65
+ function revisionCacheKey(revision, level) {
66
+ return `level${level}-${revision}.json`;
67
+ }
68
+ // ---------------------------------------------------------------------------
69
+ // 5-question smoke fixture (no HF token required)
70
+ // ---------------------------------------------------------------------------
71
+ /**
72
+ * Minimal offline fixture for smoke tests and CI environments without HF_TOKEN.
73
+ * Answers are verified by hand against publicly known facts.
74
+ * All questions are Level 1 (no file attachments, no multi-hop tool use required).
75
+ *
76
+ * IMPORTANT: Verify every answer key with `node -e 'console.log(…)'` before
77
+ * adding entries to this list — three answer-key bugs were caught in session #2156.
78
+ */
79
+ export const SMOKE_FIXTURE = [
80
+ {
81
+ task_id: 'smoke-001',
82
+ level: 1,
83
+ question: 'What is the capital of France?',
84
+ final_answer: 'Paris',
85
+ file_name: null,
86
+ file_path: null,
87
+ },
88
+ {
89
+ task_id: 'smoke-002',
90
+ level: 1,
91
+ question: 'How many sides does a hexagon have?',
92
+ final_answer: '6',
93
+ file_name: null,
94
+ file_path: null,
95
+ },
96
+ {
97
+ task_id: 'smoke-003',
98
+ level: 1,
99
+ question: 'What is 15 multiplied by 4?',
100
+ final_answer: '60',
101
+ file_name: null,
102
+ file_path: null,
103
+ },
104
+ {
105
+ task_id: 'smoke-004',
106
+ level: 1,
107
+ question: 'In what year did the Berlin Wall fall?',
108
+ final_answer: '1989',
109
+ file_name: null,
110
+ file_path: null,
111
+ },
112
+ {
113
+ task_id: 'smoke-005',
114
+ level: 1,
115
+ question: 'What chemical symbol represents gold on the periodic table?',
116
+ final_answer: 'Au',
117
+ file_name: null,
118
+ file_path: null,
119
+ },
120
+ ];
121
+ // ---------------------------------------------------------------------------
122
+ // HF dataset download (requires HF_TOKEN)
123
+ // ---------------------------------------------------------------------------
124
+ const HF_DATASET_REPO = 'gaia-benchmark/GAIA';
125
+ const HF_API_BASE = 'https://huggingface.co';
126
+ const HF_DATASETS_API = 'https://datasets-server.huggingface.co';
127
+ /**
128
+ * Fetch JSON from a URL with an Authorization header.
129
+ * Returns parsed JSON or throws.
130
+ */
131
+ async function fetchJson(url, token) {
132
+ return new Promise((resolve, reject) => {
133
+ const req = https.get(url, {
134
+ headers: {
135
+ Authorization: `Bearer ${token}`,
136
+ 'User-Agent': 'ruflo-gaia-loader/1.0',
137
+ },
138
+ }, (res) => {
139
+ if (res.statusCode === 401 || res.statusCode === 403) {
140
+ reject(new Error(`HF auth error ${res.statusCode} for ${url}. Check HF_TOKEN permissions (need read access to gaia-benchmark/GAIA).`));
141
+ res.resume();
142
+ return;
143
+ }
144
+ if (res.statusCode !== 200) {
145
+ reject(new Error(`HTTP ${res.statusCode} from ${url}`));
146
+ res.resume();
147
+ return;
148
+ }
149
+ const chunks = [];
150
+ res.on('data', (c) => chunks.push(c));
151
+ res.on('end', () => {
152
+ try {
153
+ resolve(JSON.parse(Buffer.concat(chunks).toString('utf-8')));
154
+ }
155
+ catch (e) {
156
+ reject(new Error(`JSON parse failed for ${url}: ${String(e)}`));
157
+ }
158
+ });
159
+ res.on('error', reject);
160
+ });
161
+ req.on('error', reject);
162
+ req.setTimeout(30_000, () => {
163
+ req.destroy(new Error(`Timeout fetching ${url}`));
164
+ });
165
+ });
166
+ }
167
+ /**
168
+ * Download the GAIA validation split for a given level from Hugging Face.
169
+ *
170
+ * Uses the HF Datasets Server API (paginated parquet/JSON rows endpoint).
171
+ * Caches the result locally so subsequent runs are instant.
172
+ *
173
+ * Uses per-level HF configs (2023_level1, 2023_level2, 2023_level3) so
174
+ * all questions for a given level fit within the 100-row API limit.
175
+ */
176
+ async function downloadGaiaLevel(level, token, cacheDir) {
177
+ ensureDir(cacheDir);
178
+ // Query a stable revision identifier (the dataset card commit SHA).
179
+ // For now we use 'main' as the revision key for the cache filename.
180
+ const revision = 'main';
181
+ const cacheFile = path.join(cacheDir, revisionCacheKey(revision, level));
182
+ if (fs.existsSync(cacheFile)) {
183
+ const cached = JSON.parse(fs.readFileSync(cacheFile, 'utf-8'));
184
+ return cached;
185
+ }
186
+ // HF Datasets Server rows endpoint for the per-level 2023 config.
187
+ // Using "2023_level{N}" config instead of "2023_all" avoids the pagination
188
+ // problem: "2023_all" has 165 rows but the API caps at 100 per request,
189
+ // which silently drops 23 of 53 Level-1 questions. The per-level configs
190
+ // each have ≤100 rows (L1=53, L2=86, L3=26) and fit in a single request.
191
+ const levelConfig = `2023_level${level}`;
192
+ const url = `${HF_DATASETS_API}/rows?dataset=${encodeURIComponent(HF_DATASET_REPO)}` +
193
+ `&config=${levelConfig}&split=validation&offset=0&length=100`;
194
+ const data = await fetchJson(url, token);
195
+ if (!data.rows || !Array.isArray(data.rows)) {
196
+ throw new Error(`Unexpected HF response shape — missing .rows array. Got: ${JSON.stringify(data).slice(0, 200)}`);
197
+ }
198
+ const questions = data.rows
199
+ .map((r) => r.row)
200
+ .filter((row) => Number(row['Level']) === level)
201
+ .map((row) => ({
202
+ task_id: String(row['task_id'] ?? ''),
203
+ level,
204
+ question: String(row['Question'] ?? ''),
205
+ final_answer: String(row['Final answer'] ?? '').trim(),
206
+ file_name: row['file_name'] ? String(row['file_name']) : null,
207
+ file_path: null, // attachment resolution is a PR-2 concern
208
+ annotator_metadata: row['Annotator Metadata'],
209
+ }))
210
+ .filter((q) => q.task_id && q.question && q.final_answer);
211
+ fs.writeFileSync(cacheFile, JSON.stringify(questions, null, 2), 'utf-8');
212
+ return questions;
213
+ }
214
+ // ---------------------------------------------------------------------------
215
+ // Attachment download (iter-53b)
216
+ // ---------------------------------------------------------------------------
217
+ /**
218
+ * Base URL for GAIA validation attachment files.
219
+ * Individual files are at: HF_FILE_BASE/<task_id>/<file_name>
220
+ *
221
+ * HF now serves files through the Xet storage layer; requests will redirect
222
+ * (301/302/303) to a Xet download URL that requires NO auth header.
223
+ * We only send the Authorization header to requests targeting huggingface.co.
224
+ */
225
+ const HF_FILE_BASE = 'https://huggingface.co/datasets/gaia-benchmark/GAIA/resolve/main/2023/validation';
226
+ /**
227
+ * Download a single GAIA attachment file to the cache directory.
228
+ * Follows HTTP redirects, sending auth only to huggingface.co domains.
229
+ * Returns the local file path on success, or null on any error.
230
+ */
231
+ async function downloadAttachment(taskId, fileName, token, cacheDir) {
232
+ const destPath = path.join(cacheDir, fileName);
233
+ if (fs.existsSync(destPath))
234
+ return destPath; // already cached
235
+ const fileUrl = `${HF_FILE_BASE}/${taskId}/${encodeURIComponent(fileName)}`;
236
+ ensureDir(cacheDir);
237
+ return new Promise((resolve) => {
238
+ function doGet(requestUrl, depth) {
239
+ if (depth > 5) {
240
+ resolve(null);
241
+ return;
242
+ }
243
+ const parsed = new url.URL(requestUrl);
244
+ const headers = {
245
+ 'User-Agent': 'ruflo-gaia-loader/1.0',
246
+ };
247
+ if (parsed.hostname.includes('huggingface.co')) {
248
+ headers['Authorization'] = `Bearer ${token}`;
249
+ }
250
+ const req = https.get(requestUrl, { headers }, (res) => {
251
+ if (res.statusCode && [301, 302, 303, 307, 308].includes(res.statusCode)) {
252
+ const loc = res.headers['location'];
253
+ res.resume();
254
+ if (loc)
255
+ doGet(loc, depth + 1);
256
+ else
257
+ resolve(null);
258
+ return;
259
+ }
260
+ if (res.statusCode !== 200) {
261
+ res.resume();
262
+ resolve(null);
263
+ return;
264
+ }
265
+ const chunks = [];
266
+ res.on('data', (c) => chunks.push(c));
267
+ res.on('end', () => {
268
+ try {
269
+ fs.writeFileSync(destPath, Buffer.concat(chunks));
270
+ resolve(destPath);
271
+ }
272
+ catch {
273
+ resolve(null);
274
+ }
275
+ });
276
+ res.on('error', () => resolve(null));
277
+ });
278
+ req.on('error', () => resolve(null));
279
+ req.setTimeout(60_000, () => { req.destroy(); resolve(null); });
280
+ }
281
+ doGet(fileUrl, 0);
282
+ });
283
+ }
284
+ /**
285
+ * Download all attachment files referenced by the questions list.
286
+ * Mutates each question's `file_path` field in place.
287
+ * Skips questions without a file_name.
288
+ */
289
+ export async function resolveAttachments(questions, token, cacheDir) {
290
+ const withFiles = questions.filter((q) => q.file_name);
291
+ await Promise.all(withFiles.map(async (q) => {
292
+ const localPath = await downloadAttachment(q.task_id, q.file_name, token, cacheDir);
293
+ q.file_path = localPath;
294
+ }));
295
+ }
296
+ // ---------------------------------------------------------------------------
297
+ // Public API
298
+ // ---------------------------------------------------------------------------
299
+ /**
300
+ * Load GAIA validation questions.
301
+ *
302
+ * - With `smokeOnly: true` (or when HF_TOKEN is unavailable): returns the 5-question
303
+ * smoke fixture — no network call, no token required.
304
+ * - Otherwise: authenticates to HF, downloads level N questions, caches locally.
305
+ *
306
+ * @throws if HF_TOKEN is missing and smokeOnly is false
307
+ */
308
+ export async function loadGaia(options = {}) {
309
+ const { level = 1, limit, smokeOnly = false, cacheDir = defaultCacheDir() } = options;
310
+ if (smokeOnly) {
311
+ const filtered = SMOKE_FIXTURE.filter((q) => q.level === level);
312
+ return limit !== undefined ? filtered.slice(0, limit) : filtered;
313
+ }
314
+ const token = resolveHfToken();
315
+ const questions = await downloadGaiaLevel(level, token, cacheDir);
316
+ // Resolve attachment files in parallel (iter-53b: populates file_path on each question)
317
+ await resolveAttachments(questions, token, cacheDir);
318
+ return limit !== undefined ? questions.slice(0, limit) : questions;
319
+ }
320
+ /**
321
+ * Returns the cache directory path (does not create it).
322
+ */
323
+ export function getGaiaCacheDir(override) {
324
+ return override ?? defaultCacheDir();
325
+ }
326
+ //# sourceMappingURL=gaia-loader.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"gaia-loader.js","sourceRoot":"","sources":["../../../src/benchmarks/gaia-loader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,KAAK,MAAM,YAAY,CAAC;AACpC,OAAO,KAAK,GAAG,MAAM,UAAU,CAAC;AAoChC,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E;;;;;;GAMG;AACH,MAAM,UAAU,cAAc;IAC5B,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IACtC,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,EAAE;QAAE,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;IAExD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,QAAQ,CAClB,8EAA8E,EAC9E,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CACvC,CAAC,IAAI,EAAE,CAAC;QACT,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,kBAAkB;IACpB,CAAC;IAED,MAAM,IAAI,KAAK,CACb,uFAAuF;QACvF,8GAA8G,CAC/G,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,gBAAgB;AAChB,8EAA8E;AAE9E,SAAS,eAAe;IACtB,OAAO,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;AAC5D,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB;IAChC,OAAO,eAAe,EAAE,CAAC;AAC3B,CAAC;AAED,SAAS,SAAS,CAAC,GAAW;IAC5B,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,gBAAgB,CAAC,QAAgB,EAAE,KAAgB;IAC1D,OAAO,QAAQ,KAAK,IAAI,QAAQ,OAAO,CAAC;AAC1C,CAAC;AAED,8EAA8E;AAC9E,kDAAkD;AAClD,8EAA8E;AAE9E;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,aAAa,GAAmB;IAC3C;QACE,OAAO,EAAE,WAAW;QACpB,KAAK,EAAE,CAAC;QACR,QAAQ,EAAE,gCAAgC;QAC1C,YAAY,EAAE,OAAO;QACrB,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,IAAI;KAChB;IACD;QACE,OAAO,EAAE,WAAW;QACpB,KAAK,EAAE,CAAC;QACR,QAAQ,EAAE,qCAAqC;QAC/C,YAAY,EAAE,GAAG;QACjB,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,IAAI;KAChB;IACD;QACE,OAAO,EAAE,WAAW;QACpB,KAAK,EAAE,CAAC;QACR,QAAQ,EAAE,6BAA6B;QACvC,YAAY,EAAE,IAAI;QAClB,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,IAAI;KAChB;IACD;QACE,OAAO,EAAE,WAAW;QACpB,KAAK,EAAE,CAAC;QACR,QAAQ,EAAE,wCAAwC;QAClD,YAAY,EAAE,MAAM;QACpB,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,IAAI;KAChB;IACD;QACE,OAAO,EAAE,WAAW;QACpB,KAAK,EAAE,CAAC;QACR,QAAQ,EAAE,6DAA6D;QACvE,YAAY,EAAE,IAAI;QAClB,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,IAAI;KAChB;CACF,CAAC;AAEF,8EAA8E;AAC9E,0CAA0C;AAC1C,8EAA8E;AAE9E,MAAM,eAAe,GAAG,qBAAqB,CAAC;AAC9C,MAAM,WAAW,GAAG,wBAAwB,CAAC;AAC7C,MAAM,eAAe,GAAG,wCAAwC,CAAC;AAEjE;;;GAGG;AACH,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,KAAa;IACjD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CACnB,GAAG,EACH;YACE,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,KAAK,EAAE;gBAChC,YAAY,EAAE,uBAAuB;aACtC;SACF,EACD,CAAC,GAAG,EAAE,EAAE;YACN,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;gBACrD,MAAM,CAAC,IAAI,KAAK,CAAC,iBAAiB,GAAG,CAAC,UAAU,QAAQ,GAAG,yEAAyE,CAAC,CAAC,CAAC;gBACvI,GAAG,CAAC,MAAM,EAAE,CAAC;gBACb,OAAO;YACT,CAAC;YACD,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;gBAC3B,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,UAAU,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC;gBACxD,GAAG,CAAC,MAAM,EAAE,CAAC;gBACb,OAAO;YACT,CAAC;YACD,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9C,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACjB,IAAI,CAAC;oBACH,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAC/D,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBAClE,CAAC;YACH,CAAC,CAAC,CAAC;YACH,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1B,CAAC,CACF,CAAC;QACF,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACxB,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,EAAE;YAC1B,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;GAQG;AACH,KAAK,UAAU,iBAAiB,CAC9B,KAAgB,EAChB,KAAa,EACb,QAAgB;IAEhB,SAAS,CAAC,QAAQ,CAAC,CAAC;IAEpB,oEAAoE;IACpE,oEAAoE;IACpE,MAAM,QAAQ,GAAG,MAAM,CAAC;IACxB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;IAEzE,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAmB,CAAC;QACjF,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,kEAAkE;IAClE,2EAA2E;IAC3E,wEAAwE;IACxE,yEAAyE;IACzE,yEAAyE;IACzE,MAAM,WAAW,GAAG,aAAa,KAAK,EAAE,CAAC;IACzC,MAAM,GAAG,GACP,GAAG,eAAe,iBAAiB,kBAAkB,CAAC,eAAe,CAAC,EAAE;QACxE,WAAW,WAAW,uCAAuC,CAAC;IAEhE,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,KAAK,CAAsD,CAAC;IAE9F,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,4DAA4D,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACpH,CAAC;IAED,MAAM,SAAS,GAAmB,IAAI,CAAC,IAAI;SACxC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;SACjB,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,KAAK,KAAK,CAAC;SAC/C,GAAG,CAAC,CAAC,GAAG,EAAgB,EAAE,CAAC,CAAC;QAC3B,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QACrC,KAAK;QACL,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QACvC,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;QACtD,SAAS,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;QAC7D,SAAS,EAAE,IAAI,EAAE,0CAA0C;QAC3D,kBAAkB,EAAE,GAAG,CAAC,oBAAoB,CAAwC;KACrF,CAAC,CAAC;SACF,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC;IAE5D,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACzE,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,8EAA8E;AAC9E,iCAAiC;AACjC,8EAA8E;AAE9E;;;;;;;GAOG;AACH,MAAM,YAAY,GAChB,kFAAkF,CAAC;AAErF;;;;GAIG;AACH,KAAK,UAAU,kBAAkB,CAC/B,MAAc,EACd,QAAgB,EAChB,KAAa,EACb,QAAgB;IAEhB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC/C,IAAI,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,QAAQ,CAAC,CAAC,iBAAiB;IAE/D,MAAM,OAAO,GAAG,GAAG,YAAY,IAAI,MAAM,IAAI,kBAAkB,CAAC,QAAQ,CAAC,EAAE,CAAC;IAC5E,SAAS,CAAC,QAAQ,CAAC,CAAC;IAEpB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,SAAS,KAAK,CAAC,UAAkB,EAAE,KAAa;YAC9C,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YAEzC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YACvC,MAAM,OAAO,GAA2B;gBACtC,YAAY,EAAE,uBAAuB;aACtC,CAAC;YACF,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBAC/C,OAAO,CAAC,eAAe,CAAC,GAAG,UAAU,KAAK,EAAE,CAAC;YAC/C,CAAC;YAED,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,EAAE,EAAE;gBACrD,IAAI,GAAG,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;oBACzE,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;oBACpC,GAAG,CAAC,MAAM,EAAE,CAAC;oBACb,IAAI,GAAG;wBAAE,KAAK,CAAC,GAAG,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;;wBAC1B,OAAO,CAAC,IAAI,CAAC,CAAC;oBACnB,OAAO;gBACT,CAAC;gBACD,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;oBAC3B,GAAG,CAAC,MAAM,EAAE,CAAC;oBACb,OAAO,CAAC,IAAI,CAAC,CAAC;oBACd,OAAO;gBACT,CAAC;gBACD,MAAM,MAAM,GAAa,EAAE,CAAC;gBAC5B,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC9C,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;oBACjB,IAAI,CAAC;wBACH,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;wBAClD,OAAO,CAAC,QAAQ,CAAC,CAAC;oBACpB,CAAC;oBAAC,MAAM,CAAC;wBACP,OAAO,CAAC,IAAI,CAAC,CAAC;oBAChB,CAAC;gBACH,CAAC,CAAC,CAAC;gBACH,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;YACvC,CAAC,CAAC,CAAC;YACH,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;YACrC,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,CAAC;QACD,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IACpB,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,SAAyB,EACzB,KAAa,EACb,QAAgB;IAEhB,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACvD,MAAM,OAAO,CAAC,GAAG,CACf,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QACxB,MAAM,SAAS,GAAG,MAAM,kBAAkB,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,SAAU,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;QACrF,CAAC,CAAC,SAAS,GAAG,SAAS,CAAC;IAC1B,CAAC,CAAC,CACH,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,UAA2B,EAAE;IAC1D,MAAM,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,EAAE,SAAS,GAAG,KAAK,EAAE,QAAQ,GAAG,eAAe,EAAE,EAAE,GAAG,OAAO,CAAC;IAEtF,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC;QAChE,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IACnE,CAAC;IAED,MAAM,KAAK,GAAG,cAAc,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;IAClE,wFAAwF;IACxF,MAAM,kBAAkB,CAAC,SAAS,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;IACrD,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACrE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,QAAiB;IAC/C,OAAO,QAAQ,IAAI,eAAe,EAAE,CAAC;AACvC,CAAC"}
@@ -0,0 +1,67 @@
1
+ /**
2
+ * GAIA Mode Router — iter 58 (#2156)
3
+ *
4
+ * Rule-based classifier that maps each GAIA question to the appropriate
5
+ * agent mode. Empirical basis from iter 57:
6
+ * - ToolCalling (gaia-agent.ts) wins retrieval/attachment questions
7
+ * - CodeAgent (gaia-claude-p.ts) wins pure-reasoning questions
8
+ * - Naïve combination regressed by -10q, so routing is required
9
+ *
10
+ * Decision rules (evaluated in priority order):
11
+ *
12
+ * 1. ATTACHMENT — question has file_name → ToolCalling
13
+ * CodeAgent struggles with attachment piping; ToolCalling natively
14
+ * handles XLSX/PPTX/images/audio via attachment tools.
15
+ *
16
+ * 2. WEB_RETRIEVAL — question text matches retrieval keywords →
17
+ * ToolCalling (has WebSearch/WebFetch in the tool catalogue)
18
+ *
19
+ * 3. PURE_REASONING — question text matches computation/logic keywords →
20
+ * CodeAgent (executes arbitrary Python, no web round-trip needed)
21
+ *
22
+ * 4. LONG_QUESTION — question length > 400 chars → ToolCalling
23
+ * Long questions typically involve multi-hop retrieval steps.
24
+ *
25
+ * 5. DEFAULT — ToolCalling (higher baseline per iter 56b)
26
+ *
27
+ * Each decision is logged with the rule that fired so per-Q audit is trivial.
28
+ *
29
+ * Refs: iter 57 architectural finding, iter 58, #2156
30
+ */
31
+ import type { GaiaQuestion } from './gaia-loader.js';
32
+ /** The two agent modes the hybrid router can dispatch to. */
33
+ export type GaiaMode = 'ToolCalling' | 'CodeAgent';
34
+ /** Routing rule identifiers (used for audit logging). */
35
+ export type RoutingRule = 'attachment' | 'web_retrieval' | 'pure_reasoning' | 'long_question' | 'default';
36
+ /** Result of classifying a single question. */
37
+ export interface RoutingDecision {
38
+ /** Chosen agent mode. */
39
+ mode: GaiaMode;
40
+ /** The rule that fired and determined the mode. */
41
+ rule: RoutingRule;
42
+ /** Human-readable explanation (suitable for per-Q log). */
43
+ reason: string;
44
+ }
45
+ /**
46
+ * Classify a GAIA question into a mode + rule pair.
47
+ *
48
+ * Rules are evaluated in strict priority order — the first matching rule wins.
49
+ */
50
+ export declare function routeQuestion(question: GaiaQuestion): RoutingDecision;
51
+ /** Summary produced by routeQuestions for logging purposes. */
52
+ export interface RoutingSummary {
53
+ total: number;
54
+ toolCalling: number;
55
+ codeAgent: number;
56
+ /** Questions routed to ToolCalling, sorted by rule. */
57
+ byRule: Record<RoutingRule, number>;
58
+ }
59
+ /**
60
+ * Route an array of questions and return per-Q decisions plus a summary.
61
+ * Zero-allocation: decisions array is returned in input order.
62
+ */
63
+ export declare function routeQuestions(questions: GaiaQuestion[]): {
64
+ decisions: RoutingDecision[];
65
+ summary: RoutingSummary;
66
+ };
67
+ //# sourceMappingURL=gaia-mode-router.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"gaia-mode-router.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/gaia-mode-router.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMrD,6DAA6D;AAC7D,MAAM,MAAM,QAAQ,GAAG,aAAa,GAAG,WAAW,CAAC;AAEnD,yDAAyD;AACzD,MAAM,MAAM,WAAW,GACnB,YAAY,GACZ,eAAe,GACf,gBAAgB,GAChB,eAAe,GACf,SAAS,CAAC;AAEd,+CAA+C;AAC/C,MAAM,WAAW,eAAe;IAC9B,yBAAyB;IACzB,IAAI,EAAE,QAAQ,CAAC;IACf,mDAAmD;IACnD,IAAI,EAAE,WAAW,CAAC;IAClB,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;CAChB;AA2BD;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,YAAY,GAAG,eAAe,CA+CrE;AAMD,+DAA+D;AAC/D,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,uDAAuD;IACvD,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;CACrC;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,SAAS,EAAE,YAAY,EAAE,GAAG;IACzD,SAAS,EAAE,eAAe,EAAE,CAAC;IAC7B,OAAO,EAAE,cAAc,CAAC;CACzB,CAwBA"}
@@ -0,0 +1,126 @@
1
+ /**
2
+ * GAIA Mode Router — iter 58 (#2156)
3
+ *
4
+ * Rule-based classifier that maps each GAIA question to the appropriate
5
+ * agent mode. Empirical basis from iter 57:
6
+ * - ToolCalling (gaia-agent.ts) wins retrieval/attachment questions
7
+ * - CodeAgent (gaia-claude-p.ts) wins pure-reasoning questions
8
+ * - Naïve combination regressed by -10q, so routing is required
9
+ *
10
+ * Decision rules (evaluated in priority order):
11
+ *
12
+ * 1. ATTACHMENT — question has file_name → ToolCalling
13
+ * CodeAgent struggles with attachment piping; ToolCalling natively
14
+ * handles XLSX/PPTX/images/audio via attachment tools.
15
+ *
16
+ * 2. WEB_RETRIEVAL — question text matches retrieval keywords →
17
+ * ToolCalling (has WebSearch/WebFetch in the tool catalogue)
18
+ *
19
+ * 3. PURE_REASONING — question text matches computation/logic keywords →
20
+ * CodeAgent (executes arbitrary Python, no web round-trip needed)
21
+ *
22
+ * 4. LONG_QUESTION — question length > 400 chars → ToolCalling
23
+ * Long questions typically involve multi-hop retrieval steps.
24
+ *
25
+ * 5. DEFAULT — ToolCalling (higher baseline per iter 56b)
26
+ *
27
+ * Each decision is logged with the rule that fired so per-Q audit is trivial.
28
+ *
29
+ * Refs: iter 57 architectural finding, iter 58, #2156
30
+ */
31
+ // ---------------------------------------------------------------------------
32
+ // Regex patterns
33
+ // ---------------------------------------------------------------------------
34
+ /**
35
+ * Keywords that indicate the question requires live web retrieval.
36
+ * ToolCalling has WebSearch + WebFetch in its catalogue.
37
+ */
38
+ const WEB_RETRIEVAL_RE = /website|webpage|visit|browse|wikipedia|article|page about|paper titled|published in|according to|the.*\bsite\b|link|url|online|source|reference|official|org\b|\.com\b|\.gov\b|based on the.*page/i;
39
+ /**
40
+ * Keywords that indicate the question is a pure computation / logic problem.
41
+ * CodeAgent runs Python via `claude -p` Bash tool and handles these well.
42
+ */
43
+ const PURE_REASONING_RE = /calculate|compute|math|equation|how many|distance|volume|combinations|permutations|logic puzzle|deduction|deduce|sequence|series|prove|derivat|integral|differential|probability|odds|fraction|percentage of|modulo|prime\b|fibonacci|recursive|recurrence|sorting|algorithm|big-?o|time complexity|bits?|bytes?|binary|hexadecimal|base \d/i;
44
+ /** Maximum question length before we treat it as a retrieval question. */
45
+ const LONG_QUESTION_THRESHOLD = 400;
46
+ // ---------------------------------------------------------------------------
47
+ // Core classifier
48
+ // ---------------------------------------------------------------------------
49
+ /**
50
+ * Classify a GAIA question into a mode + rule pair.
51
+ *
52
+ * Rules are evaluated in strict priority order — the first matching rule wins.
53
+ */
54
+ export function routeQuestion(question) {
55
+ // Rule 1: attachment present → ToolCalling
56
+ if (question.file_name) {
57
+ return {
58
+ mode: 'ToolCalling',
59
+ rule: 'attachment',
60
+ reason: `file_name="${question.file_name}" — attachment tools in ToolCalling`,
61
+ };
62
+ }
63
+ const text = question.question;
64
+ // Rule 2: web retrieval keywords → ToolCalling
65
+ if (WEB_RETRIEVAL_RE.test(text)) {
66
+ const match = text.match(WEB_RETRIEVAL_RE);
67
+ return {
68
+ mode: 'ToolCalling',
69
+ rule: 'web_retrieval',
70
+ reason: `web-retrieval keyword "${match?.[0]}" detected`,
71
+ };
72
+ }
73
+ // Rule 3: pure reasoning / computation keywords → CodeAgent
74
+ if (PURE_REASONING_RE.test(text)) {
75
+ const match = text.match(PURE_REASONING_RE);
76
+ return {
77
+ mode: 'CodeAgent',
78
+ rule: 'pure_reasoning',
79
+ reason: `reasoning keyword "${match?.[0]}" detected`,
80
+ };
81
+ }
82
+ // Rule 4: long question → ToolCalling
83
+ if (text.length > LONG_QUESTION_THRESHOLD) {
84
+ return {
85
+ mode: 'ToolCalling',
86
+ rule: 'long_question',
87
+ reason: `question length ${text.length} > ${LONG_QUESTION_THRESHOLD} chars`,
88
+ };
89
+ }
90
+ // Rule 5: default → ToolCalling (higher baseline per iter 56b)
91
+ return {
92
+ mode: 'ToolCalling',
93
+ rule: 'default',
94
+ reason: 'no specific signal — defaulting to ToolCalling (iter 56b baseline)',
95
+ };
96
+ }
97
+ /**
98
+ * Route an array of questions and return per-Q decisions plus a summary.
99
+ * Zero-allocation: decisions array is returned in input order.
100
+ */
101
+ export function routeQuestions(questions) {
102
+ const decisions = [];
103
+ const byRule = {
104
+ attachment: 0,
105
+ web_retrieval: 0,
106
+ pure_reasoning: 0,
107
+ long_question: 0,
108
+ default: 0,
109
+ };
110
+ let toolCalling = 0;
111
+ let codeAgent = 0;
112
+ for (const q of questions) {
113
+ const decision = routeQuestion(q);
114
+ decisions.push(decision);
115
+ byRule[decision.rule]++;
116
+ if (decision.mode === 'ToolCalling')
117
+ toolCalling++;
118
+ else
119
+ codeAgent++;
120
+ }
121
+ return {
122
+ decisions,
123
+ summary: { total: questions.length, toolCalling, codeAgent, byRule },
124
+ };
125
+ }
126
+ //# sourceMappingURL=gaia-mode-router.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"gaia-mode-router.js","sourceRoot":"","sources":["../../../src/benchmarks/gaia-mode-router.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AA6BH,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E;;;GAGG;AACH,MAAM,gBAAgB,GACpB,oMAAoM,CAAC;AAEvM;;;GAGG;AACH,MAAM,iBAAiB,GACrB,8UAA8U,CAAC;AAEjV,0EAA0E;AAC1E,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAEpC,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,QAAsB;IAClD,2CAA2C;IAC3C,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;QACvB,OAAO;YACL,IAAI,EAAE,aAAa;YACnB,IAAI,EAAE,YAAY;YAClB,MAAM,EAAE,cAAc,QAAQ,CAAC,SAAS,qCAAqC;SAC9E,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC;IAE/B,+CAA+C;IAC/C,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QAC3C,OAAO;YACL,IAAI,EAAE,aAAa;YACnB,IAAI,EAAE,eAAe;YACrB,MAAM,EAAE,0BAA0B,KAAK,EAAE,CAAC,CAAC,CAAC,YAAY;SACzD,CAAC;IACJ,CAAC;IAED,4DAA4D;IAC5D,IAAI,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAC5C,OAAO;YACL,IAAI,EAAE,WAAW;YACjB,IAAI,EAAE,gBAAgB;YACtB,MAAM,EAAE,sBAAsB,KAAK,EAAE,CAAC,CAAC,CAAC,YAAY;SACrD,CAAC;IACJ,CAAC;IAED,sCAAsC;IACtC,IAAI,IAAI,CAAC,MAAM,GAAG,uBAAuB,EAAE,CAAC;QAC1C,OAAO;YACL,IAAI,EAAE,aAAa;YACnB,IAAI,EAAE,eAAe;YACrB,MAAM,EAAE,mBAAmB,IAAI,CAAC,MAAM,MAAM,uBAAuB,QAAQ;SAC5E,CAAC;IACJ,CAAC;IAED,+DAA+D;IAC/D,OAAO;QACL,IAAI,EAAE,aAAa;QACnB,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,oEAAoE;KAC7E,CAAC;AACJ,CAAC;AAeD;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,SAAyB;IAItD,MAAM,SAAS,GAAsB,EAAE,CAAC;IACxC,MAAM,MAAM,GAAgC;QAC1C,UAAU,EAAE,CAAC;QACb,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,CAAC;QACjB,aAAa,EAAE,CAAC;QAChB,OAAO,EAAE,CAAC;KACX,CAAC;IACF,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QAClC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzB,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,IAAI,QAAQ,CAAC,IAAI,KAAK,aAAa;YAAE,WAAW,EAAE,CAAC;;YAC9C,SAAS,EAAE,CAAC;IACnB,CAAC;IAED,OAAO;QACL,SAAS;QACT,OAAO,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE;KACrE,CAAC;AACJ,CAAC"}
@@ -0,0 +1,15 @@
1
+ /**
2
+ * Smoke tests for gaia-mode-router.ts — iter 58 (#2156)
3
+ *
4
+ * Synthetic question texts covering each routing rule.
5
+ * All 10 cases must pass (no API calls — purely rule-based).
6
+ *
7
+ * Run:
8
+ * npx ts-node src/benchmarks/gaia-mode-router.smoke.ts
9
+ * # or after build:
10
+ * node dist/src/benchmarks/gaia-mode-router.smoke.js
11
+ *
12
+ * Refs: iter 58, #2156
13
+ */
14
+ export {};
15
+ //# sourceMappingURL=gaia-mode-router.smoke.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"gaia-mode-router.smoke.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/gaia-mode-router.smoke.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG"}