@claude-flow/cli 3.11.0 → 3.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. package/dist/src/commands/doctor.d.ts.map +1 -1
  2. package/dist/src/commands/doctor.js +175 -1
  3. package/dist/src/commands/doctor.js.map +1 -1
  4. package/dist/src/commands/eject.d.ts +33 -0
  5. package/dist/src/commands/eject.d.ts.map +1 -0
  6. package/dist/src/commands/eject.js +195 -0
  7. package/dist/src/commands/eject.js.map +1 -0
  8. package/dist/src/commands/hooks.d.ts.map +1 -1
  9. package/dist/src/commands/hooks.js +2 -1
  10. package/dist/src/commands/hooks.js.map +1 -1
  11. package/dist/src/commands/index.d.ts.map +1 -1
  12. package/dist/src/commands/index.js +4 -0
  13. package/dist/src/commands/index.js.map +1 -1
  14. package/dist/src/commands/init.d.ts.map +1 -1
  15. package/dist/src/commands/init.js +3 -0
  16. package/dist/src/commands/init.js.map +1 -1
  17. package/dist/src/commands/metaharness.d.ts +39 -0
  18. package/dist/src/commands/metaharness.d.ts.map +1 -0
  19. package/dist/src/commands/metaharness.js +194 -0
  20. package/dist/src/commands/metaharness.js.map +1 -0
  21. package/dist/src/mcp-client.d.ts.map +1 -1
  22. package/dist/src/mcp-client.js +4 -0
  23. package/dist/src/mcp-client.js.map +1 -1
  24. package/dist/src/mcp-tools/index.d.ts +1 -0
  25. package/dist/src/mcp-tools/index.d.ts.map +1 -1
  26. package/dist/src/mcp-tools/index.js +2 -0
  27. package/dist/src/mcp-tools/index.js.map +1 -1
  28. package/dist/src/mcp-tools/metaharness-tools.d.ts +31 -0
  29. package/dist/src/mcp-tools/metaharness-tools.d.ts.map +1 -0
  30. package/dist/src/mcp-tools/metaharness-tools.js +372 -0
  31. package/dist/src/mcp-tools/metaharness-tools.js.map +1 -0
  32. package/dist/src/plugins/store/discovery.d.ts.map +1 -1
  33. package/dist/src/plugins/store/discovery.js +3 -0
  34. package/dist/src/plugins/store/discovery.js.map +1 -1
  35. package/dist/src/plugins/store/types.d.ts +6 -1
  36. package/dist/src/plugins/store/types.d.ts.map +1 -1
  37. package/dist/src/ruvector/model-router.d.ts.map +1 -1
  38. package/dist/src/ruvector/model-router.js +37 -0
  39. package/dist/src/ruvector/model-router.js.map +1 -1
  40. package/dist/src/ruvector/router-parallel-recorder.d.ts +127 -0
  41. package/dist/src/ruvector/router-parallel-recorder.d.ts.map +1 -0
  42. package/dist/src/ruvector/router-parallel-recorder.js +183 -0
  43. package/dist/src/ruvector/router-parallel-recorder.js.map +1 -0
  44. package/dist/tsconfig.tsbuildinfo +1 -1
  45. package/package.json +11 -9
  46. package/dist/src/benchmarks/capability-tasks.d.ts +0 -126
  47. package/dist/src/benchmarks/capability-tasks.d.ts.map +0 -1
  48. package/dist/src/benchmarks/capability-tasks.js +0 -147
  49. package/dist/src/benchmarks/capability-tasks.js.map +0 -1
  50. package/dist/src/benchmarks/gaia-agent-contrastive.d.ts +0 -53
  51. package/dist/src/benchmarks/gaia-agent-contrastive.d.ts.map +0 -1
  52. package/dist/src/benchmarks/gaia-agent-contrastive.js +0 -177
  53. package/dist/src/benchmarks/gaia-agent-contrastive.js.map +0 -1
  54. package/dist/src/benchmarks/gaia-agent-gemini.d.ts +0 -51
  55. package/dist/src/benchmarks/gaia-agent-gemini.d.ts.map +0 -1
  56. package/dist/src/benchmarks/gaia-agent-gemini.js +0 -352
  57. package/dist/src/benchmarks/gaia-agent-gemini.js.map +0 -1
  58. package/dist/src/benchmarks/gaia-claude-p.d.ts +0 -105
  59. package/dist/src/benchmarks/gaia-claude-p.d.ts.map +0 -1
  60. package/dist/src/benchmarks/gaia-claude-p.js +0 -269
  61. package/dist/src/benchmarks/gaia-claude-p.js.map +0 -1
  62. package/dist/src/benchmarks/gaia-claude-p.smoke.d.ts +0 -20
  63. package/dist/src/benchmarks/gaia-claude-p.smoke.d.ts.map +0 -1
  64. package/dist/src/benchmarks/gaia-claude-p.smoke.js +0 -168
  65. package/dist/src/benchmarks/gaia-claude-p.smoke.js.map +0 -1
  66. package/dist/src/benchmarks/gaia-codeagent-runner.py +0 -556
  67. package/dist/src/benchmarks/gaia-codeagent.d.ts +0 -129
  68. package/dist/src/benchmarks/gaia-codeagent.d.ts.map +0 -1
  69. package/dist/src/benchmarks/gaia-codeagent.js +0 -590
  70. package/dist/src/benchmarks/gaia-codeagent.js.map +0 -1
  71. package/dist/src/benchmarks/gaia-codeagent.smoke.d.ts +0 -33
  72. package/dist/src/benchmarks/gaia-codeagent.smoke.d.ts.map +0 -1
  73. package/dist/src/benchmarks/gaia-codeagent.smoke.js +0 -241
  74. package/dist/src/benchmarks/gaia-codeagent.smoke.js.map +0 -1
  75. package/dist/src/benchmarks/gaia-dag-pilot.d.ts +0 -22
  76. package/dist/src/benchmarks/gaia-dag-pilot.d.ts.map +0 -1
  77. package/dist/src/benchmarks/gaia-dag-pilot.js +0 -158
  78. package/dist/src/benchmarks/gaia-dag-pilot.js.map +0 -1
  79. package/dist/src/benchmarks/gaia-dag.d.ts +0 -110
  80. package/dist/src/benchmarks/gaia-dag.d.ts.map +0 -1
  81. package/dist/src/benchmarks/gaia-dag.js +0 -658
  82. package/dist/src/benchmarks/gaia-dag.js.map +0 -1
  83. package/dist/src/benchmarks/gaia-dag.smoke.d.ts +0 -21
  84. package/dist/src/benchmarks/gaia-dag.smoke.d.ts.map +0 -1
  85. package/dist/src/benchmarks/gaia-dag.smoke.js +0 -290
  86. package/dist/src/benchmarks/gaia-dag.smoke.js.map +0 -1
  87. package/dist/src/benchmarks/gaia-ensemble-pilot.d.ts +0 -17
  88. package/dist/src/benchmarks/gaia-ensemble-pilot.d.ts.map +0 -1
  89. package/dist/src/benchmarks/gaia-ensemble-pilot.js +0 -106
  90. package/dist/src/benchmarks/gaia-ensemble-pilot.js.map +0 -1
  91. package/dist/src/benchmarks/gaia-ensemble.d.ts +0 -103
  92. package/dist/src/benchmarks/gaia-ensemble.d.ts.map +0 -1
  93. package/dist/src/benchmarks/gaia-ensemble.js +0 -462
  94. package/dist/src/benchmarks/gaia-ensemble.js.map +0 -1
  95. package/dist/src/benchmarks/gaia-gemini-pilot.d.ts +0 -17
  96. package/dist/src/benchmarks/gaia-gemini-pilot.d.ts.map +0 -1
  97. package/dist/src/benchmarks/gaia-gemini-pilot.js +0 -168
  98. package/dist/src/benchmarks/gaia-gemini-pilot.js.map +0 -1
  99. package/dist/src/benchmarks/gaia-mode-router.d.ts +0 -67
  100. package/dist/src/benchmarks/gaia-mode-router.d.ts.map +0 -1
  101. package/dist/src/benchmarks/gaia-mode-router.js +0 -126
  102. package/dist/src/benchmarks/gaia-mode-router.js.map +0 -1
  103. package/dist/src/benchmarks/gaia-mode-router.smoke.d.ts +0 -15
  104. package/dist/src/benchmarks/gaia-mode-router.smoke.d.ts.map +0 -1
  105. package/dist/src/benchmarks/gaia-mode-router.smoke.js +0 -131
  106. package/dist/src/benchmarks/gaia-mode-router.smoke.js.map +0 -1
  107. package/dist/src/benchmarks/gaia-tools/grounded_query.smoke.d.ts +0 -24
  108. package/dist/src/benchmarks/gaia-tools/grounded_query.smoke.d.ts.map +0 -1
  109. package/dist/src/benchmarks/gaia-tools/grounded_query.smoke.js +0 -276
  110. package/dist/src/benchmarks/gaia-tools/grounded_query.smoke.js.map +0 -1
  111. package/dist/src/benchmarks/gaia-tools/image_describe.d.ts +0 -45
  112. package/dist/src/benchmarks/gaia-tools/image_describe.d.ts.map +0 -1
  113. package/dist/src/benchmarks/gaia-tools/image_describe.js +0 -222
  114. package/dist/src/benchmarks/gaia-tools/image_describe.js.map +0 -1
  115. package/dist/src/benchmarks/gaia-tools/image_describe.smoke.d.ts +0 -24
  116. package/dist/src/benchmarks/gaia-tools/image_describe.smoke.d.ts.map +0 -1
  117. package/dist/src/benchmarks/gaia-tools/image_describe.smoke.js +0 -169
  118. package/dist/src/benchmarks/gaia-tools/image_describe.smoke.js.map +0 -1
  119. package/dist/src/benchmarks/gaia-tools/pdf_read.d.ts +0 -27
  120. package/dist/src/benchmarks/gaia-tools/pdf_read.d.ts.map +0 -1
  121. package/dist/src/benchmarks/gaia-tools/pdf_read.js +0 -226
  122. package/dist/src/benchmarks/gaia-tools/pdf_read.js.map +0 -1
  123. package/dist/src/benchmarks/gaia-tools/python_exec.d.ts +0 -31
  124. package/dist/src/benchmarks/gaia-tools/python_exec.d.ts.map +0 -1
  125. package/dist/src/benchmarks/gaia-tools/python_exec.js +0 -94
  126. package/dist/src/benchmarks/gaia-tools/python_exec.js.map +0 -1
  127. package/dist/src/benchmarks/gaia-tools/python_exec.smoke.d.ts +0 -13
  128. package/dist/src/benchmarks/gaia-tools/python_exec.smoke.d.ts.map +0 -1
  129. package/dist/src/benchmarks/gaia-tools/python_exec.smoke.js +0 -98
  130. package/dist/src/benchmarks/gaia-tools/python_exec.smoke.js.map +0 -1
  131. package/dist/src/benchmarks/gaia-tools/visit_webpage.d.ts +0 -52
  132. package/dist/src/benchmarks/gaia-tools/visit_webpage.d.ts.map +0 -1
  133. package/dist/src/benchmarks/gaia-tools/visit_webpage.js +0 -304
  134. package/dist/src/benchmarks/gaia-tools/visit_webpage.js.map +0 -1
  135. package/dist/src/benchmarks/gaia-tools/web_browse.d.ts +0 -61
  136. package/dist/src/benchmarks/gaia-tools/web_browse.d.ts.map +0 -1
  137. package/dist/src/benchmarks/gaia-tools/web_browse.js +0 -217
  138. package/dist/src/benchmarks/gaia-tools/web_browse.js.map +0 -1
  139. package/dist/src/benchmarks/gaia-tools/web_browse.smoke.d.ts +0 -26
  140. package/dist/src/benchmarks/gaia-tools/web_browse.smoke.d.ts.map +0 -1
  141. package/dist/src/benchmarks/gaia-tools/web_browse.smoke.js +0 -152
  142. package/dist/src/benchmarks/gaia-tools/web_browse.smoke.js.map +0 -1
  143. package/dist/src/benchmarks/gaia-tools/web_search.smoke.d.ts +0 -24
  144. package/dist/src/benchmarks/gaia-tools/web_search.smoke.d.ts.map +0 -1
  145. package/dist/src/benchmarks/gaia-tools/web_search.smoke.js +0 -311
  146. package/dist/src/benchmarks/gaia-tools/web_search.smoke.js.map +0 -1
  147. package/dist/src/benchmarks/gaia-visit-webpage.smoke.d.ts +0 -25
  148. package/dist/src/benchmarks/gaia-visit-webpage.smoke.d.ts.map +0 -1
  149. package/dist/src/benchmarks/gaia-visit-webpage.smoke.js +0 -304
  150. package/dist/src/benchmarks/gaia-visit-webpage.smoke.js.map +0 -1
  151. package/dist/src/commands/performance-capability.d.ts +0 -30
  152. package/dist/src/commands/performance-capability.d.ts.map +0 -1
  153. package/dist/src/commands/performance-capability.js +0 -349
  154. package/dist/src/commands/performance-capability.js.map +0 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@claude-flow/cli",
3
- "version": "3.11.0",
3
+ "version": "3.12.0",
4
4
  "type": "module",
5
5
  "description": "Ruflo CLI - Enterprise AI agent orchestration with 60+ specialized agents, swarm coordination, MCP server, self-learning hooks, and vector memory for Claude Code",
6
6
  "main": "dist/src/index.js",
@@ -95,13 +95,13 @@
95
95
  "vitest": "^4.0.16"
96
96
  },
97
97
  "dependencies": {
98
- "@claude-flow/cli-core": "^3.7.0-alpha.5",
99
- "@claude-flow/mcp": "^3.0.0-alpha.8",
100
- "@claude-flow/neural": "^3.0.0-alpha.9",
101
- "@claude-flow/shared": "^3.0.0-alpha.7",
102
- "@noble/ed25519": "^2.1.0",
103
- "@ruvector/rabitq-wasm": "^0.1.0",
104
- "semver": "^7.6.0",
98
+ "@claude-flow/cli-core": "3.7.0-alpha.5",
99
+ "@claude-flow/mcp": "3.0.0-alpha.8",
100
+ "@claude-flow/neural": "3.0.0-alpha.9",
101
+ "@claude-flow/shared": "3.0.0-alpha.7",
102
+ "@noble/ed25519": "2.3.0",
103
+ "@ruvector/rabitq-wasm": "0.1.0",
104
+ "semver": "7.7.3",
105
105
  "yaml": "^2.8.0"
106
106
  },
107
107
  "optionalDependencies": {
@@ -112,7 +112,9 @@
112
112
  "@claude-flow/memory": "^3.0.0-alpha.20",
113
113
  "@claude-flow/plugin-gastown-bridge": "^0.1.3",
114
114
  "@claude-flow/security": "^3.0.0-alpha.10",
115
- "@metaharness/router": "^0.3.2",
115
+ "@metaharness/kernel": "~0.1.0",
116
+ "@metaharness/router": "~0.3.2",
117
+ "metaharness": "~0.1.11",
116
118
  "@ruvector/attention": "^0.1.32",
117
119
  "@ruvector/attention-darwin-arm64": "0.1.32",
118
120
  "@ruvector/diskann": "^0.1.0",
@@ -1,126 +0,0 @@
1
- export declare const BUILTIN_CAPABILITY_TASKS: {
2
- readonly version: "1.3";
3
- readonly description: "Text-only agent capability benchmark — verifiable multi-step reasoning tasks scoreable without tool use. Format inspired by GAIA / SWE-bench / GSM8K. The fixture mixes EASY (regression-floor) and HARD (model-gradient) questions so the pass rate has signal across Haiku → Sonnet → Opus. Real GAIA (web browsing, attachments, HF dataset) remains future work.";
4
- readonly answerFormat: "Each question requires the model to reply with the answer wrapped in <answer>...</answer> tags. The harness extracts the tag contents and checks against `expected` per `matchMode`. Per-task `maxTokens` overrides the default cap.";
5
- readonly tasks: readonly [{
6
- readonly id: "math-prime";
7
- readonly category: "easy:reasoning";
8
- readonly prompt: "What is the smallest 3-digit prime number that does not contain the digit 7?";
9
- readonly expected: "101";
10
- readonly matchMode: "exact";
11
- readonly maxTokens: 192;
12
- }, {
13
- readonly id: "logic-syllogism";
14
- readonly category: "easy:reasoning";
15
- readonly prompt: "All routers in tier 1 cost less than $0.001 per call. The Booster router is in tier 1. The Sonnet router costs $0.003 per call. Is the Sonnet router in tier 1? Answer with just \"yes\" or \"no\".";
16
- readonly expected: "no";
17
- readonly matchMode: "exact";
18
- readonly maxTokens: 160;
19
- }, {
20
- readonly id: "regex-match";
21
- readonly category: "easy:code-reasoning";
22
- readonly prompt: "Given the regex /^([a-z]+)-(\\d+)$/ and the input string 'pattern-1779526376', what is the value of capture group 2?";
23
- readonly expected: "1779526376";
24
- readonly matchMode: "exact";
25
- readonly maxTokens: 192;
26
- }, {
27
- readonly id: "gsm8k-trip";
28
- readonly category: "hard:gsm8k-style";
29
- readonly prompt: "A delivery van starts a route with 240 packages. At stop A it drops off 1/4 of its current load and picks up 6 new packages. At stop B it drops off 1/3 of its current load and picks up 4 new packages. At stop C it drops off half of its current load. How many packages does the van have after stop C? Answer with the integer.";
30
- readonly expected: "64";
31
- readonly matchMode: "exact";
32
- readonly maxTokens: 256;
33
- }, {
34
- readonly id: "gsm8k-discount";
35
- readonly category: "hard:gsm8k-style";
36
- readonly prompt: "A store sells 3 widgets and 2 sprockets for $23. It also sells 2 widgets and 4 sprockets for $26. What is the price of one widget? Answer with the integer dollar amount only.";
37
- readonly expected: "5";
38
- readonly matchMode: "exact";
39
- readonly maxTokens: 256;
40
- }, {
41
- readonly id: "code-trace";
42
- readonly category: "hard:code-trace";
43
- readonly prompt: "Consider this JavaScript code:\n```\nconst counts = new Map();\nfor (const c of 'abracadabra') {\n counts.set(c, (counts.get(c) ?? 0) + 1);\n}\nlet maxK = '', maxV = 0;\nfor (const [k, v] of counts) {\n if (v > maxV || (v === maxV && k < maxK)) { maxK = k; maxV = v; }\n}\nconsole.log(`${maxK}:${maxV}`);\n```\nWhat does it print? Answer with just the printed string.";
44
- readonly expected: "a:5";
45
- readonly matchMode: "exact";
46
- readonly maxTokens: 192;
47
- }, {
48
- readonly id: "hard-graph-shortest";
49
- readonly category: "hard:graph-reasoning";
50
- readonly prompt: "A directed graph has these weighted edges: A→B(3), A→C(7), B→C(2), B→D(5), C→D(1), C→E(4), D→E(2). What is the cost of the shortest path from A to E? Answer with the integer.";
51
- readonly expected: "8";
52
- readonly matchMode: "exact";
53
- readonly maxTokens: 192;
54
- }, {
55
- readonly id: "hard-probability";
56
- readonly category: "hard:probability";
57
- readonly prompt: "A bag contains 5 red, 3 blue, and 2 green balls. Two balls are drawn without replacement. What is the probability that both balls are the same color? Express as a fraction in lowest terms in the form a/b (e.g. 3/10).";
58
- readonly expected: "14/45";
59
- readonly matchMode: "exact";
60
- readonly maxTokens: 256;
61
- }, {
62
- readonly id: "expert-marble-inverse";
63
- readonly category: "expert:inverse-arithmetic";
64
- readonly prompt: "A bag of marbles is split as follows: 40% are given to Alice, then 25% of the REMAINING marbles are given to Bob, then half of the remaining marbles (after Alice and Bob) are given to Carol. Carol receives exactly 18 marbles. How many marbles were in the bag originally? Answer with the integer.";
65
- readonly expected: "80";
66
- readonly matchMode: "exact";
67
- readonly maxTokens: 512;
68
- }, {
69
- readonly id: "expert-crt";
70
- readonly category: "expert:number-theory";
71
- readonly prompt: "Find the smallest positive integer n such that all three of these hold simultaneously: n mod 7 = 3, n mod 9 = 4, n mod 11 = 5. Answer with just the integer.";
72
- readonly expected: "346";
73
- readonly matchMode: "exact";
74
- readonly maxTokens: 768;
75
- }, {
76
- readonly id: "expert-bayes";
77
- readonly category: "expert:bayesian";
78
- readonly prompt: "A medical test has 95% sensitivity (true positive rate) and 90% specificity (true negative rate). In the screened population, 1% of people have the disease. A patient tests positive. What is the probability the patient actually has the disease, rounded to the nearest whole percent? Answer with just the integer percentage (no '%' sign).";
79
- readonly expected: "9";
80
- readonly matchMode: "exact";
81
- readonly maxTokens: 512;
82
- }, {
83
- readonly id: "expert-banana";
84
- readonly category: "expert:combinatorics";
85
- readonly prompt: "In how many distinct arrangements of the letters of the word BANANA do no two N's appear next to each other? Answer with the integer.";
86
- readonly expected: "40";
87
- readonly matchMode: "exact";
88
- readonly maxTokens: 512;
89
- }, {
90
- readonly id: "expert-rectangle";
91
- readonly category: "expert:diophantine";
92
- readonly prompt: "A rectangle has positive integer side lengths. Its perimeter (in linear units) is numerically equal to its area (in square units). Two rectangles that are rotations of each other (e.g. 3x6 and 6x3) count as the same rectangle. What is the sum of the areas of all distinct such rectangles? Answer with the integer.";
93
- readonly expected: "34";
94
- readonly matchMode: "exact";
95
- readonly maxTokens: 512;
96
- }, {
97
- readonly id: "expert-dice-reroll";
98
- readonly category: "expert:expected-value";
99
- readonly prompt: "You roll a fair 6-sided die. If the result is a 6, you reroll exactly once and take the new result. Otherwise, you keep the original result. What is the expected value of your final number? Express the answer as a fraction in lowest terms in the form a/b (no spaces, no surrounding text).";
100
- readonly expected: "37/12";
101
- readonly matchMode: "exact";
102
- readonly maxTokens: 512;
103
- }, {
104
- readonly id: "sonnet-killer-knights";
105
- readonly category: "sonnet-killer:logic-puzzle";
106
- readonly prompt: "On an island, knights always tell the truth and knaves always lie. You meet four people named Alice, Bob, Carol, and Dan. They make the following statements: Alice says 'Bob and Carol are different types (one is a knight, the other is a knave).' Bob says 'Alice is a knave.' Carol says 'Dan is a knave.' Dan says 'Carol is a knave.' How many knaves are among the four people? Answer with just the integer.";
107
- readonly expected: "2";
108
- readonly matchMode: "exact";
109
- readonly maxTokens: 768;
110
- }, {
111
- readonly id: "sonnet-killer-hofstadter";
112
- readonly category: "sonnet-killer:recursive-sequence";
113
- readonly prompt: "A sequence is defined on the positive integers by f(1) = 1 and, for every n > 1, f(n) = f(n - f(n-1)) + 1. Compute f(10). Answer with just the integer.";
114
- readonly expected: "4";
115
- readonly matchMode: "exact";
116
- readonly maxTokens: 768;
117
- }, {
118
- readonly id: "sonnet-killer-modexp";
119
- readonly category: "sonnet-killer:number-theory";
120
- readonly prompt: "What are the last two digits of 7 raised to the 2026th power? Answer with exactly the two-digit number (e.g. '07' if it's seven, '49' if it's forty-nine).";
121
- readonly expected: "49";
122
- readonly matchMode: "exact";
123
- readonly maxTokens: 512;
124
- }];
125
- };
126
- //# sourceMappingURL=capability-tasks.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"capability-tasks.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/capability-tasks.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8I3B,CAAC"}
@@ -1,147 +0,0 @@
1
- // Auto-generated from src/benchmarks/capability-tasks.json — keep in sync.
2
- // This module exists so the fixture is bundled into dist/ by tsc (JSON files
3
- // are not copied by tsc, and the published CLI ships only dist/).
4
- export const BUILTIN_CAPABILITY_TASKS = {
5
- "version": "1.3",
6
- "description": "Text-only agent capability benchmark — verifiable multi-step reasoning tasks scoreable without tool use. Format inspired by GAIA / SWE-bench / GSM8K. The fixture mixes EASY (regression-floor) and HARD (model-gradient) questions so the pass rate has signal across Haiku → Sonnet → Opus. Real GAIA (web browsing, attachments, HF dataset) remains future work.",
7
- "answerFormat": "Each question requires the model to reply with the answer wrapped in <answer>...</answer> tags. The harness extracts the tag contents and checks against `expected` per `matchMode`. Per-task `maxTokens` overrides the default cap.",
8
- "tasks": [
9
- {
10
- "id": "math-prime",
11
- "category": "easy:reasoning",
12
- "prompt": "What is the smallest 3-digit prime number that does not contain the digit 7?",
13
- "expected": "101",
14
- "matchMode": "exact",
15
- "maxTokens": 192
16
- },
17
- {
18
- "id": "logic-syllogism",
19
- "category": "easy:reasoning",
20
- "prompt": "All routers in tier 1 cost less than $0.001 per call. The Booster router is in tier 1. The Sonnet router costs $0.003 per call. Is the Sonnet router in tier 1? Answer with just \"yes\" or \"no\".",
21
- "expected": "no",
22
- "matchMode": "exact",
23
- "maxTokens": 160
24
- },
25
- {
26
- "id": "regex-match",
27
- "category": "easy:code-reasoning",
28
- "prompt": "Given the regex /^([a-z]+)-(\\d+)$/ and the input string 'pattern-1779526376', what is the value of capture group 2?",
29
- "expected": "1779526376",
30
- "matchMode": "exact",
31
- "maxTokens": 192
32
- },
33
- {
34
- "id": "gsm8k-trip",
35
- "category": "hard:gsm8k-style",
36
- "prompt": "A delivery van starts a route with 240 packages. At stop A it drops off 1/4 of its current load and picks up 6 new packages. At stop B it drops off 1/3 of its current load and picks up 4 new packages. At stop C it drops off half of its current load. How many packages does the van have after stop C? Answer with the integer.",
37
- "expected": "64",
38
- "matchMode": "exact",
39
- "maxTokens": 256
40
- },
41
- {
42
- "id": "gsm8k-discount",
43
- "category": "hard:gsm8k-style",
44
- "prompt": "A store sells 3 widgets and 2 sprockets for $23. It also sells 2 widgets and 4 sprockets for $26. What is the price of one widget? Answer with the integer dollar amount only.",
45
- "expected": "5",
46
- "matchMode": "exact",
47
- "maxTokens": 256
48
- },
49
- {
50
- "id": "code-trace",
51
- "category": "hard:code-trace",
52
- "prompt": "Consider this JavaScript code:\n```\nconst counts = new Map();\nfor (const c of 'abracadabra') {\n counts.set(c, (counts.get(c) ?? 0) + 1);\n}\nlet maxK = '', maxV = 0;\nfor (const [k, v] of counts) {\n if (v > maxV || (v === maxV && k < maxK)) { maxK = k; maxV = v; }\n}\nconsole.log(`${maxK}:${maxV}`);\n```\nWhat does it print? Answer with just the printed string.",
53
- "expected": "a:5",
54
- "matchMode": "exact",
55
- "maxTokens": 192
56
- },
57
- {
58
- "id": "hard-graph-shortest",
59
- "category": "hard:graph-reasoning",
60
- "prompt": "A directed graph has these weighted edges: A→B(3), A→C(7), B→C(2), B→D(5), C→D(1), C→E(4), D→E(2). What is the cost of the shortest path from A to E? Answer with the integer.",
61
- "expected": "8",
62
- "matchMode": "exact",
63
- "maxTokens": 192
64
- },
65
- {
66
- "id": "hard-probability",
67
- "category": "hard:probability",
68
- "prompt": "A bag contains 5 red, 3 blue, and 2 green balls. Two balls are drawn without replacement. What is the probability that both balls are the same color? Express as a fraction in lowest terms in the form a/b (e.g. 3/10).",
69
- "expected": "14/45",
70
- "matchMode": "exact",
71
- "maxTokens": 256
72
- },
73
- {
74
- "id": "expert-marble-inverse",
75
- "category": "expert:inverse-arithmetic",
76
- "prompt": "A bag of marbles is split as follows: 40% are given to Alice, then 25% of the REMAINING marbles are given to Bob, then half of the remaining marbles (after Alice and Bob) are given to Carol. Carol receives exactly 18 marbles. How many marbles were in the bag originally? Answer with the integer.",
77
- "expected": "80",
78
- "matchMode": "exact",
79
- "maxTokens": 512
80
- },
81
- {
82
- "id": "expert-crt",
83
- "category": "expert:number-theory",
84
- "prompt": "Find the smallest positive integer n such that all three of these hold simultaneously: n mod 7 = 3, n mod 9 = 4, n mod 11 = 5. Answer with just the integer.",
85
- "expected": "346",
86
- "matchMode": "exact",
87
- "maxTokens": 768
88
- },
89
- {
90
- "id": "expert-bayes",
91
- "category": "expert:bayesian",
92
- "prompt": "A medical test has 95% sensitivity (true positive rate) and 90% specificity (true negative rate). In the screened population, 1% of people have the disease. A patient tests positive. What is the probability the patient actually has the disease, rounded to the nearest whole percent? Answer with just the integer percentage (no '%' sign).",
93
- "expected": "9",
94
- "matchMode": "exact",
95
- "maxTokens": 512
96
- },
97
- {
98
- "id": "expert-banana",
99
- "category": "expert:combinatorics",
100
- "prompt": "In how many distinct arrangements of the letters of the word BANANA do no two N's appear next to each other? Answer with the integer.",
101
- "expected": "40",
102
- "matchMode": "exact",
103
- "maxTokens": 512
104
- },
105
- {
106
- "id": "expert-rectangle",
107
- "category": "expert:diophantine",
108
- "prompt": "A rectangle has positive integer side lengths. Its perimeter (in linear units) is numerically equal to its area (in square units). Two rectangles that are rotations of each other (e.g. 3x6 and 6x3) count as the same rectangle. What is the sum of the areas of all distinct such rectangles? Answer with the integer.",
109
- "expected": "34",
110
- "matchMode": "exact",
111
- "maxTokens": 512
112
- },
113
- {
114
- "id": "expert-dice-reroll",
115
- "category": "expert:expected-value",
116
- "prompt": "You roll a fair 6-sided die. If the result is a 6, you reroll exactly once and take the new result. Otherwise, you keep the original result. What is the expected value of your final number? Express the answer as a fraction in lowest terms in the form a/b (no spaces, no surrounding text).",
117
- "expected": "37/12",
118
- "matchMode": "exact",
119
- "maxTokens": 512
120
- },
121
- {
122
- "id": "sonnet-killer-knights",
123
- "category": "sonnet-killer:logic-puzzle",
124
- "prompt": "On an island, knights always tell the truth and knaves always lie. You meet four people named Alice, Bob, Carol, and Dan. They make the following statements: Alice says 'Bob and Carol are different types (one is a knight, the other is a knave).' Bob says 'Alice is a knave.' Carol says 'Dan is a knave.' Dan says 'Carol is a knave.' How many knaves are among the four people? Answer with just the integer.",
125
- "expected": "2",
126
- "matchMode": "exact",
127
- "maxTokens": 768
128
- },
129
- {
130
- "id": "sonnet-killer-hofstadter",
131
- "category": "sonnet-killer:recursive-sequence",
132
- "prompt": "A sequence is defined on the positive integers by f(1) = 1 and, for every n > 1, f(n) = f(n - f(n-1)) + 1. Compute f(10). Answer with just the integer.",
133
- "expected": "4",
134
- "matchMode": "exact",
135
- "maxTokens": 768
136
- },
137
- {
138
- "id": "sonnet-killer-modexp",
139
- "category": "sonnet-killer:number-theory",
140
- "prompt": "What are the last two digits of 7 raised to the 2026th power? Answer with exactly the two-digit number (e.g. '07' if it's seven, '49' if it's forty-nine).",
141
- "expected": "49",
142
- "matchMode": "exact",
143
- "maxTokens": 512
144
- }
145
- ]
146
- };
147
- //# sourceMappingURL=capability-tasks.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"capability-tasks.js","sourceRoot":"","sources":["../../../src/benchmarks/capability-tasks.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,6EAA6E;AAC7E,kEAAkE;AAElE,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,SAAS,EAAE,KAAK;IAChB,aAAa,EAAE,sWAAsW;IACrX,cAAc,EAAE,sOAAsO;IACtP,OAAO,EAAE;QACP;YACE,IAAI,EAAE,YAAY;YAClB,UAAU,EAAE,gBAAgB;YAC5B,QAAQ,EAAE,8EAA8E;YACxF,UAAU,EAAE,KAAK;YACjB,WAAW,EAAE,OAAO;YACpB,WAAW,EAAE,GAAG;SACjB;QACD;YACE,IAAI,EAAE,iBAAiB;YACvB,UAAU,EAAE,gBAAgB;YAC5B,QAAQ,EAAE,qMAAqM;YAC/M,UAAU,EAAE,IAAI;YAChB,WAAW,EAAE,OAAO;YACpB,WAAW,EAAE,GAAG;SACjB;QACD;YACE,IAAI,EAAE,aAAa;YACnB,UAAU,EAAE,qBAAqB;YACjC,QAAQ,EAAE,sHAAsH;YAChI,UAAU,EAAE,YAAY;YACxB,WAAW,EAAE,OAAO;YACpB,WAAW,EAAE,GAAG;SACjB;QACD;YACE,IAAI,EAAE,YAAY;YAClB,UAAU,EAAE,kBAAkB;YAC9B,QAAQ,EAAE,sUAAsU;YAChV,UAAU,EAAE,IAAI;YAChB,WAAW,EAAE,OAAO;YACpB,WAAW,EAAE,GAAG;SACjB;QACD;YACE,IAAI,EAAE,gBAAgB;YACtB,UAAU,EAAE,kBAAkB;YAC9B,QAAQ,EAAE,gLAAgL;YAC1L,UAAU,EAAE,GAAG;YACf,WAAW,EAAE,OAAO;YACpB,WAAW,EAAE,GAAG;SACjB;QACD;YACE,IAAI,EAAE,YAAY;YAClB,UAAU,EAAE,iBAAiB;YAC7B,QAAQ,EAAE,mXAAmX;YAC7X,UAAU,EAAE,KAAK;YACjB,WAAW,EAAE,OAAO;YACpB,WAAW,EAAE,GAAG;SACjB;QACD;YACE,IAAI,EAAE,qBAAqB;YAC3B,UAAU,EAAE,sBAAsB;YAClC,QAAQ,EAAE,gLAAgL;YAC1L,UAAU,EAAE,GAAG;YACf,WAAW,EAAE,OAAO;YACpB,WAAW,EAAE,GAAG;SACjB;QACD;YACE,IAAI,EAAE,kBAAkB;YACxB,UAAU,EAAE,kBAAkB;YAC9B,QAAQ,EAAE,0NAA0N;YACpO,UAAU,EAAE,OAAO;YACnB,WAAW,EAAE,OAAO;YACpB,WAAW,EAAE,GAAG;SACjB;QACD;YACE,IAAI,EAAE,uBAAuB;YAC7B,UAAU,EAAE,2BAA2B;YACvC,QAAQ,EAAE,ySAAyS;YACnT,UAAU,EAAE,IAAI;YAChB,WAAW,EAAE,OAAO;YACpB,WAAW,EAAE,GAAG;SACjB;QACD;YACE,IAAI,EAAE,YAAY;YAClB,UAAU,EAAE,sBAAsB;YAClC,QAAQ,EAAE,8JAA8J;YACxK,UAAU,EAAE,KAAK;YACjB,WAAW,EAAE,OAAO;YACpB,WAAW,EAAE,GAAG;SACjB;QACD;YACE,IAAI,EAAE,cAAc;YACpB,UAAU,EAAE,iBAAiB;YAC7B,QAAQ,EAAE,mVAAmV;YAC7V,UAAU,EAAE,GAAG;YACf,WAAW,EAAE,OAAO;YACpB,WAAW,EAAE,GAAG;SACjB;QACD;YACE,IAAI,EAAE,eAAe;YACrB,UAAU,EAAE,sBAAsB;YAClC,QAAQ,EAAE,uIAAuI;YACjJ,UAAU,EAAE,IAAI;YAChB,WAAW,EAAE,OAAO;YACpB,WAAW,EAAE,GAAG;SACjB;QACD;YACE,IAAI,EAAE,kBAAkB;YACxB,UAAU,EAAE,oBAAoB;YAChC,QAAQ,EAAE,2TAA2T;YACrU,UAAU,EAAE,IAAI;YAChB,WAAW,EAAE,OAAO;YACpB,WAAW,EAAE,GAAG;SACjB;QACD;YACE,IAAI,EAAE,oBAAoB;YAC1B,UAAU,EAAE,uBAAuB;YACnC,QAAQ,EAAE,kSAAkS;YAC5S,UAAU,EAAE,OAAO;YACnB,WAAW,EAAE,OAAO;YACpB,WAAW,EAAE,GAAG;SACjB;QACD;YACE,IAAI,EAAE,uBAAuB;YAC7B,UAAU,EAAE,4BAA4B;YACxC,QAAQ,EAAE,uZAAuZ;YACja,UAAU,EAAE,GAAG;YACf,WAAW,EAAE,OAAO;YACpB,WAAW,EAAE,GAAG;SACjB;QACD;YACE,IAAI,EAAE,0BAA0B;YAChC,UAAU,EAAE,kCAAkC;YAC9C,QAAQ,EAAE,yJAAyJ;YACnK,UAAU,EAAE,GAAG;YACf,WAAW,EAAE,OAAO;YACpB,WAAW,EAAE,GAAG;SACjB;QACD;YACE,IAAI,EAAE,sBAAsB;YAC5B,UAAU,EAAE,6BAA6B;YACzC,QAAQ,EAAE,4JAA4J;YACtK,UAAU,EAAE,IAAI;YAChB,WAAW,EAAE,OAAO;YACpB,WAAW,EAAE,GAAG;SACjB;KACF;CACO,CAAC"}
@@ -1,53 +0,0 @@
1
- /**
2
- * GAIA Agent Contrastive Wrapper — iter 49.5
3
- *
4
- * Thin wrapper around runGaiaAgent that adds three ruflo intelligence hooks:
5
- * 1. PRE: memory_search_unified for relevant patterns
6
- * 2. DURING: trajectory recording (start/step/end) via memory store
7
- * 3. POST: memory_store of question+answer+verdict for future recall
8
- *
9
- * Purpose: measure whether ruflo's AgentDB intelligence (5,930 entries,
10
- * 26,490 patterns) provides a measurable lift on GAIA L1 when injected
11
- * as system context before the agent loop.
12
- *
13
- * Design constraints:
14
- * - Does NOT change the underlying agent loop logic
15
- * - Does NOT enable ADR-135 tracks (critic, decomposition, voting, MoE, KG)
16
- * - Pure addition: memory before, trajectory during, store after
17
- * - Graceful fallback: any hook failure is logged but does NOT fail the question
18
- *
19
- * Refs: ADR-133, iter 49.5, #2156
20
- */
21
- import type { GaiaQuestion } from './gaia-loader.js';
22
- import { type GaiaAgentOptions, type GaiaAgentResult } from './gaia-agent.js';
23
- export interface ContrastiveStats {
24
- memorySearchCalls: number;
25
- memorySearchHits: number;
26
- memoryStoreCount: number;
27
- trajectoryCount: number;
28
- totalPatternsInjected: number;
29
- }
30
- export interface ContrastiveAgentResult extends GaiaAgentResult {
31
- /** Number of memory patterns injected as context for this question. */
32
- patternsInjected?: number;
33
- /** Whether a trajectory was recorded for this question. */
34
- trajectoryRecorded?: boolean;
35
- /** Whether the question+answer was stored in memory after completion. */
36
- memoryStored?: boolean;
37
- }
38
- /**
39
- * Wrap runGaiaAgent with ruflo intelligence hooks.
40
- *
41
- * Hook points:
42
- * 1. memory_search before the agent loop (context injection)
43
- * 2. trajectory-start/end records via memory store
44
- * 3. memory_store after the final answer
45
- *
46
- * The underlying agent loop is called unchanged.
47
- */
48
- export declare function runGaiaAgentContrastive(question: GaiaQuestion, options: GaiaAgentOptions, stats: ContrastiveStats): Promise<ContrastiveAgentResult>;
49
- /**
50
- * Create a fresh ContrastiveStats counter object.
51
- */
52
- export declare function createContrastiveStats(): ContrastiveStats;
53
- //# sourceMappingURL=gaia-agent-contrastive.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"gaia-agent-contrastive.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/gaia-agent-contrastive.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAgB,KAAK,gBAAgB,EAAE,KAAK,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAM5F,MAAM,WAAW,gBAAgB;IAC/B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,sBAAuB,SAAQ,eAAe;IAC7D,uEAAuE;IACvE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,2DAA2D;IAC3D,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,yEAAyE;IACzE,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AA2FD;;;;;;;;;GASG;AACH,wBAAsB,uBAAuB,CAC3C,QAAQ,EAAE,YAAY,EACtB,OAAO,EAAE,gBAAgB,EACzB,KAAK,EAAE,gBAAgB,GACtB,OAAO,CAAC,sBAAsB,CAAC,CA+DjC;AAED;;GAEG;AACH,wBAAgB,sBAAsB,IAAI,gBAAgB,CAQzD"}
@@ -1,177 +0,0 @@
1
- /**
2
- * GAIA Agent Contrastive Wrapper — iter 49.5
3
- *
4
- * Thin wrapper around runGaiaAgent that adds three ruflo intelligence hooks:
5
- * 1. PRE: memory_search_unified for relevant patterns
6
- * 2. DURING: trajectory recording (start/step/end) via memory store
7
- * 3. POST: memory_store of question+answer+verdict for future recall
8
- *
9
- * Purpose: measure whether ruflo's AgentDB intelligence (5,930 entries,
10
- * 26,490 patterns) provides a measurable lift on GAIA L1 when injected
11
- * as system context before the agent loop.
12
- *
13
- * Design constraints:
14
- * - Does NOT change the underlying agent loop logic
15
- * - Does NOT enable ADR-135 tracks (critic, decomposition, voting, MoE, KG)
16
- * - Pure addition: memory before, trajectory during, store after
17
- * - Graceful fallback: any hook failure is logged but does NOT fail the question
18
- *
19
- * Refs: ADR-133, iter 49.5, #2156
20
- */
21
- import { execSync } from 'node:child_process';
22
- import { runGaiaAgent } from './gaia-agent.js';
23
- // ---------------------------------------------------------------------------
24
- // Ruflo CLI helpers (best-effort — never throw)
25
- // ---------------------------------------------------------------------------
26
- const CLI_TIMEOUT_MS = 10_000;
27
- const GAIA_NAMESPACE = 'gaia-l1-questions';
28
- /**
29
- * Run memory search via CLI and return up to `limit` result summaries.
30
- * Returns empty array on any error (graceful fallback).
31
- */
32
- function memorySearch(query, limit) {
33
- try {
34
- const raw = execSync(`npx @claude-flow/cli@latest memory search --query ${JSON.stringify(query)} --limit ${limit}`, { encoding: 'utf-8', timeout: CLI_TIMEOUT_MS, stdio: ['pipe', 'pipe', 'pipe'] });
35
- // Parse table rows: extract Preview column (5th column in pipe-separated table)
36
- const rows = raw
37
- .split('\n')
38
- .filter((line) => line.startsWith('|') && !line.includes('Key') && !line.includes('---'))
39
- .map((line) => {
40
- const cols = line.split('|').map((c) => c.trim()).filter(Boolean);
41
- // cols: [Key, Score, Namespace, Preview]
42
- return cols[3] ?? '';
43
- })
44
- .filter((preview) => preview.length > 0 && !preview.includes('Preview'));
45
- return rows.slice(0, limit);
46
- }
47
- catch {
48
- return [];
49
- }
50
- }
51
- /**
52
- * Store a key/value in memory under the gaia namespace.
53
- * Best-effort — swallows errors.
54
- */
55
- function memoryStore(key, value) {
56
- try {
57
- const valueStr = JSON.stringify(value);
58
- execSync(`npx @claude-flow/cli@latest memory store --key ${JSON.stringify(key)} --value ${JSON.stringify(valueStr)} --namespace ${GAIA_NAMESPACE}`, { encoding: 'utf-8', timeout: CLI_TIMEOUT_MS, stdio: ['pipe', 'pipe', 'pipe'] });
59
- return true;
60
- }
61
- catch {
62
- return false;
63
- }
64
- }
65
- /**
66
- * Record a trajectory event (start/step/end) via memory store.
67
- * This is a lightweight proxy — the MCP trajectory tools are not available
68
- * from inside the benchmark harness. We persist the trajectory as memory
69
- * entries so they can be recalled later.
70
- */
71
- function trajectoryRecord(type, data) {
72
- try {
73
- const key = `traj-gaia-l1-${type}-${Date.now()}`;
74
- execSync(`npx @claude-flow/cli@latest memory store --key ${JSON.stringify(key)} --value ${JSON.stringify(JSON.stringify(data))} --namespace trajectories`, { encoding: 'utf-8', timeout: CLI_TIMEOUT_MS, stdio: ['pipe', 'pipe', 'pipe'] });
75
- }
76
- catch {
77
- // Best-effort — trajectory recording never blocks progress
78
- }
79
- }
80
- // ---------------------------------------------------------------------------
81
- // System prompt injection
82
- // ---------------------------------------------------------------------------
83
- /**
84
- * Build a context prefix to prepend to the system prompt when patterns
85
- * are found in memory. Returns empty string when patterns is empty.
86
- */
87
- function buildMemoryContextPrefix(patterns) {
88
- if (patterns.length === 0)
89
- return '';
90
- return ('Relevant patterns from prior ruflo work:\n' +
91
- patterns.map((p) => `- ${p.slice(0, 120)}`).join('\n') +
92
- '\n\n');
93
- }
94
- // ---------------------------------------------------------------------------
95
- // Contrastive wrapper
96
- // ---------------------------------------------------------------------------
97
- /**
98
- * Wrap runGaiaAgent with ruflo intelligence hooks.
99
- *
100
- * Hook points:
101
- * 1. memory_search before the agent loop (context injection)
102
- * 2. trajectory-start/end records via memory store
103
- * 3. memory_store after the final answer
104
- *
105
- * The underlying agent loop is called unchanged.
106
- */
107
- export async function runGaiaAgentContrastive(question, options, stats) {
108
- const questionId = question.task_id;
109
- // ---- 1. PRE: memory_search for relevant patterns ----------------------
110
- stats.memorySearchCalls++;
111
- const patterns = memorySearch(question.question, 3);
112
- const patternsInjected = patterns.length;
113
- if (patterns.length > 0) {
114
- stats.memorySearchHits++;
115
- stats.totalPatternsInjected += patterns.length;
116
- }
117
- // ---- 2. TRAJECTORY: start recording -----------------------------------
118
- trajectoryRecord('start', {
119
- task: 'gaia-l1',
120
- question_id: questionId,
121
- question_snippet: question.question.slice(0, 80),
122
- startedAt: new Date().toISOString(),
123
- patternsInjected,
124
- });
125
- // ---- Build patched options with memory context injected ---------------
126
- // We inject context by wrapping the question text (system prompt cannot be
127
- // overridden via GaiaAgentOptions today; instead we prepend context to
128
- // the question text as a user-visible hint). This is intentionally minimal.
129
- const contextPrefix = buildMemoryContextPrefix(patterns);
130
- const patchedQuestion = contextPrefix
131
- ? { ...question, question: contextPrefix + question.question }
132
- : question;
133
- // ---- Run the actual agent loop (unchanged) ----------------------------
134
- const result = await runGaiaAgent(patchedQuestion, options);
135
- // ---- 3. POST: store question+answer+verdict ---------------------------
136
- const questionKey = `gaia-l1-q${questionId}-${Date.now()}`;
137
- const stored = memoryStore(questionKey, {
138
- question: question.question,
139
- answer: result.finalAnswer,
140
- questionId,
141
- model: options.model ?? 'claude-sonnet-4-6',
142
- turns: result.turns,
143
- timedOut: result.timedOut ?? false,
144
- storedAt: new Date().toISOString(),
145
- });
146
- if (stored)
147
- stats.memoryStoreCount++;
148
- // ---- TRAJECTORY: end recording ----------------------------------------
149
- trajectoryRecord('end', {
150
- task: 'gaia-l1',
151
- question_id: questionId,
152
- answer: result.finalAnswer,
153
- turns: result.turns,
154
- timedOut: result.timedOut ?? false,
155
- endedAt: new Date().toISOString(),
156
- });
157
- stats.trajectoryCount++;
158
- return {
159
- ...result,
160
- patternsInjected,
161
- trajectoryRecorded: true,
162
- memoryStored: stored,
163
- };
164
- }
165
- /**
166
- * Create a fresh ContrastiveStats counter object.
167
- */
168
- export function createContrastiveStats() {
169
- return {
170
- memorySearchCalls: 0,
171
- memorySearchHits: 0,
172
- memoryStoreCount: 0,
173
- trajectoryCount: 0,
174
- totalPatternsInjected: 0,
175
- };
176
- }
177
- //# sourceMappingURL=gaia-agent-contrastive.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"gaia-agent-contrastive.js","sourceRoot":"","sources":["../../../src/benchmarks/gaia-agent-contrastive.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAE9C,OAAO,EAAE,YAAY,EAA+C,MAAM,iBAAiB,CAAC;AAuB5F,8EAA8E;AAC9E,gDAAgD;AAChD,8EAA8E;AAE9E,MAAM,cAAc,GAAG,MAAM,CAAC;AAC9B,MAAM,cAAc,GAAG,mBAAmB,CAAC;AAE3C;;;GAGG;AACH,SAAS,YAAY,CAAC,KAAa,EAAE,KAAa;IAChD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,QAAQ,CAClB,qDAAqD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,YAAY,KAAK,EAAE,EAC7F,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAChF,CAAC;QACF,gFAAgF;QAChF,MAAM,IAAI,GAAG,GAAG;aACb,KAAK,CAAC,IAAI,CAAC;aACX,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;aACxF,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACZ,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAClE,yCAAyC;YACzC,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACvB,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;QAC3E,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,WAAW,CAAC,GAAW,EAAE,KAAa;IAC7C,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACvC,QAAQ,CACN,kDAAkD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,gBAAgB,cAAc,EAAE,EACzI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAChF,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,SAAS,gBAAgB,CAAC,IAAqB,EAAE,IAAY;IAC3D,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,gBAAgB,IAAI,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QACjD,QAAQ,CACN,kDAAkD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,2BAA2B,EAChJ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAChF,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,2DAA2D;IAC7D,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,0BAA0B;AAC1B,8EAA8E;AAE9E;;;GAGG;AACH,SAAS,wBAAwB,CAAC,QAAkB;IAClD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACrC,OAAO,CACL,4CAA4C;QAC5C,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QACtD,MAAM,CACP,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,QAAsB,EACtB,OAAyB,EACzB,KAAuB;IAEvB,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC;IAEpC,0EAA0E;IAC1E,KAAK,CAAC,iBAAiB,EAAE,CAAC;IAC1B,MAAM,QAAQ,GAAG,YAAY,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IACpD,MAAM,gBAAgB,GAAG,QAAQ,CAAC,MAAM,CAAC;IACzC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,KAAK,CAAC,gBAAgB,EAAE,CAAC;QACzB,KAAK,CAAC,qBAAqB,IAAI,QAAQ,CAAC,MAAM,CAAC;IACjD,CAAC;IAED,0EAA0E;IAC1E,gBAAgB,CAAC,OAAO,EAAE;QACxB,IAAI,EAAE,SAAS;QACf,WAAW,EAAE,UAAU;QACvB,gBAAgB,EAAE,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;QAChD,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,gBAAgB;KACjB,CAAC,CAAC;IAEH,0EAA0E;IAC1E,2EAA2E;IAC3E,uEAAuE;IACvE,4EAA4E;IAC5E,MAAM,aAAa,GAAG,wBAAwB,CAAC,QAAQ,CAAC,CAAC;IACzD,MAAM,eAAe,GAAiB,aAAa;QACjD,CAAC,CAAC,EAAE,GAAG,QAAQ,EAAE,QAAQ,EAAE,aAAa,GAAG,QAAQ,CAAC,QAAQ,EAAE;QAC9D,CAAC,CAAC,QAAQ,CAAC;IAEb,0EAA0E;IAC1E,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;IAE5D,0EAA0E;IAC1E,MAAM,WAAW,GAAG,YAAY,UAAU,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAC3D,MAAM,MAAM,GAAG,WAAW,CAAC,WAAW,EAAE;QACtC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,MAAM,EAAE,MAAM,CAAC,WAAW;QAC1B,UAAU;QACV,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,mBAAmB;QAC3C,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,KAAK;QAClC,QAAQ,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACnC,CAAC,CAAC;IACH,IAAI,MAAM;QAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC;IAErC,0EAA0E;IAC1E,gBAAgB,CAAC,KAAK,EAAE;QACtB,IAAI,EAAE,SAAS;QACf,WAAW,EAAE,UAAU;QACvB,MAAM,EAAE,MAAM,CAAC,WAAW;QAC1B,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,KAAK;QAClC,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KAClC,CAAC,CAAC;IACH,KAAK,CAAC,eAAe,EAAE,CAAC;IAExB,OAAO;QACL,GAAG,MAAM;QACT,gBAAgB;QAChB,kBAAkB,EAAE,IAAI;QACxB,YAAY,EAAE,MAAM;KACrB,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,sBAAsB;IACpC,OAAO;QACL,iBAAiB,EAAE,CAAC;QACpB,gBAAgB,EAAE,CAAC;QACnB,gBAAgB,EAAE,CAAC;QACnB,eAAe,EAAE,CAAC;QAClB,qBAAqB,EAAE,CAAC;KACzB,CAAC;AACJ,CAAC"}
@@ -1,51 +0,0 @@
1
- /**
2
- * GAIA Agent — Gemini 2.5 Pro Thinking adapter (Phase 2 pilot)
3
- *
4
- * Adapts the existing GAIA tool catalogue to Google's generateContent API,
5
- * translating Anthropic tool_use ↔ Gemini functionCall/functionResponse.
6
- *
7
- * Loop algorithm mirrors gaia-agent.ts:
8
- * 1. Build initial contents array with system instruction + first user turn.
9
- * 2. Call Gemini generateContent with functionDeclarations.
10
- * 3. On functionCall parts: execute tools, append functionResponse, repeat.
11
- * 4. On text-only response: scan for FINAL_ANSWER pattern.
12
- * 5. On maxTurns: return timedOut result.
13
- *
14
- * Gemini 2.5 Pro pricing (2026-05-28, per million tokens):
15
- * Input: $1.25 (≤200k tokens), $2.50 (>200k)
16
- * Output: $10.00 (≤200k tokens), $15.00 (>200k)
17
- * Thinking tokens are billed as output tokens.
18
- *
19
- * Cost gate for pilot: ≤$0.12 per question average.
20
- * Expected per-question cost at ~10k input + ~2k output: ~$0.032 — well within gate.
21
- *
22
- * Refs: ADR-133, ADR-135, #2156 Phase 2
23
- */
24
- import { GaiaQuestion } from './gaia-loader.js';
25
- import { GaiaToolCatalogue } from './gaia-tools/index.js';
26
- export declare const DEFAULT_GEMINI_MODEL = "gemini-2.5-pro";
27
- export interface GeminiAgentResult {
28
- questionId: string;
29
- finalAnswer: string | null;
30
- turns: number;
31
- toolCallsByName: Record<string, number>;
32
- totalInputTokens: number;
33
- totalOutputTokens: number;
34
- totalThinkingTokens: number;
35
- wallMs: number;
36
- estimatedCostUsd: number;
37
- timedOut?: boolean;
38
- error?: string;
39
- }
40
- export interface GeminiAgentOptions {
41
- model?: string;
42
- maxTurns?: number;
43
- maxTokensPerTurn?: number;
44
- perTurnTimeoutMs?: number;
45
- apiKey?: string;
46
- catalogue?: GaiaToolCatalogue;
47
- }
48
- export declare function resolveGeminiApiKey(apiKey?: string): string;
49
- export declare function runGeminiAgent(question: GaiaQuestion, options?: GeminiAgentOptions): Promise<GeminiAgentResult>;
50
- export declare function isGeminiAnswerCorrect(modelAnswer: string, expected: string): boolean;
51
- //# sourceMappingURL=gaia-agent-gemini.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"gaia-agent-gemini.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/gaia-agent-gemini.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAKH,OAAO,EACL,YAAY,EACb,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAEL,iBAAiB,EAElB,MAAM,uBAAuB,CAAC;AAO/B,eAAO,MAAM,oBAAoB,mBAAmB,CAAC;AAerD,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAMD,wBAAgB,mBAAmB,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAmB3D;AAiND,wBAAsB,cAAc,CAClC,QAAQ,EAAE,YAAY,EACtB,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,iBAAiB,CAAC,CA0J5B;AAaD,wBAAgB,qBAAqB,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAgBpF"}