anvil-engineering-workbench 1.0.8 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (162) hide show
  1. package/CHANGELOG.md +37 -0
  2. package/README.md +23 -18
  3. package/dist/ai/claude-code-provider.js +55 -9
  4. package/dist/ai/claude-code-provider.js.map +1 -1
  5. package/dist/ai/provider.js +1 -1
  6. package/dist/ai/provider.js.map +1 -1
  7. package/dist/cli.js +837 -278
  8. package/dist/cli.js.map +1 -1
  9. package/dist/core/adr-engine.js +9 -11
  10. package/dist/core/adr-engine.js.map +1 -1
  11. package/dist/core/agent-config-release.js +29 -13
  12. package/dist/core/agent-config-release.js.map +1 -1
  13. package/dist/core/agent-config-versioning.js +32 -17
  14. package/dist/core/agent-config-versioning.js.map +1 -1
  15. package/dist/core/agent-evaluation-intelligence.js +30 -38
  16. package/dist/core/agent-evaluation-intelligence.js.map +1 -1
  17. package/dist/core/agent-evolution.js +109 -20
  18. package/dist/core/agent-evolution.js.map +1 -1
  19. package/dist/core/agent-experiment-studio.js +19 -11
  20. package/dist/core/agent-experiment-studio.js.map +1 -1
  21. package/dist/core/agent-optimization.js +69 -14
  22. package/dist/core/agent-optimization.js.map +1 -1
  23. package/dist/core/agent-registry.js +87 -15
  24. package/dist/core/agent-registry.js.map +1 -1
  25. package/dist/core/agent-security-audit.js +21 -21
  26. package/dist/core/agent-security-audit.js.map +1 -1
  27. package/dist/core/agent-workspace-context.js +37 -6
  28. package/dist/core/agent-workspace-context.js.map +1 -1
  29. package/dist/core/ai-code-review.js +60 -12
  30. package/dist/core/ai-code-review.js.map +1 -1
  31. package/dist/core/anthropic-provider.js.map +1 -1
  32. package/dist/core/architecture-map.js +87 -24
  33. package/dist/core/architecture-map.js.map +1 -1
  34. package/dist/core/benchmark-engine.js +89 -21
  35. package/dist/core/benchmark-engine.js.map +1 -1
  36. package/dist/core/benchmark-matrix.js +215 -41
  37. package/dist/core/benchmark-matrix.js.map +1 -1
  38. package/dist/core/benchmark-regression.js +124 -18
  39. package/dist/core/benchmark-regression.js.map +1 -1
  40. package/dist/core/change-intelligence.js +102 -43
  41. package/dist/core/change-intelligence.js.map +1 -1
  42. package/dist/core/code-intelligence.js +40 -13
  43. package/dist/core/code-intelligence.js.map +1 -1
  44. package/dist/core/code-review.js +145 -17
  45. package/dist/core/code-review.js.map +1 -1
  46. package/dist/core/code-search.js +79 -39
  47. package/dist/core/code-search.js.map +1 -1
  48. package/dist/core/container-sandbox.js +87 -23
  49. package/dist/core/container-sandbox.js.map +1 -1
  50. package/dist/core/dashboard-server.js +379 -93
  51. package/dist/core/dashboard-server.js.map +1 -1
  52. package/dist/core/dependency-graph.js +75 -13
  53. package/dist/core/dependency-graph.js.map +1 -1
  54. package/dist/core/dependency-impact.js +132 -33
  55. package/dist/core/dependency-impact.js.map +1 -1
  56. package/dist/core/dependency-intelligence.js +153 -56
  57. package/dist/core/dependency-intelligence.js.map +1 -1
  58. package/dist/core/doctor.js +3 -3
  59. package/dist/core/doctor.js.map +1 -1
  60. package/dist/core/engineering-risk.js +75 -40
  61. package/dist/core/engineering-risk.js.map +1 -1
  62. package/dist/core/env-compat.js.map +1 -1
  63. package/dist/core/evaluation-engine.js +188 -42
  64. package/dist/core/evaluation-engine.js.map +1 -1
  65. package/dist/core/execution-trace.js +83 -29
  66. package/dist/core/execution-trace.js.map +1 -1
  67. package/dist/core/experiment-history.js +70 -20
  68. package/dist/core/experiment-history.js.map +1 -1
  69. package/dist/core/experiment-manifest.js +92 -13
  70. package/dist/core/experiment-manifest.js.map +1 -1
  71. package/dist/core/gemini-provider.js.map +1 -1
  72. package/dist/core/git-intelligence.js +68 -16
  73. package/dist/core/git-intelligence.js.map +1 -1
  74. package/dist/core/hook-engine.js +7 -3
  75. package/dist/core/hook-engine.js.map +1 -1
  76. package/dist/core/hosted-agent-provider.js +48 -8
  77. package/dist/core/hosted-agent-provider.js.map +1 -1
  78. package/dist/core/intelligence-timeline.js +92 -43
  79. package/dist/core/intelligence-timeline.js.map +1 -1
  80. package/dist/core/mcp-setup.js +12 -12
  81. package/dist/core/mcp-setup.js.map +1 -1
  82. package/dist/core/model-router.js +153 -37
  83. package/dist/core/model-router.js.map +1 -1
  84. package/dist/core/openai-compatible-provider.js +41 -10
  85. package/dist/core/openai-compatible-provider.js.map +1 -1
  86. package/dist/core/orchestrator.js +28 -23
  87. package/dist/core/orchestrator.js.map +1 -1
  88. package/dist/core/patch-application.js +88 -24
  89. package/dist/core/patch-application.js.map +1 -1
  90. package/dist/core/patch-approval.js +43 -15
  91. package/dist/core/patch-approval.js.map +1 -1
  92. package/dist/core/patch-validation.js +93 -15
  93. package/dist/core/patch-validation.js.map +1 -1
  94. package/dist/core/permission-policy.js +8 -5
  95. package/dist/core/permission-policy.js.map +1 -1
  96. package/dist/core/policy-engine.d.ts +76 -0
  97. package/dist/core/policy-engine.js +352 -0
  98. package/dist/core/policy-engine.js.map +1 -0
  99. package/dist/core/project-cleaner.js +143 -129
  100. package/dist/core/project-cleaner.js.map +1 -1
  101. package/dist/core/project-health.js +183 -21
  102. package/dist/core/project-health.js.map +1 -1
  103. package/dist/core/project-init.js +32 -20
  104. package/dist/core/project-init.js.map +1 -1
  105. package/dist/core/project-inspector.js +81 -32
  106. package/dist/core/project-inspector.js.map +1 -1
  107. package/dist/core/project-memory.js +41 -18
  108. package/dist/core/project-memory.js.map +1 -1
  109. package/dist/core/provider-diagnostics.js +65 -12
  110. package/dist/core/provider-diagnostics.js.map +1 -1
  111. package/dist/core/quality-audit.js +167 -28
  112. package/dist/core/quality-audit.js.map +1 -1
  113. package/dist/core/regression-intelligence.js +88 -8
  114. package/dist/core/regression-intelligence.js.map +1 -1
  115. package/dist/core/remediation-planner.js +112 -40
  116. package/dist/core/remediation-planner.js.map +1 -1
  117. package/dist/core/review-history.js +39 -7
  118. package/dist/core/review-history.js.map +1 -1
  119. package/dist/core/run-history-output.js.map +1 -1
  120. package/dist/core/run-replay.js +12 -10
  121. package/dist/core/run-replay.js.map +1 -1
  122. package/dist/core/run-store.js +48 -21
  123. package/dist/core/run-store.js.map +1 -1
  124. package/dist/core/sandbox-cleanup-audit.js +1 -1
  125. package/dist/core/sandbox-cleanup-audit.js.map +1 -1
  126. package/dist/core/sandbox-compliance.js +53 -9
  127. package/dist/core/sandbox-compliance.js.map +1 -1
  128. package/dist/core/sandbox-lifecycle.js +1 -1
  129. package/dist/core/sandbox-lifecycle.js.map +1 -1
  130. package/dist/core/sandbox-policy.js +27 -3
  131. package/dist/core/sandbox-policy.js.map +1 -1
  132. package/dist/core/sandbox-runtime.js +9 -5
  133. package/dist/core/sandbox-runtime.js.map +1 -1
  134. package/dist/core/sandbox-security-profiles.js +31 -12
  135. package/dist/core/sandbox-security-profiles.js.map +1 -1
  136. package/dist/core/sarif-formatter.d.ts +10 -3
  137. package/dist/core/sarif-formatter.js +172 -57
  138. package/dist/core/sarif-formatter.js.map +1 -1
  139. package/dist/core/security-scanner.js +40 -32
  140. package/dist/core/security-scanner.js.map +1 -1
  141. package/dist/core/skill-registry.js +83 -21
  142. package/dist/core/skill-registry.js.map +1 -1
  143. package/dist/core/staged-verification.js +25 -10
  144. package/dist/core/staged-verification.js.map +1 -1
  145. package/dist/core/verification-history.js +12 -7
  146. package/dist/core/verification-history.js.map +1 -1
  147. package/dist/core/verification.js +50 -19
  148. package/dist/core/verification.js.map +1 -1
  149. package/dist/core/workflow-store.js +16 -4
  150. package/dist/core/workflow-store.js.map +1 -1
  151. package/dist/core/workspace-browser.js +57 -10
  152. package/dist/core/workspace-browser.js.map +1 -1
  153. package/dist/core/workspace-scan-policy.js +26 -4
  154. package/dist/core/workspace-scan-policy.js.map +1 -1
  155. package/dist/core/workspace-tool-broker.d.ts +1 -1
  156. package/dist/core/workspace-tool-broker.js +25 -4
  157. package/dist/core/workspace-tool-broker.js.map +1 -1
  158. package/dist/mcp/server.js +148 -55
  159. package/dist/mcp/server.js.map +1 -1
  160. package/dist/mcp/workspace-path.js +2 -2
  161. package/dist/mcp/workspace-path.js.map +1 -1
  162. package/package.json +8 -5
@@ -9,35 +9,62 @@ function summarize(trials) {
9
9
  bucket.push(trial);
10
10
  buckets.set(key, bucket);
11
11
  }
12
- return [...buckets.entries()].map(([key, items]) => {
13
- const scores = items.map(item => item.score);
12
+ return [...buckets.entries()]
13
+ .map(([key, items]) => {
14
+ const scores = items.map((item) => item.score);
14
15
  const judged = items.filter((item) => item.judgeScore !== undefined && item.judgePassed !== undefined);
15
16
  const average = scores.reduce((sum, value) => sum + value, 0) / scores.length;
16
17
  return {
17
- key, provider: items[0].provider, models: items[0].models, trials: items.length,
18
+ key,
19
+ provider: items[0].provider,
20
+ models: items[0].models,
21
+ trials: items.length,
18
22
  averageScore: Math.round(average * 10) / 10,
19
- ...(judged.length ? { averageJudgeScore: Math.round(judged.reduce((sum, item) => sum + item.judgeScore, 0) / judged.length * 10) / 10, judgePassRate: Math.round(judged.filter(item => item.judgePassed).length / judged.length * 1000) / 10 } : {}),
20
- bestScore: Math.max(...scores), worstScore: Math.min(...scores),
21
- passRate: Math.round(items.filter(item => item.passed).length / items.length * 1000) / 10,
23
+ ...(judged.length
24
+ ? {
25
+ averageJudgeScore: Math.round((judged.reduce((sum, item) => sum + item.judgeScore, 0) / judged.length) * 10) / 10,
26
+ judgePassRate: Math.round((judged.filter((item) => item.judgePassed).length / judged.length) * 1000) / 10,
27
+ }
28
+ : {}),
29
+ bestScore: Math.max(...scores),
30
+ worstScore: Math.min(...scores),
31
+ passRate: Math.round((items.filter((item) => item.passed).length / items.length) * 1000) / 10,
22
32
  scoreStdDev: Math.round(Math.sqrt(scores.reduce((sum, value) => sum + Math.pow(value - average, 2), 0) / scores.length) * 10) / 10,
23
33
  averageDurationMs: Math.round(items.reduce((sum, item) => sum + item.durationMs, 0) / items.length),
24
34
  averageEstimatedTokens: Math.round(items.reduce((sum, item) => sum + item.estimatedInputTokens + item.estimatedOutputTokens, 0) / items.length),
25
35
  totalRetries: items.reduce((sum, item) => sum + item.retries, 0),
26
36
  totalFallbacks: items.reduce((sum, item) => sum + item.fallbacks, 0),
27
37
  };
28
- }).sort((a, b) => b.averageScore - a.averageScore || a.averageDurationMs - b.averageDurationMs);
38
+ })
39
+ .sort((a, b) => b.averageScore - a.averageScore || a.averageDurationMs - b.averageDurationMs);
29
40
  }
30
41
  export function createBenchmarkTrial(report, trace) {
31
42
  if (report.runId !== trace.runId)
32
43
  throw new Error("Evaluation report and execution trace must belong to the same run.");
33
- const providers = [...new Set(trace.entries.map(entry => entry.provider))].sort();
34
- const models = [...new Set(trace.entries.map(entry => entry.model))].sort();
44
+ const providers = [...new Set(trace.entries.map((entry) => entry.provider))].sort();
45
+ const models = [...new Set(trace.entries.map((entry) => entry.model))].sort();
35
46
  return {
36
- runId: report.runId, recordedAt: new Date().toISOString(), suiteId: report.suiteId, suiteName: report.suiteName,
37
- score: report.score, ...("judge" in report ? { judgeScore: report.judge.averageScore, judgeProvider: report.judge.provider, judgePassed: report.judge.passed } : {}), passed: report.passed, provider: providers.join(", ") || "unknown", models,
38
- tasks: trace.totals.tasks, failures: trace.totals.failures, fallbacks: trace.totals.fallbacks,
47
+ runId: report.runId,
48
+ recordedAt: new Date().toISOString(),
49
+ suiteId: report.suiteId,
50
+ suiteName: report.suiteName,
51
+ score: report.score,
52
+ ...("judge" in report
53
+ ? {
54
+ judgeScore: report.judge.averageScore,
55
+ judgeProvider: report.judge.provider,
56
+ judgePassed: report.judge.passed,
57
+ }
58
+ : {}),
59
+ passed: report.passed,
60
+ provider: providers.join(", ") || "unknown",
61
+ models,
62
+ tasks: trace.totals.tasks,
63
+ failures: trace.totals.failures,
64
+ fallbacks: trace.totals.fallbacks,
39
65
  retries: trace.entries.reduce((sum, entry) => sum + (entry.retries ?? 0), 0),
40
- estimatedInputTokens: trace.totals.estimatedInputTokens, estimatedOutputTokens: trace.totals.estimatedOutputTokens,
66
+ estimatedInputTokens: trace.totals.estimatedInputTokens,
67
+ estimatedOutputTokens: trace.totals.estimatedOutputTokens,
41
68
  durationMs: trace.totals.durationMs,
42
69
  };
43
70
  }
@@ -48,7 +75,10 @@ export async function recordBenchmarkTrial(projectPath, trial) {
48
75
  let trials = [];
49
76
  try {
50
77
  const previous = JSON.parse(await readFile(destination, "utf8"));
51
- if (!previous || typeof previous !== "object" || previous.version !== 1 || !Array.isArray(previous.trials))
78
+ if (!previous ||
79
+ typeof previous !== "object" ||
80
+ previous.version !== 1 ||
81
+ !Array.isArray(previous.trials))
52
82
  throw new Error("Existing benchmark history is invalid.");
53
83
  trials = previous.trials;
54
84
  }
@@ -56,8 +86,15 @@ export async function recordBenchmarkTrial(projectPath, trial) {
56
86
  if (!(error && typeof error === "object" && "code" in error && error.code === "ENOENT"))
57
87
  throw error;
58
88
  }
59
- trials = [...trials.filter(item => item.runId !== trial.runId), trial].sort((a, b) => a.recordedAt.localeCompare(b.recordedAt));
60
- const document = { version: 1, suiteId: trial.suiteId, suiteName: trial.suiteName, updatedAt: new Date().toISOString(), trials, groups: summarize(trials) };
89
+ trials = [...trials.filter((item) => item.runId !== trial.runId), trial].sort((a, b) => a.recordedAt.localeCompare(b.recordedAt));
90
+ const document = {
91
+ version: 1,
92
+ suiteId: trial.suiteId,
93
+ suiteName: trial.suiteName,
94
+ updatedAt: new Date().toISOString(),
95
+ trials,
96
+ groups: summarize(trials),
97
+ };
61
98
  const temporary = path.join(directory, "." + randomUUID() + ".tmp");
62
99
  try {
63
100
  await writeFile(temporary, JSON.stringify(document, null, 2) + "\n", { encoding: "utf8", flag: "wx", mode: 0o600 });
@@ -74,7 +111,11 @@ export async function loadBenchmark(projectPath, suiteId) {
74
111
  throw new Error("Invalid benchmark suite identifier.");
75
112
  const file = path.join(path.resolve(projectPath), ".forge", "benchmarks", suiteId + ".json");
76
113
  const value = JSON.parse(await readFile(file, "utf8"));
77
- if (!value || typeof value !== "object" || value.version !== 1 || value.suiteId !== suiteId || !Array.isArray(value.trials))
114
+ if (!value ||
115
+ typeof value !== "object" ||
116
+ value.version !== 1 ||
117
+ value.suiteId !== suiteId ||
118
+ !Array.isArray(value.trials))
78
119
  throw new Error("Stored benchmark document is invalid.");
79
120
  return value;
80
121
  }
@@ -82,7 +123,7 @@ export async function listBenchmarks(projectPath) {
82
123
  const directory = path.join(path.resolve(projectPath), ".forge", "benchmarks");
83
124
  let files;
84
125
  try {
85
- files = (await readdir(directory)).filter(file => file.endsWith(".json")).sort();
126
+ files = (await readdir(directory)).filter((file) => file.endsWith(".json")).sort();
86
127
  }
87
128
  catch (error) {
88
129
  if (error && typeof error === "object" && "code" in error && error.code === "ENOENT")
@@ -97,12 +138,39 @@ export async function listBenchmarks(projectPath) {
97
138
  return documents;
98
139
  }
99
140
  export function formatBenchmark(document) {
100
- const lines = ["ANVIL Benchmark Lab · " + document.suiteName, "Trials: " + document.trials.length, "", "Provider / model | Trials | Avg rubric | Avg judge | Pass rate | Variance | Avg time | Avg tokens"];
141
+ const lines = [
142
+ "ANVIL Benchmark Lab · " + document.suiteName,
143
+ "Trials: " + document.trials.length,
144
+ "",
145
+ "Provider / model | Trials | Avg rubric | Avg judge | Pass rate | Variance | Avg time | Avg tokens",
146
+ ];
101
147
  for (const group of document.groups)
102
- lines.push([group.key, group.trials, group.averageScore + "/100", group.averageJudgeScore === undefined ? "—" : group.averageJudgeScore + "/100 (" + group.judgePassRate + "% pass)", group.passRate + "%", "σ " + group.scoreStdDev, group.averageDurationMs + " ms", group.averageEstimatedTokens].join(" | "));
148
+ lines.push([
149
+ group.key,
150
+ group.trials,
151
+ group.averageScore + "/100",
152
+ group.averageJudgeScore === undefined
153
+ ? "—"
154
+ : group.averageJudgeScore + "/100 (" + group.judgePassRate + "% pass)",
155
+ group.passRate + "%",
156
+ "σ " + group.scoreStdDev,
157
+ group.averageDurationMs + " ms",
158
+ group.averageEstimatedTokens,
159
+ ].join(" | "));
103
160
  lines.push("", "Recent trials:");
104
161
  for (const trial of document.trials.slice(-10).reverse())
105
- lines.push(" " + trial.runId + " · " + trial.provider + " / " + (trial.models.join(", ") || "unknown") + " · " + trial.score + "/100 rubric" + (trial.judgeScore === undefined ? "" : " · " + trial.judgeScore + "/100 judge") + " · " + (trial.passed ? "PASS" : "REVIEW"));
162
+ lines.push(" " +
163
+ trial.runId +
164
+ " · " +
165
+ trial.provider +
166
+ " / " +
167
+ (trial.models.join(", ") || "unknown") +
168
+ " · " +
169
+ trial.score +
170
+ "/100 rubric" +
171
+ (trial.judgeScore === undefined ? "" : " · " + trial.judgeScore + "/100 judge") +
172
+ " · " +
173
+ (trial.passed ? "PASS" : "REVIEW"));
106
174
  lines.push("", "Token counts are estimates, not provider-billed usage. Benchmarks compare recorded runs; they do not automatically rerun tasks.");
107
175
  return lines.join("\n");
108
176
  }
@@ -1 +1 @@
1
- {"version":3,"file":"benchmark-engine.js","sourceRoot":"","sources":["../../src/core/benchmark-engine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAkD7B,SAAS,SAAS,CAAC,MAAwB;IACzC,MAAM,OAAO,GAAG,IAAI,GAAG,EAA4B,CAAC;IACpD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,GAAG,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QACjD,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAyE,EAAE,CAAC,IAAI,CAAC,UAAU,KAAK,SAAS,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC;QAC9K,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAC9E,OAAO;YACL,GAAG,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM;YACjF,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,GAAG,EAAE;YAC3C,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpP,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;YAC/D,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE;YACzF,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE;YAClI,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;YACnG,sBAAsB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;YAC/I,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YAChE,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;SACrE,CAAC;IACJ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,iBAAiB,GAAG,CAAC,CAAC,iBAAiB,CAAC,CAAC;AAClG,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,MAAiD,EAAE,KAA6B;IACnH,IAAI,MAAM,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;IACxH,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAClF,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5E,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS;QAC/G,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY,EAAE,aAAa,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,EAAE,MAAM;QAChP,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;QAC7F,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QAC5E,oBAAoB,EAAE,KAAK,CAAC,MAAM,CAAC,oBAAoB,EAAE,qBAAqB,EAAE,KAAK,CAAC,MAAM,CAAC,qBAAqB;QAClH,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;KACpC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,WAAmB,EAAE,KAAqB;IACnF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC/E,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC,CAAC;IAClE,IAAI,MAAM,GAAqB,EAAE,CAAC;IAClC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAY,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC;QAC1E,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAK,QAA8B,CAAC,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAE,QAA8B,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QACpN,MAAM,GAAI,QAA8B,CAAC,MAAM,CAAC;IAClD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC;YAAE,MAAM,KAAK,CAAC;IACvG,CAAC;IACD,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;IAChI,MAAM,QAAQ,GAAsB,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;IAC/K,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,GAAG,UAAU,EAAE,GAAG,MAAM,CAAC,CAAC;IACpE,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACpH,MAAM,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC5D,MAAM,KAAK,CAAC;IACd,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,WAAmB,EAAE,OAAe;IACtE,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACvG,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,GAAG,OAAO,CAAC,CAAC;IAC7F,MAAM,KAAK,GAAY,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAChE,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAK,KAA2B,CAAC,OAAO,KAAK,CAAC,IAAK,KAA2B,CAAC,OAAO,KAAK,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAE,KAA2B,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3P,OAAO,KAA0B,CAAC;AACpC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,WAAmB;IACtD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC/E,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QAAC,KAAK,GAAG,CAAC,MAAM,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAAC,CAAC;IACzF,OAAO,KAAK,EAAE,CAAC;QAAC,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAC;QAAC,MAAM,KAAK,CAAC;IAAC,CAAC;IAC/H,MAAM,SAAS,GAAwB,EAAE,CAAC;IAC1C,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7B,SAAS,CAAC,IAAI,CAAC,MAAM,aAAa,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,QAA2B;IACzD,MAAM,KAAK,GAAG,CAAC,wBAAwB,GAAG,QAAQ,CAAC,SAAS,EAAE,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,EAAE,mGAAmG,CAAC,CAAC;IAC5M,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,GAAG,MAAM,EAAE,KAAK,CAAC,iBAAiB,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,iBAAiB,GAAG,QAAQ,GAAG,KAAK,CAAC,aAAa,GAAG,SAAS,EAAE,KAAK,CAAC,QAAQ,GAAG,GAAG,EAAE,IAAI,GAAG,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,iBAAiB,GAAG,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IACvV,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IACjC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC,QAAQ,GAAG,KAAK,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,GAAG,KAAK,GAAG,KAAK,CAAC,KAAK,GAAG,aAAa,GAAG,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,UAAU,GAAG,YAAY,CAAC,GAAG,KAAK,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IACxU,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,iIAAiI,CAAC,CAAC;IAClJ,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
1
+ {"version":3,"file":"benchmark-engine.js","sourceRoot":"","sources":["../../src/core/benchmark-engine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAkD7B,SAAS,SAAS,CAAC,MAAwB;IACzC,MAAM,OAAO,GAAG,IAAI,GAAG,EAA4B,CAAC;IACpD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,GAAG,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;SAC1B,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QACpB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CACzB,CAAC,IAAI,EAAyE,EAAE,CAC9E,IAAI,CAAC,UAAU,KAAK,SAAS,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,CAClE,CAAC;QACF,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAC9E,OAAO;YACL,GAAG;YACH,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,QAAQ;YAC5B,MAAM,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,MAAM;YACxB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,GAAG,EAAE;YAC3C,GAAG,CAAC,MAAM,CAAC,MAAM;gBACf,CAAC,CAAC;oBACE,iBAAiB,EACf,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE;oBAChG,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE;iBAC1G;gBACH,CAAC,CAAC,EAAE,CAAC;YACP,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;YAC9B,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;YAC/B,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE;YAC7F,WAAW,EACT,IAAI,CAAC,KAAK,CACR,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CACrG,GAAG,EAAE;YACR,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;YACnG,sBAAsB,EAAE,IAAI,CAAC,KAAK,CAChC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAC5G;YACD,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YAChE,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;SACrE,CAAC;IACJ,CAAC,CAAC;SACD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,iBAAiB,GAAG,CAAC,CAAC,iBAAiB,CAAC,CAAC;AAClG,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,MAAiD,EACjD,KAA6B;IAE7B,IAAI,MAAM,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK;QAC9B,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;IACxF,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACpF,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9E,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACpC,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,GAAG,CAAC,OAAO,IAAI,MAAM;YACnB,CAAC,CAAC;gBACE,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY;gBACrC,aAAa,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ;gBACpC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM;aACjC;YACH,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS;QAC3C,MAAM;QACN,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK;QACzB,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,QAAQ;QAC/B,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;QACjC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QAC5E,oBAAoB,EAAE,KAAK,CAAC,MAAM,CAAC,oBAAoB;QACvD,qBAAqB,EAAE,KAAK,CAAC,MAAM,CAAC,qBAAqB;QACzD,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;KACpC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,WAAmB,EAAE,KAAqB;IACnF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC/E,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC,CAAC;IAClE,IAAI,MAAM,GAAqB,EAAE,CAAC;IAClC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAY,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC;QAC1E,IACE,CAAC,QAAQ;YACT,OAAO,QAAQ,KAAK,QAAQ;YAC3B,QAA8B,CAAC,OAAO,KAAK,CAAC;YAC7C,CAAC,KAAK,CAAC,OAAO,CAAE,QAA8B,CAAC,MAAM,CAAC;YAEtD,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC5D,MAAM,GAAI,QAA8B,CAAC,MAAM,CAAC;IAClD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC;YAAE,MAAM,KAAK,CAAC;IACvG,CAAC;IACD,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACrF,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CACzC,CAAC;IACF,MAAM,QAAQ,GAAsB;QAClC,OAAO,EAAE,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM;QACN,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC;KAC1B,CAAC;IACF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,GAAG,UAAU,EAAE,GAAG,MAAM,CAAC,CAAC;IACpE,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACpH,MAAM,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC5D,MAAM,KAAK,CAAC;IACd,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,WAAmB,EAAE,OAAe;IACtE,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACvG,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,GAAG,OAAO,CAAC,CAAC;IAC7F,MAAM,KAAK,GAAY,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAChE,IACE,CAAC,KAAK;QACN,OAAO,KAAK,KAAK,QAAQ;QACxB,KAA2B,CAAC,OAAO,KAAK,CAAC;QACzC,KAA2B,CAAC,OAAO,KAAK,OAAO;QAChD,CAAC,KAAK,CAAC,OAAO,CAAE,KAA2B,CAAC,MAAM,CAAC;QAEnD,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,OAAO,KAA0B,CAAC;AACpC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,WAAmB;IACtD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC/E,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,CAAC,MAAM,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACrF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAC;QAChG,MAAM,KAAK,CAAC;IACd,CAAC;IACD,MAAM,SAAS,GAAwB,EAAE,CAAC;IAC1C,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7B,SAAS,CAAC,IAAI,CAAC,MAAM,aAAa,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,QAA2B;IACzD,MAAM,KAAK,GAAG;QACZ,wBAAwB,GAAG,QAAQ,CAAC,SAAS;QAC7C,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM;QACnC,EAAE;QACF,mGAAmG;KACpG,CAAC;IACF,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM;QACjC,KAAK,CAAC,IAAI,CACR;YACE,KAAK,CAAC,GAAG;YACT,KAAK,CAAC,MAAM;YACZ,KAAK,CAAC,YAAY,GAAG,MAAM;YAC3B,KAAK,CAAC,iBAAiB,KAAK,SAAS;gBACnC,CAAC,CAAC,GAAG;gBACL,CAAC,CAAC,KAAK,CAAC,iBAAiB,GAAG,QAAQ,GAAG,KAAK,CAAC,aAAa,GAAG,SAAS;YACxE,KAAK,CAAC,QAAQ,GAAG,GAAG;YACpB,IAAI,GAAG,KAAK,CAAC,WAAW;YACxB,KAAK,CAAC,iBAAiB,GAAG,KAAK;YAC/B,KAAK,CAAC,sBAAsB;SAC7B,CAAC,IAAI,CAAC,KAAK,CAAC,CACd,CAAC;IACJ,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IACjC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE;QACtD,KAAK,CAAC,IAAI,CACR,IAAI;YACF,KAAK,CAAC,KAAK;YACX,KAAK;YACL,KAAK,CAAC,QAAQ;YACd,KAAK;YACL,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC;YACtC,KAAK;YACL,KAAK,CAAC,KAAK;YACX,aAAa;YACb,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,UAAU,GAAG,YAAY,CAAC;YAC/E,KAAK;YACL,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CACrC,CAAC;IACJ,KAAK,CAAC,IAAI,CACR,EAAE,EACF,iIAAiI,CAClI,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
@@ -25,27 +25,38 @@ export function validateBenchmarkMatrix(value) {
25
25
  throw new Error("Candidate configVersionId must be a valid agent configuration snapshot ID.");
26
26
  }
27
27
  for (const candidate of config.candidates) {
28
- if (candidate.skillIds !== undefined && (!Array.isArray(candidate.skillIds) || candidate.skillIds.some(id => typeof id !== "string" || !/^[a-z0-9][a-z0-9-]{0,63}$/.test(id)) || new Set(candidate.skillIds).size !== candidate.skillIds.length))
28
+ if (candidate.skillIds !== undefined &&
29
+ (!Array.isArray(candidate.skillIds) ||
30
+ candidate.skillIds.some((id) => typeof id !== "string" || !/^[a-z0-9][a-z0-9-]{0,63}$/.test(id)) ||
31
+ new Set(candidate.skillIds).size !== candidate.skillIds.length))
29
32
  throw new Error("Candidate skillIds must be unique valid skill IDs.");
30
33
  }
31
34
  if (config.skillExperiment) {
32
35
  const experiment = config.skillExperiment;
33
- const baseline = config.candidates.find(item => item.id === experiment.baselineCandidateId);
34
- const treatment = config.candidates.find(item => item.id === experiment.treatmentCandidateId);
36
+ const baseline = config.candidates.find((item) => item.id === experiment.baselineCandidateId);
37
+ const treatment = config.candidates.find((item) => item.id === experiment.treatmentCandidateId);
35
38
  if (!baseline || !treatment || baseline.id === treatment.id)
36
39
  throw new Error("Skill experiment must reference distinct baseline and treatment candidates.");
37
40
  if (baseline.provider !== treatment.provider || baseline.model !== treatment.model)
38
41
  throw new Error("Skill experiment candidates must use the same provider and model.");
39
- if (!Array.isArray(experiment.skillIds) || !experiment.skillIds.length || experiment.skillIds.some(id => typeof id !== "string" || !/^[a-z0-9][a-z0-9-]{0,63}$/.test(id)) || new Set(experiment.skillIds).size !== experiment.skillIds.length)
42
+ if (!Array.isArray(experiment.skillIds) ||
43
+ !experiment.skillIds.length ||
44
+ experiment.skillIds.some((id) => typeof id !== "string" || !/^[a-z0-9][a-z0-9-]{0,63}$/.test(id)) ||
45
+ new Set(experiment.skillIds).size !== experiment.skillIds.length)
40
46
  throw new Error("Skill experiment must specify unique valid skill IDs.");
41
- if (!Array.isArray(baseline.skillIds) || baseline.skillIds.length !== 0 || !Array.isArray(treatment.skillIds) || [...treatment.skillIds].sort().join(",") !== [...experiment.skillIds].sort().join(","))
47
+ if (!Array.isArray(baseline.skillIds) ||
48
+ baseline.skillIds.length !== 0 ||
49
+ !Array.isArray(treatment.skillIds) ||
50
+ [...treatment.skillIds].sort().join(",") !== [...experiment.skillIds].sort().join(","))
42
51
  throw new Error("Baseline must set skillIds to [] and treatment skillIds must exactly match skillExperiment.skillIds.");
43
52
  }
44
53
  return config;
45
54
  }
46
55
  export function fingerprintAgentConfig(files) {
47
- const canonical = [...files].sort((a, b) => a.path.localeCompare(b.path))
48
- .map(file => file.path + ":" + file.sha256).join("\\n");
56
+ const canonical = [...files]
57
+ .sort((a, b) => a.path.localeCompare(b.path))
58
+ .map((file) => file.path + ":" + file.sha256)
59
+ .join("\\n");
49
60
  return createHash("sha256").update(canonical, "utf8").digest("hex");
50
61
  }
51
62
  export async function loadBenchmarkMatrix(file) {
@@ -58,52 +69,155 @@ export async function executeBenchmarkMatrix(config, execute) {
58
69
  for (let repeat = 1; repeat <= config.repeats; repeat += 1) {
59
70
  try {
60
71
  const result = await execute(candidate, repeat);
61
- results.push({ candidateId: candidate.id, provider: candidate.provider, model: candidate.model, repeat, status: "completed", ...result });
72
+ results.push({
73
+ candidateId: candidate.id,
74
+ provider: candidate.provider,
75
+ model: candidate.model,
76
+ repeat,
77
+ status: "completed",
78
+ ...result,
79
+ });
62
80
  }
63
81
  catch (error) {
64
- results.push({ candidateId: candidate.id, provider: candidate.provider, model: candidate.model, repeat, runId: "", score: 0, passed: false, durationMs: 0, estimatedTokens: 0, status: "failed", error: error instanceof Error ? error.message : String(error) });
82
+ results.push({
83
+ candidateId: candidate.id,
84
+ provider: candidate.provider,
85
+ model: candidate.model,
86
+ repeat,
87
+ runId: "",
88
+ score: 0,
89
+ passed: false,
90
+ durationMs: 0,
91
+ estimatedTokens: 0,
92
+ status: "failed",
93
+ error: error instanceof Error ? error.message : String(error),
94
+ });
65
95
  }
66
96
  }
67
97
  }
68
- const summary = config.candidates.map(candidate => {
69
- const items = results.filter(item => item.candidateId === candidate.id && item.status === "completed");
70
- const scores = items.map(item => item.score);
98
+ const summary = config.candidates
99
+ .map((candidate) => {
100
+ const items = results.filter((item) => item.candidateId === candidate.id && item.status === "completed");
101
+ const scores = items.map((item) => item.score);
71
102
  const average = scores.length ? scores.reduce((sum, score) => sum + score, 0) / scores.length : 0;
72
103
  return {
73
- candidateId: candidate.id, provider: candidate.provider, model: candidate.model, trials: config.repeats,
104
+ candidateId: candidate.id,
105
+ provider: candidate.provider,
106
+ model: candidate.model,
107
+ trials: config.repeats,
74
108
  completed: items.length,
75
109
  averageScore: Math.round(average * 10) / 10,
76
- passRate: items.length ? Math.round(items.filter(item => item.passed).length / items.length * 1000) / 10 : 0,
77
- scoreStdDev: items.length ? Math.round(Math.sqrt(scores.reduce((sum, score) => sum + Math.pow(score - average, 2), 0) / scores.length) * 10) / 10 : 0,
78
- averageDurationMs: items.length ? Math.round(items.reduce((sum, item) => sum + item.durationMs, 0) / items.length) : 0,
79
- averageEstimatedTokens: items.length ? Math.round(items.reduce((sum, item) => sum + item.estimatedTokens, 0) / items.length) : 0,
110
+ passRate: items.length
111
+ ? Math.round((items.filter((item) => item.passed).length / items.length) * 1000) / 10
112
+ : 0,
113
+ scoreStdDev: items.length
114
+ ? Math.round(Math.sqrt(scores.reduce((sum, score) => sum + Math.pow(score - average, 2), 0) / scores.length) * 10) / 10
115
+ : 0,
116
+ averageDurationMs: items.length
117
+ ? Math.round(items.reduce((sum, item) => sum + item.durationMs, 0) / items.length)
118
+ : 0,
119
+ averageEstimatedTokens: items.length
120
+ ? Math.round(items.reduce((sum, item) => sum + item.estimatedTokens, 0) / items.length)
121
+ : 0,
80
122
  };
81
- }).sort((a, b) => b.averageScore - a.averageScore || a.averageDurationMs - b.averageDurationMs);
82
- const agentKeys = new Set(results.flatMap(item => (item.agentScores ?? []).map(score => item.candidateId + "::" + score.agent)));
83
- const agentSummary = [...agentKeys].map(key => {
123
+ })
124
+ .sort((a, b) => b.averageScore - a.averageScore || a.averageDurationMs - b.averageDurationMs);
125
+ const agentKeys = new Set(results.flatMap((item) => (item.agentScores ?? []).map((score) => item.candidateId + "::" + score.agent)));
126
+ const agentSummary = [...agentKeys]
127
+ .map((key) => {
84
128
  const separator = key.indexOf("::");
85
129
  const candidateId = key.slice(0, separator);
86
130
  const agent = key.slice(separator + 2);
87
- const candidate = config.candidates.find(item => item.id === candidateId);
88
- const scores = results.filter(item => item.candidateId === candidateId && item.status === "completed").flatMap(item => (item.agentScores ?? []).filter(score => score.agent === agent));
89
- const skillsObserved = [...new Set(scores.flatMap(item => item.skillIds))].sort();
90
- return { candidateId, provider: candidate.provider, model: candidate.model, agent, trials: config.repeats, completed: scores.length, averageScore: scores.length ? Math.round(scores.reduce((sum, item) => sum + item.score, 0) / scores.length * 10) / 10 : 0, passRate: scores.length ? Math.round(scores.filter(item => item.passed).length / scores.length * 1000) / 10 : 0, skillsObserved };
91
- }).sort((a, b) => b.averageScore - a.averageScore || a.agent.localeCompare(b.agent));
131
+ const candidate = config.candidates.find((item) => item.id === candidateId);
132
+ const scores = results
133
+ .filter((item) => item.candidateId === candidateId && item.status === "completed")
134
+ .flatMap((item) => (item.agentScores ?? []).filter((score) => score.agent === agent));
135
+ const skillsObserved = [...new Set(scores.flatMap((item) => item.skillIds))].sort();
136
+ return {
137
+ candidateId,
138
+ provider: candidate.provider,
139
+ model: candidate.model,
140
+ agent,
141
+ trials: config.repeats,
142
+ completed: scores.length,
143
+ averageScore: scores.length
144
+ ? Math.round((scores.reduce((sum, item) => sum + item.score, 0) / scores.length) * 10) / 10
145
+ : 0,
146
+ passRate: scores.length
147
+ ? Math.round((scores.filter((item) => item.passed).length / scores.length) * 1000) / 10
148
+ : 0,
149
+ skillsObserved,
150
+ };
151
+ })
152
+ .sort((a, b) => b.averageScore - a.averageScore || a.agent.localeCompare(b.agent));
92
153
  let skillExperiment;
93
154
  if (config.skillExperiment) {
94
155
  const { baselineCandidateId, treatmentCandidateId, skillIds } = config.skillExperiment;
95
- const baseline = summary.find(item => item.candidateId === baselineCandidateId);
96
- const treatment = summary.find(item => item.candidateId === treatmentCandidateId);
156
+ const baseline = summary.find((item) => item.candidateId === baselineCandidateId);
157
+ const treatment = summary.find((item) => item.candidateId === treatmentCandidateId);
97
158
  if (baseline && treatment && baseline.completed > 0 && treatment.completed > 0) {
98
- const change = (before, after) => before <= 0 ? (after > 0 ? 100 : 0) : Math.round((after - before) / before * 1000) / 10;
99
- const agentDeltas = (agentSummary.filter(item => item.candidateId === baselineCandidateId)).flatMap(before => {
100
- const after = agentSummary.find(item => item.candidateId === treatmentCandidateId && item.agent === before.agent);
101
- return after ? [{ agent: before.agent, baselineScore: before.averageScore, treatmentScore: after.averageScore, scoreDelta: Math.round((after.averageScore - before.averageScore) * 10) / 10, baselinePassRate: before.passRate, treatmentPassRate: after.passRate, passRateDelta: Math.round((after.passRate - before.passRate) * 10) / 10 }] : [];
159
+ const change = (before, after) => before <= 0 ? (after > 0 ? 100 : 0) : Math.round(((after - before) / before) * 1000) / 10;
160
+ const agentDeltas = agentSummary
161
+ .filter((item) => item.candidateId === baselineCandidateId)
162
+ .flatMap((before) => {
163
+ const after = agentSummary.find((item) => item.candidateId === treatmentCandidateId && item.agent === before.agent);
164
+ return after
165
+ ? [
166
+ {
167
+ agent: before.agent,
168
+ baselineScore: before.averageScore,
169
+ treatmentScore: after.averageScore,
170
+ scoreDelta: Math.round((after.averageScore - before.averageScore) * 10) / 10,
171
+ baselinePassRate: before.passRate,
172
+ treatmentPassRate: after.passRate,
173
+ passRateDelta: Math.round((after.passRate - before.passRate) * 10) / 10,
174
+ },
175
+ ]
176
+ : [];
102
177
  });
103
- skillExperiment = { baselineCandidateId, treatmentCandidateId, skillIds: [...skillIds], baselineScore: baseline.averageScore, treatmentScore: treatment.averageScore, scoreDelta: Math.round((treatment.averageScore - baseline.averageScore) * 10) / 10, baselinePassRate: baseline.passRate, treatmentPassRate: treatment.passRate, passRateDelta: Math.round((treatment.passRate - baseline.passRate) * 10) / 10, baselineDurationMs: baseline.averageDurationMs, treatmentDurationMs: treatment.averageDurationMs, durationChangePercent: change(baseline.averageDurationMs, treatment.averageDurationMs), baselineEstimatedTokens: baseline.averageEstimatedTokens, treatmentEstimatedTokens: treatment.averageEstimatedTokens, tokenChangePercent: change(baseline.averageEstimatedTokens, treatment.averageEstimatedTokens), agentDeltas };
178
+ skillExperiment = {
179
+ baselineCandidateId,
180
+ treatmentCandidateId,
181
+ skillIds: [...skillIds],
182
+ baselineScore: baseline.averageScore,
183
+ treatmentScore: treatment.averageScore,
184
+ scoreDelta: Math.round((treatment.averageScore - baseline.averageScore) * 10) / 10,
185
+ baselinePassRate: baseline.passRate,
186
+ treatmentPassRate: treatment.passRate,
187
+ passRateDelta: Math.round((treatment.passRate - baseline.passRate) * 10) / 10,
188
+ baselineDurationMs: baseline.averageDurationMs,
189
+ treatmentDurationMs: treatment.averageDurationMs,
190
+ durationChangePercent: change(baseline.averageDurationMs, treatment.averageDurationMs),
191
+ baselineEstimatedTokens: baseline.averageEstimatedTokens,
192
+ treatmentEstimatedTokens: treatment.averageEstimatedTokens,
193
+ tokenChangePercent: change(baseline.averageEstimatedTokens, treatment.averageEstimatedTokens),
194
+ agentDeltas,
195
+ };
104
196
  }
105
197
  }
106
- return { version: 1, matrixId: config.id, matrixName: config.name, suiteId: "", goal: "", createdAt: new Date().toISOString(), results, summary, agentSummary, ...(config.candidates.some(candidate => candidate.configVersionId) ? { candidateProvenance: config.candidates.filter(candidate => candidate.configVersionId).map(candidate => ({ candidateId: candidate.id, configVersionId: candidate.configVersionId, fingerprint: "" })) } : {}), ...(skillExperiment ? { skillExperiment } : {}) };
198
+ return {
199
+ version: 1,
200
+ matrixId: config.id,
201
+ matrixName: config.name,
202
+ suiteId: "",
203
+ goal: "",
204
+ createdAt: new Date().toISOString(),
205
+ results,
206
+ summary,
207
+ agentSummary,
208
+ ...(config.candidates.some((candidate) => candidate.configVersionId)
209
+ ? {
210
+ candidateProvenance: config.candidates
211
+ .filter((candidate) => candidate.configVersionId)
212
+ .map((candidate) => ({
213
+ candidateId: candidate.id,
214
+ configVersionId: candidate.configVersionId,
215
+ fingerprint: "",
216
+ })),
217
+ }
218
+ : {}),
219
+ ...(skillExperiment ? { skillExperiment } : {}),
220
+ };
107
221
  }
108
222
  export async function saveBenchmarkMatrix(projectPath, report) {
109
223
  const directory = path.join(path.resolve(projectPath), ".forge", "benchmark-matrices");
@@ -121,23 +235,79 @@ export async function saveBenchmarkMatrix(projectPath, report) {
121
235
  return destination;
122
236
  }
123
237
  export function formatBenchmarkMatrix(report) {
124
- const lines = ["ANVIL Automated Benchmark Matrix · " + report.matrixName, "Suite: " + report.suiteId, "Goal: " + report.goal, "", "Candidate | Trials | Completed | Avg score | Pass rate | σ score | Avg time | Avg estimated tokens"];
238
+ const lines = [
239
+ "ANVIL Automated Benchmark Matrix · " + report.matrixName,
240
+ "Suite: " + report.suiteId,
241
+ "Goal: " + report.goal,
242
+ "",
243
+ "Candidate | Trials | Completed | Avg score | Pass rate | σ score | Avg time | Avg estimated tokens",
244
+ ];
125
245
  for (const item of report.summary)
126
- lines.push([item.candidateId + " (" + item.provider + "/" + item.model + ")", item.trials, item.completed, item.averageScore + "/100", item.passRate + "%", item.scoreStdDev, item.averageDurationMs + " ms", item.averageEstimatedTokens].join(" | "));
246
+ lines.push([
247
+ item.candidateId + " (" + item.provider + "/" + item.model + ")",
248
+ item.trials,
249
+ item.completed,
250
+ item.averageScore + "/100",
251
+ item.passRate + "%",
252
+ item.scoreStdDev,
253
+ item.averageDurationMs + " ms",
254
+ item.averageEstimatedTokens,
255
+ ].join(" | "));
127
256
  if (report.agentSummary?.length) {
128
257
  lines.push("", "Per-agent scorecards:", "Candidate | Agent | Cases | Average score | Pass rate | Skills observed");
129
258
  for (const item of report.agentSummary)
130
- lines.push([item.candidateId, item.agent, item.completed, item.averageScore + "/100", item.passRate + "%", item.skillsObserved.join(", ") || "none"].join(" | "));
259
+ lines.push([
260
+ item.candidateId,
261
+ item.agent,
262
+ item.completed,
263
+ item.averageScore + "/100",
264
+ item.passRate + "%",
265
+ item.skillsObserved.join(", ") || "none",
266
+ ].join(" | "));
131
267
  }
132
268
  if (report.skillExperiment) {
133
269
  const item = report.skillExperiment;
134
- lines.push("", "Controlled skill experiment:", " Baseline: " + item.baselineCandidateId + " (no skills)", " Treatment: " + item.treatmentCandidateId + " (skills: " + item.skillIds.join(", ") + ")", " Score: " + item.baselineScore + " → " + item.treatmentScore + " (" + (item.scoreDelta > 0 ? "+" : "") + item.scoreDelta + ")", " Pass rate: " + item.baselinePassRate + "% → " + item.treatmentPassRate + "% (" + (item.passRateDelta > 0 ? "+" : "") + item.passRateDelta + " pp)", " Duration change: " + item.durationChangePercent + "% · estimated token change: " + item.tokenChangePercent + "%");
270
+ lines.push("", "Controlled skill experiment:", " Baseline: " + item.baselineCandidateId + " (no skills)", " Treatment: " + item.treatmentCandidateId + " (skills: " + item.skillIds.join(", ") + ")", " Score: " +
271
+ item.baselineScore +
272
+ " → " +
273
+ item.treatmentScore +
274
+ " (" +
275
+ (item.scoreDelta > 0 ? "+" : "") +
276
+ item.scoreDelta +
277
+ ")", " Pass rate: " +
278
+ item.baselinePassRate +
279
+ "% → " +
280
+ item.treatmentPassRate +
281
+ "% (" +
282
+ (item.passRateDelta > 0 ? "+" : "") +
283
+ item.passRateDelta +
284
+ " pp)", " Duration change: " +
285
+ item.durationChangePercent +
286
+ "% · estimated token change: " +
287
+ item.tokenChangePercent +
288
+ "%");
135
289
  for (const agent of item.agentDeltas)
136
- lines.push(" Agent " + agent.agent + ": " + agent.baselineScore + " → " + agent.treatmentScore + " (" + (agent.scoreDelta > 0 ? "+" : "") + agent.scoreDelta + ")");
290
+ lines.push(" Agent " +
291
+ agent.agent +
292
+ ": " +
293
+ agent.baselineScore +
294
+ " → " +
295
+ agent.treatmentScore +
296
+ " (" +
297
+ (agent.scoreDelta > 0 ? "+" : "") +
298
+ agent.scoreDelta +
299
+ ")");
137
300
  }
138
301
  lines.push("", "Trial details:");
139
302
  for (const item of report.results)
140
- lines.push(" " + item.candidateId + " #" + item.repeat + " · " + (item.status === "completed" ? item.score + "/100 · " + (item.passed ? "PASS" : "REVIEW") : "FAILED · " + item.error));
303
+ lines.push(" " +
304
+ item.candidateId +
305
+ " #" +
306
+ item.repeat +
307
+ " · " +
308
+ (item.status === "completed"
309
+ ? item.score + "/100 · " + (item.passed ? "PASS" : "REVIEW")
310
+ : "FAILED · " + item.error));
141
311
  if (report.candidateProvenance?.length) {
142
312
  lines.push("", "Configuration provenance:");
143
313
  for (const item of report.candidateProvenance)
@@ -150,7 +320,7 @@ export async function listBenchmarkMatrices(projectPath) {
150
320
  const directory = path.join(path.resolve(projectPath), ".forge", "benchmark-matrices");
151
321
  let files;
152
322
  try {
153
- files = (await readdir(directory)).filter(file => file.endsWith(".json")).sort();
323
+ files = (await readdir(directory)).filter((file) => file.endsWith(".json")).sort();
154
324
  }
155
325
  catch (error) {
156
326
  if (error && typeof error === "object" && "code" in error && error.code === "ENOENT")
@@ -160,7 +330,11 @@ export async function listBenchmarkMatrices(projectPath) {
160
330
  const reports = [];
161
331
  for (const file of files.slice(-100)) {
162
332
  const value = JSON.parse(await readFile(path.join(directory, file), "utf8"));
163
- if (!value || typeof value !== "object" || value.version !== 1 || !Array.isArray(value.results) || !Array.isArray(value.summary))
333
+ if (!value ||
334
+ typeof value !== "object" ||
335
+ value.version !== 1 ||
336
+ !Array.isArray(value.results) ||
337
+ !Array.isArray(value.summary))
164
338
  throw new Error("Stored benchmark matrix report is invalid: " + file);
165
339
  reports.push(value);
166
340
  }