anvil-engineering-workbench 1.0.9 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -1
- package/README.md +23 -18
- package/dist/ai/claude-code-provider.js +55 -9
- package/dist/ai/claude-code-provider.js.map +1 -1
- package/dist/ai/provider.js +1 -1
- package/dist/ai/provider.js.map +1 -1
- package/dist/cli.js +837 -278
- package/dist/cli.js.map +1 -1
- package/dist/core/adr-engine.js +9 -11
- package/dist/core/adr-engine.js.map +1 -1
- package/dist/core/agent-config-release.js +29 -13
- package/dist/core/agent-config-release.js.map +1 -1
- package/dist/core/agent-config-versioning.js +32 -17
- package/dist/core/agent-config-versioning.js.map +1 -1
- package/dist/core/agent-evaluation-intelligence.js +30 -38
- package/dist/core/agent-evaluation-intelligence.js.map +1 -1
- package/dist/core/agent-evolution.js +109 -20
- package/dist/core/agent-evolution.js.map +1 -1
- package/dist/core/agent-experiment-studio.js +19 -11
- package/dist/core/agent-experiment-studio.js.map +1 -1
- package/dist/core/agent-optimization.js +69 -14
- package/dist/core/agent-optimization.js.map +1 -1
- package/dist/core/agent-registry.js +87 -15
- package/dist/core/agent-registry.js.map +1 -1
- package/dist/core/agent-security-audit.js +21 -21
- package/dist/core/agent-security-audit.js.map +1 -1
- package/dist/core/agent-workspace-context.js +37 -6
- package/dist/core/agent-workspace-context.js.map +1 -1
- package/dist/core/ai-code-review.js +60 -12
- package/dist/core/ai-code-review.js.map +1 -1
- package/dist/core/anthropic-provider.js.map +1 -1
- package/dist/core/architecture-map.js +87 -24
- package/dist/core/architecture-map.js.map +1 -1
- package/dist/core/benchmark-engine.js +89 -21
- package/dist/core/benchmark-engine.js.map +1 -1
- package/dist/core/benchmark-matrix.js +215 -41
- package/dist/core/benchmark-matrix.js.map +1 -1
- package/dist/core/benchmark-regression.js +124 -18
- package/dist/core/benchmark-regression.js.map +1 -1
- package/dist/core/change-intelligence.js +102 -43
- package/dist/core/change-intelligence.js.map +1 -1
- package/dist/core/code-intelligence.js +40 -13
- package/dist/core/code-intelligence.js.map +1 -1
- package/dist/core/code-review.js +145 -17
- package/dist/core/code-review.js.map +1 -1
- package/dist/core/code-search.js +79 -39
- package/dist/core/code-search.js.map +1 -1
- package/dist/core/container-sandbox.js +87 -23
- package/dist/core/container-sandbox.js.map +1 -1
- package/dist/core/dashboard-server.js +379 -93
- package/dist/core/dashboard-server.js.map +1 -1
- package/dist/core/dependency-graph.js +75 -13
- package/dist/core/dependency-graph.js.map +1 -1
- package/dist/core/dependency-impact.js +132 -33
- package/dist/core/dependency-impact.js.map +1 -1
- package/dist/core/dependency-intelligence.js +153 -56
- package/dist/core/dependency-intelligence.js.map +1 -1
- package/dist/core/doctor.js +3 -3
- package/dist/core/doctor.js.map +1 -1
- package/dist/core/engineering-risk.js +75 -40
- package/dist/core/engineering-risk.js.map +1 -1
- package/dist/core/env-compat.js.map +1 -1
- package/dist/core/evaluation-engine.js +188 -42
- package/dist/core/evaluation-engine.js.map +1 -1
- package/dist/core/execution-trace.js +83 -29
- package/dist/core/execution-trace.js.map +1 -1
- package/dist/core/experiment-history.js +70 -20
- package/dist/core/experiment-history.js.map +1 -1
- package/dist/core/experiment-manifest.js +92 -13
- package/dist/core/experiment-manifest.js.map +1 -1
- package/dist/core/gemini-provider.js.map +1 -1
- package/dist/core/git-intelligence.js +68 -16
- package/dist/core/git-intelligence.js.map +1 -1
- package/dist/core/hook-engine.js +7 -3
- package/dist/core/hook-engine.js.map +1 -1
- package/dist/core/hosted-agent-provider.js +48 -8
- package/dist/core/hosted-agent-provider.js.map +1 -1
- package/dist/core/intelligence-timeline.js +92 -43
- package/dist/core/intelligence-timeline.js.map +1 -1
- package/dist/core/mcp-setup.js +12 -12
- package/dist/core/mcp-setup.js.map +1 -1
- package/dist/core/model-router.js +153 -37
- package/dist/core/model-router.js.map +1 -1
- package/dist/core/openai-compatible-provider.js +41 -10
- package/dist/core/openai-compatible-provider.js.map +1 -1
- package/dist/core/orchestrator.js +28 -23
- package/dist/core/orchestrator.js.map +1 -1
- package/dist/core/patch-application.js +88 -24
- package/dist/core/patch-application.js.map +1 -1
- package/dist/core/patch-approval.js +43 -15
- package/dist/core/patch-approval.js.map +1 -1
- package/dist/core/patch-validation.js +93 -15
- package/dist/core/patch-validation.js.map +1 -1
- package/dist/core/permission-policy.js +8 -5
- package/dist/core/permission-policy.js.map +1 -1
- package/dist/core/policy-engine.d.ts +76 -0
- package/dist/core/policy-engine.js +352 -0
- package/dist/core/policy-engine.js.map +1 -0
- package/dist/core/project-cleaner.js +143 -129
- package/dist/core/project-cleaner.js.map +1 -1
- package/dist/core/project-health.js +183 -21
- package/dist/core/project-health.js.map +1 -1
- package/dist/core/project-init.js +32 -20
- package/dist/core/project-init.js.map +1 -1
- package/dist/core/project-inspector.js +81 -32
- package/dist/core/project-inspector.js.map +1 -1
- package/dist/core/project-memory.js +41 -18
- package/dist/core/project-memory.js.map +1 -1
- package/dist/core/provider-diagnostics.js +65 -12
- package/dist/core/provider-diagnostics.js.map +1 -1
- package/dist/core/quality-audit.js +167 -28
- package/dist/core/quality-audit.js.map +1 -1
- package/dist/core/regression-intelligence.js +88 -8
- package/dist/core/regression-intelligence.js.map +1 -1
- package/dist/core/remediation-planner.js +112 -40
- package/dist/core/remediation-planner.js.map +1 -1
- package/dist/core/review-history.js +39 -7
- package/dist/core/review-history.js.map +1 -1
- package/dist/core/run-history-output.js.map +1 -1
- package/dist/core/run-replay.js +12 -10
- package/dist/core/run-replay.js.map +1 -1
- package/dist/core/run-store.js +48 -21
- package/dist/core/run-store.js.map +1 -1
- package/dist/core/sandbox-cleanup-audit.js +1 -1
- package/dist/core/sandbox-cleanup-audit.js.map +1 -1
- package/dist/core/sandbox-compliance.js +53 -9
- package/dist/core/sandbox-compliance.js.map +1 -1
- package/dist/core/sandbox-lifecycle.js +1 -1
- package/dist/core/sandbox-lifecycle.js.map +1 -1
- package/dist/core/sandbox-policy.js +27 -3
- package/dist/core/sandbox-policy.js.map +1 -1
- package/dist/core/sandbox-runtime.js +9 -5
- package/dist/core/sandbox-runtime.js.map +1 -1
- package/dist/core/sandbox-security-profiles.js +31 -12
- package/dist/core/sandbox-security-profiles.js.map +1 -1
- package/dist/core/sarif-formatter.d.ts +10 -3
- package/dist/core/sarif-formatter.js +172 -57
- package/dist/core/sarif-formatter.js.map +1 -1
- package/dist/core/security-scanner.js +40 -32
- package/dist/core/security-scanner.js.map +1 -1
- package/dist/core/skill-registry.js +83 -21
- package/dist/core/skill-registry.js.map +1 -1
- package/dist/core/staged-verification.js +25 -10
- package/dist/core/staged-verification.js.map +1 -1
- package/dist/core/verification-history.js +12 -7
- package/dist/core/verification-history.js.map +1 -1
- package/dist/core/verification.js +50 -19
- package/dist/core/verification.js.map +1 -1
- package/dist/core/workflow-store.js +16 -4
- package/dist/core/workflow-store.js.map +1 -1
- package/dist/core/workspace-browser.js +57 -10
- package/dist/core/workspace-browser.js.map +1 -1
- package/dist/core/workspace-scan-policy.js +26 -4
- package/dist/core/workspace-scan-policy.js.map +1 -1
- package/dist/core/workspace-tool-broker.d.ts +1 -1
- package/dist/core/workspace-tool-broker.js +25 -4
- package/dist/core/workspace-tool-broker.js.map +1 -1
- package/dist/mcp/server.js +148 -55
- package/dist/mcp/server.js.map +1 -1
- package/dist/mcp/workspace-path.js +2 -2
- package/dist/mcp/workspace-path.js.map +1 -1
- package/package.json +8 -5
|
@@ -9,35 +9,62 @@ function summarize(trials) {
|
|
|
9
9
|
bucket.push(trial);
|
|
10
10
|
buckets.set(key, bucket);
|
|
11
11
|
}
|
|
12
|
-
return [...buckets.entries()]
|
|
13
|
-
|
|
12
|
+
return [...buckets.entries()]
|
|
13
|
+
.map(([key, items]) => {
|
|
14
|
+
const scores = items.map((item) => item.score);
|
|
14
15
|
const judged = items.filter((item) => item.judgeScore !== undefined && item.judgePassed !== undefined);
|
|
15
16
|
const average = scores.reduce((sum, value) => sum + value, 0) / scores.length;
|
|
16
17
|
return {
|
|
17
|
-
key,
|
|
18
|
+
key,
|
|
19
|
+
provider: items[0].provider,
|
|
20
|
+
models: items[0].models,
|
|
21
|
+
trials: items.length,
|
|
18
22
|
averageScore: Math.round(average * 10) / 10,
|
|
19
|
-
...(judged.length
|
|
20
|
-
|
|
21
|
-
|
|
23
|
+
...(judged.length
|
|
24
|
+
? {
|
|
25
|
+
averageJudgeScore: Math.round((judged.reduce((sum, item) => sum + item.judgeScore, 0) / judged.length) * 10) / 10,
|
|
26
|
+
judgePassRate: Math.round((judged.filter((item) => item.judgePassed).length / judged.length) * 1000) / 10,
|
|
27
|
+
}
|
|
28
|
+
: {}),
|
|
29
|
+
bestScore: Math.max(...scores),
|
|
30
|
+
worstScore: Math.min(...scores),
|
|
31
|
+
passRate: Math.round((items.filter((item) => item.passed).length / items.length) * 1000) / 10,
|
|
22
32
|
scoreStdDev: Math.round(Math.sqrt(scores.reduce((sum, value) => sum + Math.pow(value - average, 2), 0) / scores.length) * 10) / 10,
|
|
23
33
|
averageDurationMs: Math.round(items.reduce((sum, item) => sum + item.durationMs, 0) / items.length),
|
|
24
34
|
averageEstimatedTokens: Math.round(items.reduce((sum, item) => sum + item.estimatedInputTokens + item.estimatedOutputTokens, 0) / items.length),
|
|
25
35
|
totalRetries: items.reduce((sum, item) => sum + item.retries, 0),
|
|
26
36
|
totalFallbacks: items.reduce((sum, item) => sum + item.fallbacks, 0),
|
|
27
37
|
};
|
|
28
|
-
})
|
|
38
|
+
})
|
|
39
|
+
.sort((a, b) => b.averageScore - a.averageScore || a.averageDurationMs - b.averageDurationMs);
|
|
29
40
|
}
|
|
30
41
|
export function createBenchmarkTrial(report, trace) {
|
|
31
42
|
if (report.runId !== trace.runId)
|
|
32
43
|
throw new Error("Evaluation report and execution trace must belong to the same run.");
|
|
33
|
-
const providers = [...new Set(trace.entries.map(entry => entry.provider))].sort();
|
|
34
|
-
const models = [...new Set(trace.entries.map(entry => entry.model))].sort();
|
|
44
|
+
const providers = [...new Set(trace.entries.map((entry) => entry.provider))].sort();
|
|
45
|
+
const models = [...new Set(trace.entries.map((entry) => entry.model))].sort();
|
|
35
46
|
return {
|
|
36
|
-
runId: report.runId,
|
|
37
|
-
|
|
38
|
-
|
|
47
|
+
runId: report.runId,
|
|
48
|
+
recordedAt: new Date().toISOString(),
|
|
49
|
+
suiteId: report.suiteId,
|
|
50
|
+
suiteName: report.suiteName,
|
|
51
|
+
score: report.score,
|
|
52
|
+
...("judge" in report
|
|
53
|
+
? {
|
|
54
|
+
judgeScore: report.judge.averageScore,
|
|
55
|
+
judgeProvider: report.judge.provider,
|
|
56
|
+
judgePassed: report.judge.passed,
|
|
57
|
+
}
|
|
58
|
+
: {}),
|
|
59
|
+
passed: report.passed,
|
|
60
|
+
provider: providers.join(", ") || "unknown",
|
|
61
|
+
models,
|
|
62
|
+
tasks: trace.totals.tasks,
|
|
63
|
+
failures: trace.totals.failures,
|
|
64
|
+
fallbacks: trace.totals.fallbacks,
|
|
39
65
|
retries: trace.entries.reduce((sum, entry) => sum + (entry.retries ?? 0), 0),
|
|
40
|
-
estimatedInputTokens: trace.totals.estimatedInputTokens,
|
|
66
|
+
estimatedInputTokens: trace.totals.estimatedInputTokens,
|
|
67
|
+
estimatedOutputTokens: trace.totals.estimatedOutputTokens,
|
|
41
68
|
durationMs: trace.totals.durationMs,
|
|
42
69
|
};
|
|
43
70
|
}
|
|
@@ -48,7 +75,10 @@ export async function recordBenchmarkTrial(projectPath, trial) {
|
|
|
48
75
|
let trials = [];
|
|
49
76
|
try {
|
|
50
77
|
const previous = JSON.parse(await readFile(destination, "utf8"));
|
|
51
|
-
if (!previous ||
|
|
78
|
+
if (!previous ||
|
|
79
|
+
typeof previous !== "object" ||
|
|
80
|
+
previous.version !== 1 ||
|
|
81
|
+
!Array.isArray(previous.trials))
|
|
52
82
|
throw new Error("Existing benchmark history is invalid.");
|
|
53
83
|
trials = previous.trials;
|
|
54
84
|
}
|
|
@@ -56,8 +86,15 @@ export async function recordBenchmarkTrial(projectPath, trial) {
|
|
|
56
86
|
if (!(error && typeof error === "object" && "code" in error && error.code === "ENOENT"))
|
|
57
87
|
throw error;
|
|
58
88
|
}
|
|
59
|
-
trials = [...trials.filter(item => item.runId !== trial.runId), trial].sort((a, b) => a.recordedAt.localeCompare(b.recordedAt));
|
|
60
|
-
const document = {
|
|
89
|
+
trials = [...trials.filter((item) => item.runId !== trial.runId), trial].sort((a, b) => a.recordedAt.localeCompare(b.recordedAt));
|
|
90
|
+
const document = {
|
|
91
|
+
version: 1,
|
|
92
|
+
suiteId: trial.suiteId,
|
|
93
|
+
suiteName: trial.suiteName,
|
|
94
|
+
updatedAt: new Date().toISOString(),
|
|
95
|
+
trials,
|
|
96
|
+
groups: summarize(trials),
|
|
97
|
+
};
|
|
61
98
|
const temporary = path.join(directory, "." + randomUUID() + ".tmp");
|
|
62
99
|
try {
|
|
63
100
|
await writeFile(temporary, JSON.stringify(document, null, 2) + "\n", { encoding: "utf8", flag: "wx", mode: 0o600 });
|
|
@@ -74,7 +111,11 @@ export async function loadBenchmark(projectPath, suiteId) {
|
|
|
74
111
|
throw new Error("Invalid benchmark suite identifier.");
|
|
75
112
|
const file = path.join(path.resolve(projectPath), ".forge", "benchmarks", suiteId + ".json");
|
|
76
113
|
const value = JSON.parse(await readFile(file, "utf8"));
|
|
77
|
-
if (!value ||
|
|
114
|
+
if (!value ||
|
|
115
|
+
typeof value !== "object" ||
|
|
116
|
+
value.version !== 1 ||
|
|
117
|
+
value.suiteId !== suiteId ||
|
|
118
|
+
!Array.isArray(value.trials))
|
|
78
119
|
throw new Error("Stored benchmark document is invalid.");
|
|
79
120
|
return value;
|
|
80
121
|
}
|
|
@@ -82,7 +123,7 @@ export async function listBenchmarks(projectPath) {
|
|
|
82
123
|
const directory = path.join(path.resolve(projectPath), ".forge", "benchmarks");
|
|
83
124
|
let files;
|
|
84
125
|
try {
|
|
85
|
-
files = (await readdir(directory)).filter(file => file.endsWith(".json")).sort();
|
|
126
|
+
files = (await readdir(directory)).filter((file) => file.endsWith(".json")).sort();
|
|
86
127
|
}
|
|
87
128
|
catch (error) {
|
|
88
129
|
if (error && typeof error === "object" && "code" in error && error.code === "ENOENT")
|
|
@@ -97,12 +138,39 @@ export async function listBenchmarks(projectPath) {
|
|
|
97
138
|
return documents;
|
|
98
139
|
}
|
|
99
140
|
export function formatBenchmark(document) {
|
|
100
|
-
const lines = [
|
|
141
|
+
const lines = [
|
|
142
|
+
"ANVIL Benchmark Lab · " + document.suiteName,
|
|
143
|
+
"Trials: " + document.trials.length,
|
|
144
|
+
"",
|
|
145
|
+
"Provider / model | Trials | Avg rubric | Avg judge | Pass rate | Variance | Avg time | Avg tokens",
|
|
146
|
+
];
|
|
101
147
|
for (const group of document.groups)
|
|
102
|
-
lines.push([
|
|
148
|
+
lines.push([
|
|
149
|
+
group.key,
|
|
150
|
+
group.trials,
|
|
151
|
+
group.averageScore + "/100",
|
|
152
|
+
group.averageJudgeScore === undefined
|
|
153
|
+
? "—"
|
|
154
|
+
: group.averageJudgeScore + "/100 (" + group.judgePassRate + "% pass)",
|
|
155
|
+
group.passRate + "%",
|
|
156
|
+
"σ " + group.scoreStdDev,
|
|
157
|
+
group.averageDurationMs + " ms",
|
|
158
|
+
group.averageEstimatedTokens,
|
|
159
|
+
].join(" | "));
|
|
103
160
|
lines.push("", "Recent trials:");
|
|
104
161
|
for (const trial of document.trials.slice(-10).reverse())
|
|
105
|
-
lines.push(" " +
|
|
162
|
+
lines.push(" " +
|
|
163
|
+
trial.runId +
|
|
164
|
+
" · " +
|
|
165
|
+
trial.provider +
|
|
166
|
+
" / " +
|
|
167
|
+
(trial.models.join(", ") || "unknown") +
|
|
168
|
+
" · " +
|
|
169
|
+
trial.score +
|
|
170
|
+
"/100 rubric" +
|
|
171
|
+
(trial.judgeScore === undefined ? "" : " · " + trial.judgeScore + "/100 judge") +
|
|
172
|
+
" · " +
|
|
173
|
+
(trial.passed ? "PASS" : "REVIEW"));
|
|
106
174
|
lines.push("", "Token counts are estimates, not provider-billed usage. Benchmarks compare recorded runs; they do not automatically rerun tasks.");
|
|
107
175
|
return lines.join("\n");
|
|
108
176
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"benchmark-engine.js","sourceRoot":"","sources":["../../src/core/benchmark-engine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAkD7B,SAAS,SAAS,CAAC,MAAwB;IACzC,MAAM,OAAO,GAAG,IAAI,GAAG,EAA4B,CAAC;IACpD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,GAAG,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"benchmark-engine.js","sourceRoot":"","sources":["../../src/core/benchmark-engine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAkD7B,SAAS,SAAS,CAAC,MAAwB;IACzC,MAAM,OAAO,GAAG,IAAI,GAAG,EAA4B,CAAC;IACpD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,GAAG,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;SAC1B,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QACpB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CACzB,CAAC,IAAI,EAAyE,EAAE,CAC9E,IAAI,CAAC,UAAU,KAAK,SAAS,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,CAClE,CAAC;QACF,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAC9E,OAAO;YACL,GAAG;YACH,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,QAAQ;YAC5B,MAAM,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,MAAM;YACxB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,GAAG,EAAE;YAC3C,GAAG,CAAC,MAAM,CAAC,MAAM;gBACf,CAAC,CAAC;oBACE,iBAAiB,EACf,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE;oBAChG,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE;iBAC1G;gBACH,CAAC,CAAC,EAAE,CAAC;YACP,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;YAC9B,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;YAC/B,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE;YAC7F,WAAW,EACT,IAAI,CAAC,KAAK,CACR,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CACrG,GAAG,EAAE;YACR,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;YACnG,sBAAsB,EAAE,IAAI,CAAC,KAAK,CAChC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAC5G;YACD,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YAChE,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;SACrE,CAAC;IACJ,CAAC,CAAC;SACD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,iBAAiB,GAAG,CAAC,CAAC,iBAAiB,CAAC,CAAC;AAClG,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,MAAiD,EACjD,KAA6B;IAE7B,IAAI,MAAM,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK;QAC9B,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;IACxF,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACpF,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9E,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACpC,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,GAAG,CAAC,OAAO,IAAI,MAAM;YACnB,CAAC,CAAC;gBACE,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY;gBACrC,aAAa,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ;gBACpC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM;aACjC;YACH,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS;QAC3C,MAAM;QACN,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK;QACzB,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,QAAQ;QAC/B,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;QACjC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QAC5E,oBAAoB,EAAE,KAAK,CAAC,MAAM,CAAC,oBAAoB;QACvD,qBAAqB,EAAE,KAAK,CAAC,MAAM,CAAC,qBAAqB;QACzD,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU;KACpC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,WAAmB,EAAE,KAAqB;IACnF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC/E,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC,CAAC;IAClE,IAAI,MAAM,GAAqB,EAAE,CAAC;IAClC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAY,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC;QAC1E,IACE,CAAC,QAAQ;YACT,OAAO,QAAQ,KAAK,QAAQ;YAC3B,QAA8B,CAAC,OAAO,KAAK,CAAC;YAC7C,CAAC,KAAK,CAAC,OAAO,CAAE,QAA8B,CAAC,MAAM,CAAC;YAEtD,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC5D,MAAM,GAAI,QAA8B,CAAC,MAAM,CAAC;IAClD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC;YAAE,MAAM,KAAK,CAAC;IACvG,CAAC;IACD,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACrF,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CACzC,CAAC;IACF,MAAM,QAAQ,GAAsB;QAClC,OAAO,EAAE,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM;QACN,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC;KAC1B,CAAC;IACF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,GAAG,UAAU,EAAE,GAAG,MAAM,CAAC,CAAC;IACpE,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACpH,MAAM,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC5D,MAAM,KAAK,CAAC;IACd,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,WAAmB,EAAE,OAAe;IACtE,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACvG,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,GAAG,OAAO,CAAC,CAAC;IAC7F,MAAM,KAAK,GAAY,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAChE,IACE,CAAC,KAAK;QACN,OAAO,KAAK,KAAK,QAAQ;QACxB,KAA2B,CAAC,OAAO,KAAK,CAAC;QACzC,KAA2B,CAAC,OAAO,KAAK,OAAO;QAChD,CAAC,KAAK,CAAC,OAAO,CAAE,KAA2B,CAAC,MAAM,CAAC;QAEnD,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,OAAO,KAA0B,CAAC;AACpC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,WAAmB;IACtD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC/E,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,CAAC,MAAM,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACrF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAC;QAChG,MAAM,KAAK,CAAC;IACd,CAAC;IACD,MAAM,SAAS,GAAwB,EAAE,CAAC;IAC1C,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7B,SAAS,CAAC,IAAI,CAAC,MAAM,aAAa,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,QAA2B;IACzD,MAAM,KAAK,GAAG;QACZ,wBAAwB,GAAG,QAAQ,CAAC,SAAS;QAC7C,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM;QACnC,EAAE;QACF,mGAAmG;KACpG,CAAC;IACF,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM;QACjC,KAAK,CAAC,IAAI,CACR;YACE,KAAK,CAAC,GAAG;YACT,KAAK,CAAC,MAAM;YACZ,KAAK,CAAC,YAAY,GAAG,MAAM;YAC3B,KAAK,CAAC,iBAAiB,KAAK,SAAS;gBACnC,CAAC,CAAC,GAAG;gBACL,CAAC,CAAC,KAAK,CAAC,iBAAiB,GAAG,QAAQ,GAAG,KAAK,CAAC,aAAa,GAAG,SAAS;YACxE,KAAK,CAAC,QAAQ,GAAG,GAAG;YACpB,IAAI,GAAG,KAAK,CAAC,WAAW;YACxB,KAAK,CAAC,iBAAiB,GAAG,KAAK;YAC/B,KAAK,CAAC,sBAAsB;SAC7B,CAAC,IAAI,CAAC,KAAK,CAAC,CACd,CAAC;IACJ,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IACjC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE;QACtD,KAAK,CAAC,IAAI,CACR,IAAI;YACF,KAAK,CAAC,KAAK;YACX,KAAK;YACL,KAAK,CAAC,QAAQ;YACd,KAAK;YACL,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC;YACtC,KAAK;YACL,KAAK,CAAC,KAAK;YACX,aAAa;YACb,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,UAAU,GAAG,YAAY,CAAC;YAC/E,KAAK;YACL,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CACrC,CAAC;IACJ,KAAK,CAAC,IAAI,CACR,EAAE,EACF,iIAAiI,CAClI,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
|
|
@@ -25,27 +25,38 @@ export function validateBenchmarkMatrix(value) {
|
|
|
25
25
|
throw new Error("Candidate configVersionId must be a valid agent configuration snapshot ID.");
|
|
26
26
|
}
|
|
27
27
|
for (const candidate of config.candidates) {
|
|
28
|
-
if (candidate.skillIds !== undefined &&
|
|
28
|
+
if (candidate.skillIds !== undefined &&
|
|
29
|
+
(!Array.isArray(candidate.skillIds) ||
|
|
30
|
+
candidate.skillIds.some((id) => typeof id !== "string" || !/^[a-z0-9][a-z0-9-]{0,63}$/.test(id)) ||
|
|
31
|
+
new Set(candidate.skillIds).size !== candidate.skillIds.length))
|
|
29
32
|
throw new Error("Candidate skillIds must be unique valid skill IDs.");
|
|
30
33
|
}
|
|
31
34
|
if (config.skillExperiment) {
|
|
32
35
|
const experiment = config.skillExperiment;
|
|
33
|
-
const baseline = config.candidates.find(item => item.id === experiment.baselineCandidateId);
|
|
34
|
-
const treatment = config.candidates.find(item => item.id === experiment.treatmentCandidateId);
|
|
36
|
+
const baseline = config.candidates.find((item) => item.id === experiment.baselineCandidateId);
|
|
37
|
+
const treatment = config.candidates.find((item) => item.id === experiment.treatmentCandidateId);
|
|
35
38
|
if (!baseline || !treatment || baseline.id === treatment.id)
|
|
36
39
|
throw new Error("Skill experiment must reference distinct baseline and treatment candidates.");
|
|
37
40
|
if (baseline.provider !== treatment.provider || baseline.model !== treatment.model)
|
|
38
41
|
throw new Error("Skill experiment candidates must use the same provider and model.");
|
|
39
|
-
if (!Array.isArray(experiment.skillIds) ||
|
|
42
|
+
if (!Array.isArray(experiment.skillIds) ||
|
|
43
|
+
!experiment.skillIds.length ||
|
|
44
|
+
experiment.skillIds.some((id) => typeof id !== "string" || !/^[a-z0-9][a-z0-9-]{0,63}$/.test(id)) ||
|
|
45
|
+
new Set(experiment.skillIds).size !== experiment.skillIds.length)
|
|
40
46
|
throw new Error("Skill experiment must specify unique valid skill IDs.");
|
|
41
|
-
if (!Array.isArray(baseline.skillIds) ||
|
|
47
|
+
if (!Array.isArray(baseline.skillIds) ||
|
|
48
|
+
baseline.skillIds.length !== 0 ||
|
|
49
|
+
!Array.isArray(treatment.skillIds) ||
|
|
50
|
+
[...treatment.skillIds].sort().join(",") !== [...experiment.skillIds].sort().join(","))
|
|
42
51
|
throw new Error("Baseline must set skillIds to [] and treatment skillIds must exactly match skillExperiment.skillIds.");
|
|
43
52
|
}
|
|
44
53
|
return config;
|
|
45
54
|
}
|
|
46
55
|
export function fingerprintAgentConfig(files) {
|
|
47
|
-
const canonical = [...files]
|
|
48
|
-
.
|
|
56
|
+
const canonical = [...files]
|
|
57
|
+
.sort((a, b) => a.path.localeCompare(b.path))
|
|
58
|
+
.map((file) => file.path + ":" + file.sha256)
|
|
59
|
+
.join("\\n");
|
|
49
60
|
return createHash("sha256").update(canonical, "utf8").digest("hex");
|
|
50
61
|
}
|
|
51
62
|
export async function loadBenchmarkMatrix(file) {
|
|
@@ -58,52 +69,155 @@ export async function executeBenchmarkMatrix(config, execute) {
|
|
|
58
69
|
for (let repeat = 1; repeat <= config.repeats; repeat += 1) {
|
|
59
70
|
try {
|
|
60
71
|
const result = await execute(candidate, repeat);
|
|
61
|
-
results.push({
|
|
72
|
+
results.push({
|
|
73
|
+
candidateId: candidate.id,
|
|
74
|
+
provider: candidate.provider,
|
|
75
|
+
model: candidate.model,
|
|
76
|
+
repeat,
|
|
77
|
+
status: "completed",
|
|
78
|
+
...result,
|
|
79
|
+
});
|
|
62
80
|
}
|
|
63
81
|
catch (error) {
|
|
64
|
-
results.push({
|
|
82
|
+
results.push({
|
|
83
|
+
candidateId: candidate.id,
|
|
84
|
+
provider: candidate.provider,
|
|
85
|
+
model: candidate.model,
|
|
86
|
+
repeat,
|
|
87
|
+
runId: "",
|
|
88
|
+
score: 0,
|
|
89
|
+
passed: false,
|
|
90
|
+
durationMs: 0,
|
|
91
|
+
estimatedTokens: 0,
|
|
92
|
+
status: "failed",
|
|
93
|
+
error: error instanceof Error ? error.message : String(error),
|
|
94
|
+
});
|
|
65
95
|
}
|
|
66
96
|
}
|
|
67
97
|
}
|
|
68
|
-
const summary = config.candidates
|
|
69
|
-
|
|
70
|
-
const
|
|
98
|
+
const summary = config.candidates
|
|
99
|
+
.map((candidate) => {
|
|
100
|
+
const items = results.filter((item) => item.candidateId === candidate.id && item.status === "completed");
|
|
101
|
+
const scores = items.map((item) => item.score);
|
|
71
102
|
const average = scores.length ? scores.reduce((sum, score) => sum + score, 0) / scores.length : 0;
|
|
72
103
|
return {
|
|
73
|
-
candidateId: candidate.id,
|
|
104
|
+
candidateId: candidate.id,
|
|
105
|
+
provider: candidate.provider,
|
|
106
|
+
model: candidate.model,
|
|
107
|
+
trials: config.repeats,
|
|
74
108
|
completed: items.length,
|
|
75
109
|
averageScore: Math.round(average * 10) / 10,
|
|
76
|
-
passRate: items.length
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
110
|
+
passRate: items.length
|
|
111
|
+
? Math.round((items.filter((item) => item.passed).length / items.length) * 1000) / 10
|
|
112
|
+
: 0,
|
|
113
|
+
scoreStdDev: items.length
|
|
114
|
+
? Math.round(Math.sqrt(scores.reduce((sum, score) => sum + Math.pow(score - average, 2), 0) / scores.length) * 10) / 10
|
|
115
|
+
: 0,
|
|
116
|
+
averageDurationMs: items.length
|
|
117
|
+
? Math.round(items.reduce((sum, item) => sum + item.durationMs, 0) / items.length)
|
|
118
|
+
: 0,
|
|
119
|
+
averageEstimatedTokens: items.length
|
|
120
|
+
? Math.round(items.reduce((sum, item) => sum + item.estimatedTokens, 0) / items.length)
|
|
121
|
+
: 0,
|
|
80
122
|
};
|
|
81
|
-
})
|
|
82
|
-
|
|
83
|
-
const
|
|
123
|
+
})
|
|
124
|
+
.sort((a, b) => b.averageScore - a.averageScore || a.averageDurationMs - b.averageDurationMs);
|
|
125
|
+
const agentKeys = new Set(results.flatMap((item) => (item.agentScores ?? []).map((score) => item.candidateId + "::" + score.agent)));
|
|
126
|
+
const agentSummary = [...agentKeys]
|
|
127
|
+
.map((key) => {
|
|
84
128
|
const separator = key.indexOf("::");
|
|
85
129
|
const candidateId = key.slice(0, separator);
|
|
86
130
|
const agent = key.slice(separator + 2);
|
|
87
|
-
const candidate = config.candidates.find(item => item.id === candidateId);
|
|
88
|
-
const scores = results
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
131
|
+
const candidate = config.candidates.find((item) => item.id === candidateId);
|
|
132
|
+
const scores = results
|
|
133
|
+
.filter((item) => item.candidateId === candidateId && item.status === "completed")
|
|
134
|
+
.flatMap((item) => (item.agentScores ?? []).filter((score) => score.agent === agent));
|
|
135
|
+
const skillsObserved = [...new Set(scores.flatMap((item) => item.skillIds))].sort();
|
|
136
|
+
return {
|
|
137
|
+
candidateId,
|
|
138
|
+
provider: candidate.provider,
|
|
139
|
+
model: candidate.model,
|
|
140
|
+
agent,
|
|
141
|
+
trials: config.repeats,
|
|
142
|
+
completed: scores.length,
|
|
143
|
+
averageScore: scores.length
|
|
144
|
+
? Math.round((scores.reduce((sum, item) => sum + item.score, 0) / scores.length) * 10) / 10
|
|
145
|
+
: 0,
|
|
146
|
+
passRate: scores.length
|
|
147
|
+
? Math.round((scores.filter((item) => item.passed).length / scores.length) * 1000) / 10
|
|
148
|
+
: 0,
|
|
149
|
+
skillsObserved,
|
|
150
|
+
};
|
|
151
|
+
})
|
|
152
|
+
.sort((a, b) => b.averageScore - a.averageScore || a.agent.localeCompare(b.agent));
|
|
92
153
|
let skillExperiment;
|
|
93
154
|
if (config.skillExperiment) {
|
|
94
155
|
const { baselineCandidateId, treatmentCandidateId, skillIds } = config.skillExperiment;
|
|
95
|
-
const baseline = summary.find(item => item.candidateId === baselineCandidateId);
|
|
96
|
-
const treatment = summary.find(item => item.candidateId === treatmentCandidateId);
|
|
156
|
+
const baseline = summary.find((item) => item.candidateId === baselineCandidateId);
|
|
157
|
+
const treatment = summary.find((item) => item.candidateId === treatmentCandidateId);
|
|
97
158
|
if (baseline && treatment && baseline.completed > 0 && treatment.completed > 0) {
|
|
98
|
-
const change = (before, after) => before <= 0 ? (after > 0 ? 100 : 0) : Math.round((after - before) / before * 1000) / 10;
|
|
99
|
-
const agentDeltas =
|
|
100
|
-
|
|
101
|
-
|
|
159
|
+
const change = (before, after) => before <= 0 ? (after > 0 ? 100 : 0) : Math.round(((after - before) / before) * 1000) / 10;
|
|
160
|
+
const agentDeltas = agentSummary
|
|
161
|
+
.filter((item) => item.candidateId === baselineCandidateId)
|
|
162
|
+
.flatMap((before) => {
|
|
163
|
+
const after = agentSummary.find((item) => item.candidateId === treatmentCandidateId && item.agent === before.agent);
|
|
164
|
+
return after
|
|
165
|
+
? [
|
|
166
|
+
{
|
|
167
|
+
agent: before.agent,
|
|
168
|
+
baselineScore: before.averageScore,
|
|
169
|
+
treatmentScore: after.averageScore,
|
|
170
|
+
scoreDelta: Math.round((after.averageScore - before.averageScore) * 10) / 10,
|
|
171
|
+
baselinePassRate: before.passRate,
|
|
172
|
+
treatmentPassRate: after.passRate,
|
|
173
|
+
passRateDelta: Math.round((after.passRate - before.passRate) * 10) / 10,
|
|
174
|
+
},
|
|
175
|
+
]
|
|
176
|
+
: [];
|
|
102
177
|
});
|
|
103
|
-
skillExperiment = {
|
|
178
|
+
skillExperiment = {
|
|
179
|
+
baselineCandidateId,
|
|
180
|
+
treatmentCandidateId,
|
|
181
|
+
skillIds: [...skillIds],
|
|
182
|
+
baselineScore: baseline.averageScore,
|
|
183
|
+
treatmentScore: treatment.averageScore,
|
|
184
|
+
scoreDelta: Math.round((treatment.averageScore - baseline.averageScore) * 10) / 10,
|
|
185
|
+
baselinePassRate: baseline.passRate,
|
|
186
|
+
treatmentPassRate: treatment.passRate,
|
|
187
|
+
passRateDelta: Math.round((treatment.passRate - baseline.passRate) * 10) / 10,
|
|
188
|
+
baselineDurationMs: baseline.averageDurationMs,
|
|
189
|
+
treatmentDurationMs: treatment.averageDurationMs,
|
|
190
|
+
durationChangePercent: change(baseline.averageDurationMs, treatment.averageDurationMs),
|
|
191
|
+
baselineEstimatedTokens: baseline.averageEstimatedTokens,
|
|
192
|
+
treatmentEstimatedTokens: treatment.averageEstimatedTokens,
|
|
193
|
+
tokenChangePercent: change(baseline.averageEstimatedTokens, treatment.averageEstimatedTokens),
|
|
194
|
+
agentDeltas,
|
|
195
|
+
};
|
|
104
196
|
}
|
|
105
197
|
}
|
|
106
|
-
return {
|
|
198
|
+
return {
|
|
199
|
+
version: 1,
|
|
200
|
+
matrixId: config.id,
|
|
201
|
+
matrixName: config.name,
|
|
202
|
+
suiteId: "",
|
|
203
|
+
goal: "",
|
|
204
|
+
createdAt: new Date().toISOString(),
|
|
205
|
+
results,
|
|
206
|
+
summary,
|
|
207
|
+
agentSummary,
|
|
208
|
+
...(config.candidates.some((candidate) => candidate.configVersionId)
|
|
209
|
+
? {
|
|
210
|
+
candidateProvenance: config.candidates
|
|
211
|
+
.filter((candidate) => candidate.configVersionId)
|
|
212
|
+
.map((candidate) => ({
|
|
213
|
+
candidateId: candidate.id,
|
|
214
|
+
configVersionId: candidate.configVersionId,
|
|
215
|
+
fingerprint: "",
|
|
216
|
+
})),
|
|
217
|
+
}
|
|
218
|
+
: {}),
|
|
219
|
+
...(skillExperiment ? { skillExperiment } : {}),
|
|
220
|
+
};
|
|
107
221
|
}
|
|
108
222
|
export async function saveBenchmarkMatrix(projectPath, report) {
|
|
109
223
|
const directory = path.join(path.resolve(projectPath), ".forge", "benchmark-matrices");
|
|
@@ -121,23 +235,79 @@ export async function saveBenchmarkMatrix(projectPath, report) {
|
|
|
121
235
|
return destination;
|
|
122
236
|
}
|
|
123
237
|
export function formatBenchmarkMatrix(report) {
|
|
124
|
-
const lines = [
|
|
238
|
+
const lines = [
|
|
239
|
+
"ANVIL Automated Benchmark Matrix · " + report.matrixName,
|
|
240
|
+
"Suite: " + report.suiteId,
|
|
241
|
+
"Goal: " + report.goal,
|
|
242
|
+
"",
|
|
243
|
+
"Candidate | Trials | Completed | Avg score | Pass rate | σ score | Avg time | Avg estimated tokens",
|
|
244
|
+
];
|
|
125
245
|
for (const item of report.summary)
|
|
126
|
-
lines.push([
|
|
246
|
+
lines.push([
|
|
247
|
+
item.candidateId + " (" + item.provider + "/" + item.model + ")",
|
|
248
|
+
item.trials,
|
|
249
|
+
item.completed,
|
|
250
|
+
item.averageScore + "/100",
|
|
251
|
+
item.passRate + "%",
|
|
252
|
+
item.scoreStdDev,
|
|
253
|
+
item.averageDurationMs + " ms",
|
|
254
|
+
item.averageEstimatedTokens,
|
|
255
|
+
].join(" | "));
|
|
127
256
|
if (report.agentSummary?.length) {
|
|
128
257
|
lines.push("", "Per-agent scorecards:", "Candidate | Agent | Cases | Average score | Pass rate | Skills observed");
|
|
129
258
|
for (const item of report.agentSummary)
|
|
130
|
-
lines.push([
|
|
259
|
+
lines.push([
|
|
260
|
+
item.candidateId,
|
|
261
|
+
item.agent,
|
|
262
|
+
item.completed,
|
|
263
|
+
item.averageScore + "/100",
|
|
264
|
+
item.passRate + "%",
|
|
265
|
+
item.skillsObserved.join(", ") || "none",
|
|
266
|
+
].join(" | "));
|
|
131
267
|
}
|
|
132
268
|
if (report.skillExperiment) {
|
|
133
269
|
const item = report.skillExperiment;
|
|
134
|
-
lines.push("", "Controlled skill experiment:", " Baseline: " + item.baselineCandidateId + " (no skills)", " Treatment: " + item.treatmentCandidateId + " (skills: " + item.skillIds.join(", ") + ")", " Score: " +
|
|
270
|
+
lines.push("", "Controlled skill experiment:", " Baseline: " + item.baselineCandidateId + " (no skills)", " Treatment: " + item.treatmentCandidateId + " (skills: " + item.skillIds.join(", ") + ")", " Score: " +
|
|
271
|
+
item.baselineScore +
|
|
272
|
+
" → " +
|
|
273
|
+
item.treatmentScore +
|
|
274
|
+
" (" +
|
|
275
|
+
(item.scoreDelta > 0 ? "+" : "") +
|
|
276
|
+
item.scoreDelta +
|
|
277
|
+
")", " Pass rate: " +
|
|
278
|
+
item.baselinePassRate +
|
|
279
|
+
"% → " +
|
|
280
|
+
item.treatmentPassRate +
|
|
281
|
+
"% (" +
|
|
282
|
+
(item.passRateDelta > 0 ? "+" : "") +
|
|
283
|
+
item.passRateDelta +
|
|
284
|
+
" pp)", " Duration change: " +
|
|
285
|
+
item.durationChangePercent +
|
|
286
|
+
"% · estimated token change: " +
|
|
287
|
+
item.tokenChangePercent +
|
|
288
|
+
"%");
|
|
135
289
|
for (const agent of item.agentDeltas)
|
|
136
|
-
lines.push(" Agent " +
|
|
290
|
+
lines.push(" Agent " +
|
|
291
|
+
agent.agent +
|
|
292
|
+
": " +
|
|
293
|
+
agent.baselineScore +
|
|
294
|
+
" → " +
|
|
295
|
+
agent.treatmentScore +
|
|
296
|
+
" (" +
|
|
297
|
+
(agent.scoreDelta > 0 ? "+" : "") +
|
|
298
|
+
agent.scoreDelta +
|
|
299
|
+
")");
|
|
137
300
|
}
|
|
138
301
|
lines.push("", "Trial details:");
|
|
139
302
|
for (const item of report.results)
|
|
140
|
-
lines.push(" " +
|
|
303
|
+
lines.push(" " +
|
|
304
|
+
item.candidateId +
|
|
305
|
+
" #" +
|
|
306
|
+
item.repeat +
|
|
307
|
+
" · " +
|
|
308
|
+
(item.status === "completed"
|
|
309
|
+
? item.score + "/100 · " + (item.passed ? "PASS" : "REVIEW")
|
|
310
|
+
: "FAILED · " + item.error));
|
|
141
311
|
if (report.candidateProvenance?.length) {
|
|
142
312
|
lines.push("", "Configuration provenance:");
|
|
143
313
|
for (const item of report.candidateProvenance)
|
|
@@ -150,7 +320,7 @@ export async function listBenchmarkMatrices(projectPath) {
|
|
|
150
320
|
const directory = path.join(path.resolve(projectPath), ".forge", "benchmark-matrices");
|
|
151
321
|
let files;
|
|
152
322
|
try {
|
|
153
|
-
files = (await readdir(directory)).filter(file => file.endsWith(".json")).sort();
|
|
323
|
+
files = (await readdir(directory)).filter((file) => file.endsWith(".json")).sort();
|
|
154
324
|
}
|
|
155
325
|
catch (error) {
|
|
156
326
|
if (error && typeof error === "object" && "code" in error && error.code === "ENOENT")
|
|
@@ -160,7 +330,11 @@ export async function listBenchmarkMatrices(projectPath) {
|
|
|
160
330
|
const reports = [];
|
|
161
331
|
for (const file of files.slice(-100)) {
|
|
162
332
|
const value = JSON.parse(await readFile(path.join(directory, file), "utf8"));
|
|
163
|
-
if (!value ||
|
|
333
|
+
if (!value ||
|
|
334
|
+
typeof value !== "object" ||
|
|
335
|
+
value.version !== 1 ||
|
|
336
|
+
!Array.isArray(value.results) ||
|
|
337
|
+
!Array.isArray(value.summary))
|
|
164
338
|
throw new Error("Stored benchmark matrix report is invalid: " + file);
|
|
165
339
|
reports.push(value);
|
|
166
340
|
}
|