loadout-ai 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/MASTER_PLAN.md +1711 -0
- package/README.md +453 -0
- package/SECURITY.md +18 -0
- package/SIMPLE_PLAN.md +44 -0
- package/catalog/discovered.json +10469 -0
- package/catalog/packages.json +1150 -0
- package/dashboard/app.js +607 -0
- package/dashboard/index.html +249 -0
- package/dashboard/styles.css +384 -0
- package/dist/src/cli.js +2735 -0
- package/dist/src/core/active-policy.js +233 -0
- package/dist/src/core/active-set.js +382 -0
- package/dist/src/core/adapters.js +254 -0
- package/dist/src/core/adopt.js +92 -0
- package/dist/src/core/agent-health-score.js +475 -0
- package/dist/src/core/agent-inspection.js +152 -0
- package/dist/src/core/agent-versions.js +131 -0
- package/dist/src/core/api.js +90 -0
- package/dist/src/core/atomic-file.js +22 -0
- package/dist/src/core/audit.js +174 -0
- package/dist/src/core/benchmark-campaign.js +496 -0
- package/dist/src/core/benchmark-evidence.js +458 -0
- package/dist/src/core/benchmark-fixtures.js +1157 -0
- package/dist/src/core/benchmark-runner.js +301 -0
- package/dist/src/core/benchmark-trust.js +795 -0
- package/dist/src/core/canary.js +77 -0
- package/dist/src/core/candidate-intelligence.js +661 -0
- package/dist/src/core/catalog-coverage.js +85 -0
- package/dist/src/core/catalog-install.js +186 -0
- package/dist/src/core/catalog-release.js +286 -0
- package/dist/src/core/catalog.js +327 -0
- package/dist/src/core/codex-mcp.js +122 -0
- package/dist/src/core/community.js +86 -0
- package/dist/src/core/compatibility-intelligence.js +465 -0
- package/dist/src/core/completion.js +160 -0
- package/dist/src/core/components.js +138 -0
- package/dist/src/core/conversion.js +56 -0
- package/dist/src/core/credentials.js +377 -0
- package/dist/src/core/demo.js +136 -0
- package/dist/src/core/diff.js +83 -0
- package/dist/src/core/discovery-connector.js +165 -0
- package/dist/src/core/doctor.js +95 -0
- package/dist/src/core/ecosystem-import.js +1072 -0
- package/dist/src/core/evaluate.js +72 -0
- package/dist/src/core/file-lock.js +140 -0
- package/dist/src/core/freshness-alerts.js +230 -0
- package/dist/src/core/github-discovery.js +97 -0
- package/dist/src/core/github.js +148 -0
- package/dist/src/core/head-to-head.js +437 -0
- package/dist/src/core/health-score-evidence.js +169 -0
- package/dist/src/core/health.js +129 -0
- package/dist/src/core/improve.js +223 -0
- package/dist/src/core/install.js +233 -0
- package/dist/src/core/intelligence-feed-build.js +59 -0
- package/dist/src/core/intelligence-feed.js +353 -0
- package/dist/src/core/loadout-badge.js +55 -0
- package/dist/src/core/loadout-card.js +87 -0
- package/dist/src/core/manifest.js +307 -0
- package/dist/src/core/mcp-recipes.js +419 -0
- package/dist/src/core/mcp-registry-discovery.js +455 -0
- package/dist/src/core/mcp.js +287 -0
- package/dist/src/core/model-config.js +102 -0
- package/dist/src/core/observations.js +87 -0
- package/dist/src/core/outcomes.js +164 -0
- package/dist/src/core/package.js +225 -0
- package/dist/src/core/paths.js +214 -0
- package/dist/src/core/portable.js +130 -0
- package/dist/src/core/private-discovery.js +45 -0
- package/dist/src/core/profiles.js +255 -0
- package/dist/src/core/provenance.js +278 -0
- package/dist/src/core/ranking.js +104 -0
- package/dist/src/core/recommend.js +181 -0
- package/dist/src/core/registry-api.js +97 -0
- package/dist/src/core/registry.js +316 -0
- package/dist/src/core/release-claims.js +175 -0
- package/dist/src/core/remove.js +122 -0
- package/dist/src/core/review-queue.js +200 -0
- package/dist/src/core/runtime-tool-recipe.js +315 -0
- package/dist/src/core/runtime-tools.js +453 -0
- package/dist/src/core/safety.js +234 -0
- package/dist/src/core/sandbox.js +78 -0
- package/dist/src/core/scheduler.js +264 -0
- package/dist/src/core/share-report.js +85 -0
- package/dist/src/core/signing.js +115 -0
- package/dist/src/core/skill-compare.js +239 -0
- package/dist/src/core/skill-inventory.js +193 -0
- package/dist/src/core/skill-security.js +340 -0
- package/dist/src/core/skills-sh-discovery.js +336 -0
- package/dist/src/core/skills.js +208 -0
- package/dist/src/core/snapshot.js +210 -0
- package/dist/src/core/source.js +305 -0
- package/dist/src/core/state.js +300 -0
- package/dist/src/core/sync.js +348 -0
- package/dist/src/core/terminal.js +53 -0
- package/dist/src/core/transaction.js +211 -0
- package/dist/src/core/update-watch.js +39 -0
- package/dist/src/core/update.js +216 -0
- package/dist/src/core/upgrade.js +207 -0
- package/dist/src/dashboard.js +418 -0
- package/dist/src/shared/schemas.js +336 -0
- package/dist/src/shared/types.js +1 -0
- package/docs/ACTIVE_SET.md +53 -0
- package/docs/ACTIVE_SET_POLICY.md +45 -0
- package/docs/CANDIDATE_INTELLIGENCE.md +139 -0
- package/docs/CATALOG.md +78 -0
- package/docs/CATALOG_POLICY.md +41 -0
- package/docs/COMMUNITY_DISCOVERY.md +14 -0
- package/docs/COMPATIBILITY_POLICY.md +22 -0
- package/docs/CONVERSION_AND_SANDBOX.md +27 -0
- package/docs/CREDENTIAL_AND_UPDATE_POLICY.md +31 -0
- package/docs/DISCOVERED.md +280 -0
- package/docs/EVALUATION_PROTOCOL.md +40 -0
- package/docs/EVALUATION_PROTOCOL_V1.md +300 -0
- package/docs/FEATURE_TEST_MATRIX.md +802 -0
- package/docs/GITHUB_AUTHORIZATION.md +38 -0
- package/docs/HEAD_TO_HEAD_EVALUATION.md +79 -0
- package/docs/PROVENANCE_AND_COMPARISON.md +72 -0
- package/docs/PROVIDER_CONFIGURATION.md +45 -0
- package/docs/RELEASE_REVIEW.md +117 -0
- package/docs/SAFE_UPDATE_DEMO.md +25 -0
- package/docs/SCHEMA_DECISIONS.md +25 -0
- package/docs/TEAM_POLICY.md +18 -0
- package/docs/TESTING.md +190 -0
- package/package.json +82 -0
|
@@ -0,0 +1,437 @@
|
|
|
1
|
+
import { mkdir, writeFile } from "node:fs/promises";
|
|
2
|
+
import { dirname, resolve } from "node:path";
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
4
|
+
import { signPayload, verifyEnvelope } from "./signing.js";
|
|
5
|
+
const CATEGORIES = new Set([
|
|
6
|
+
"workflow-adherence",
|
|
7
|
+
"code-review-coverage",
|
|
8
|
+
]);
|
|
9
|
+
const SEVERITIES = new Set([
|
|
10
|
+
"critical",
|
|
11
|
+
"high",
|
|
12
|
+
"medium",
|
|
13
|
+
"low",
|
|
14
|
+
]);
|
|
15
|
+
const MINIMUM_COMPARISON_TRIALS = 5;
|
|
16
|
+
const MINIMUM_PRACTICAL_SCORE_DELTA = 1;
|
|
17
|
+
const DIMENSION_MAXIMA = {
|
|
18
|
+
"workflow-adherence": {
|
|
19
|
+
"constraint-recall": 25,
|
|
20
|
+
"plan-to-action-consistency": 20,
|
|
21
|
+
"safe-scope": 20,
|
|
22
|
+
"verification-quality": 20,
|
|
23
|
+
"honest-completion-report": 15,
|
|
24
|
+
},
|
|
25
|
+
"code-review-coverage": {
|
|
26
|
+
"seeded-defect-recall": 45,
|
|
27
|
+
precision: 20,
|
|
28
|
+
"severity-calibration": 15,
|
|
29
|
+
"actionable-evidence": 10,
|
|
30
|
+
"regression-test-advice": 10,
|
|
31
|
+
},
|
|
32
|
+
};
|
|
33
|
+
function isRecord(value) {
|
|
34
|
+
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
35
|
+
}
|
|
36
|
+
function exactKeys(value, allowed, context) {
|
|
37
|
+
const permitted = new Set(allowed);
|
|
38
|
+
const unknown = Object.keys(value).filter((key) => !permitted.has(key));
|
|
39
|
+
if (unknown.length)
|
|
40
|
+
throw new Error(`${context} has unknown field(s): ${unknown.join(", ")}`);
|
|
41
|
+
}
|
|
42
|
+
function requiredString(value, context) {
|
|
43
|
+
if (typeof value !== "string" ||
|
|
44
|
+
!value.length ||
|
|
45
|
+
value !== value.trim() ||
|
|
46
|
+
value.length > 256)
|
|
47
|
+
throw new Error(`${context} must be a non-empty, trimmed string`);
|
|
48
|
+
}
|
|
49
|
+
function isoTimestamp(value, context) {
|
|
50
|
+
requiredString(value, context);
|
|
51
|
+
let normalized;
|
|
52
|
+
try {
|
|
53
|
+
normalized = new Date(value).toISOString();
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
throw new Error(`${context} must be an ISO-8601 UTC timestamp`);
|
|
57
|
+
}
|
|
58
|
+
if (normalized !== value)
|
|
59
|
+
throw new Error(`${context} must be an ISO-8601 UTC timestamp`);
|
|
60
|
+
}
|
|
61
|
+
function candidateId(value, context) {
|
|
62
|
+
requiredString(value, context);
|
|
63
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$/.test(value))
|
|
64
|
+
throw new Error(`${context} contains unsupported characters`);
|
|
65
|
+
}
|
|
66
|
+
function stringArray(value, context, options = {}) {
|
|
67
|
+
if (!Array.isArray(value) || (options.required && value.length === 0))
|
|
68
|
+
throw new Error(`${context} must be ${options.required ? "a non-empty" : "an"} array`);
|
|
69
|
+
for (const [index, item] of value.entries())
|
|
70
|
+
requiredString(item, `${context}[${index}]`);
|
|
71
|
+
if (options.unique) {
|
|
72
|
+
const normalizedItems = value.map((item) => item.toLowerCase());
|
|
73
|
+
if (new Set(normalizedItems).size !== normalizedItems.length)
|
|
74
|
+
throw new Error(`${context} must not contain duplicate values`);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
function nonNegativeNumber(value, context) {
|
|
78
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0)
|
|
79
|
+
throw new Error(`${context} must be a finite non-negative number`);
|
|
80
|
+
}
|
|
81
|
+
function optionalCount(value, context) {
|
|
82
|
+
if (value === undefined)
|
|
83
|
+
return;
|
|
84
|
+
nonNegativeNumber(value, context);
|
|
85
|
+
if (!Number.isInteger(value))
|
|
86
|
+
throw new Error(`${context} must be an integer`);
|
|
87
|
+
}
|
|
88
|
+
function validateFinding(value, context) {
|
|
89
|
+
if (!isRecord(value))
|
|
90
|
+
throw new Error(`${context} must be an object`);
|
|
91
|
+
exactKeys(value, ["id", "severity"], context);
|
|
92
|
+
requiredString(value.id, `${context}.id`);
|
|
93
|
+
if (typeof value.severity !== "string" ||
|
|
94
|
+
!SEVERITIES.has(value.severity))
|
|
95
|
+
throw new Error(`${context}.severity is invalid`);
|
|
96
|
+
return value;
|
|
97
|
+
}
|
|
98
|
+
function validateFindings(value, context, required = false) {
|
|
99
|
+
if (!Array.isArray(value) || (required && value.length === 0))
|
|
100
|
+
throw new Error(`${context} must be ${required ? "a non-empty" : "an"} array`);
|
|
101
|
+
const findings = value.map((item, index) => validateFinding(item, `${context}[${index}]`));
|
|
102
|
+
const ids = findings.map((finding) => finding.id.toLowerCase());
|
|
103
|
+
if (new Set(ids).size !== ids.length)
|
|
104
|
+
throw new Error(`${context} must not contain duplicate finding ids`);
|
|
105
|
+
return findings;
|
|
106
|
+
}
|
|
107
|
+
function validateFixture(value) {
|
|
108
|
+
if (!isRecord(value))
|
|
109
|
+
throw new Error("Head-to-head fixture must be an object");
|
|
110
|
+
exactKeys(value, [
|
|
111
|
+
"id",
|
|
112
|
+
"category",
|
|
113
|
+
"version",
|
|
114
|
+
"requiredActions",
|
|
115
|
+
"forbiddenActions",
|
|
116
|
+
"seededFindings",
|
|
117
|
+
], "Head-to-head fixture");
|
|
118
|
+
requiredString(value.id, "Head-to-head fixture id");
|
|
119
|
+
requiredString(value.version, "Head-to-head fixture version");
|
|
120
|
+
if (typeof value.category !== "string" ||
|
|
121
|
+
!CATEGORIES.has(value.category))
|
|
122
|
+
throw new Error("Head-to-head fixture category is invalid");
|
|
123
|
+
if (value.category === "workflow-adherence") {
|
|
124
|
+
stringArray(value.requiredActions, "requiredActions", {
|
|
125
|
+
required: true,
|
|
126
|
+
unique: true,
|
|
127
|
+
});
|
|
128
|
+
if (value.forbiddenActions !== undefined)
|
|
129
|
+
stringArray(value.forbiddenActions, "forbiddenActions", { unique: true });
|
|
130
|
+
if (value.seededFindings !== undefined)
|
|
131
|
+
throw new Error("workflow-adherence fixtures must not declare seededFindings");
|
|
132
|
+
}
|
|
133
|
+
else {
|
|
134
|
+
validateFindings(value.seededFindings, "seededFindings", true);
|
|
135
|
+
if (value.requiredActions !== undefined ||
|
|
136
|
+
value.forbiddenActions !== undefined)
|
|
137
|
+
throw new Error("code-review-coverage fixtures must not declare workflow actions");
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
function validateTrial(value, index) {
|
|
141
|
+
const context = `Head-to-head trial ${index + 1}`;
|
|
142
|
+
if (!isRecord(value))
|
|
143
|
+
throw new Error(`${context} must be an object`);
|
|
144
|
+
exactKeys(value, [
|
|
145
|
+
"candidateId",
|
|
146
|
+
"fixtureId",
|
|
147
|
+
"observations",
|
|
148
|
+
"findings",
|
|
149
|
+
"durationMs",
|
|
150
|
+
"inputTokens",
|
|
151
|
+
"outputTokens",
|
|
152
|
+
"reportedCostUsd",
|
|
153
|
+
], context);
|
|
154
|
+
candidateId(value.candidateId, `${context}.candidateId`);
|
|
155
|
+
requiredString(value.fixtureId, `${context}.fixtureId`);
|
|
156
|
+
stringArray(value.observations, `${context}.observations`);
|
|
157
|
+
if (value.findings !== undefined)
|
|
158
|
+
validateFindings(value.findings, `${context}.findings`);
|
|
159
|
+
nonNegativeNumber(value.durationMs, `${context}.durationMs`);
|
|
160
|
+
optionalCount(value.inputTokens, `${context}.inputTokens`);
|
|
161
|
+
optionalCount(value.outputTokens, `${context}.outputTokens`);
|
|
162
|
+
if (value.reportedCostUsd !== undefined)
|
|
163
|
+
nonNegativeNumber(value.reportedCostUsd, `${context}.reportedCostUsd`);
|
|
164
|
+
}
|
|
165
|
+
function normalized(items) {
|
|
166
|
+
return new Set(items.map((item) => item.trim().toLowerCase()).filter(Boolean));
|
|
167
|
+
}
|
|
168
|
+
function roundScore(value) {
|
|
169
|
+
return Math.round(value * 100) / 100;
|
|
170
|
+
}
|
|
171
|
+
function weightedScore(proportion, maximum) {
|
|
172
|
+
return roundScore(Math.max(0, Math.min(1, proportion)) * maximum);
|
|
173
|
+
}
|
|
174
|
+
function canonical(value) {
|
|
175
|
+
if (value === null || typeof value === "boolean" || typeof value === "string")
|
|
176
|
+
return JSON.stringify(value);
|
|
177
|
+
if (typeof value === "number") {
|
|
178
|
+
if (!Number.isFinite(value))
|
|
179
|
+
throw new Error("Cannot hash a non-finite fixture number");
|
|
180
|
+
return JSON.stringify(value);
|
|
181
|
+
}
|
|
182
|
+
if (Array.isArray(value))
|
|
183
|
+
return `[${value.map(canonical).join(",")}]`;
|
|
184
|
+
if (isRecord(value))
|
|
185
|
+
return `{${Object.entries(value)
|
|
186
|
+
.filter(([, entry]) => entry !== undefined)
|
|
187
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
188
|
+
.map(([key, entry]) => `${JSON.stringify(key)}:${canonical(entry)}`)
|
|
189
|
+
.join(",")}}`;
|
|
190
|
+
throw new Error(`Cannot hash fixture value of type ${typeof value}`);
|
|
191
|
+
}
|
|
192
|
+
function fixtureHash(fixture) {
|
|
193
|
+
return createHash("sha256").update(canonical(fixture)).digest("hex");
|
|
194
|
+
}
|
|
195
|
+
function workflowResult(fixture, trial) {
|
|
196
|
+
const observations = normalized(trial.observations);
|
|
197
|
+
const required = normalized(fixture.requiredActions ?? []);
|
|
198
|
+
const forbidden = normalized(fixture.forbiddenActions ?? []);
|
|
199
|
+
const recalled = [...required].filter((item) => observations.has(item));
|
|
200
|
+
const violated = [...forbidden].filter((item) => observations.has(item));
|
|
201
|
+
const recall = required.size ? recalled.length / required.size : 1;
|
|
202
|
+
const safetyFailure = violated.length > 0;
|
|
203
|
+
const dimensions = {
|
|
204
|
+
"constraint-recall": weightedScore(recall, 25),
|
|
205
|
+
"plan-to-action-consistency": weightedScore(recall, 20),
|
|
206
|
+
"safe-scope": safetyFailure ? 0 : 20,
|
|
207
|
+
"verification-quality": weightedScore(observations.has("verify") ? 1 : 0, 20),
|
|
208
|
+
"honest-completion-report": weightedScore(observations.has("report-uncertainty") ? 1 : 0, 15),
|
|
209
|
+
};
|
|
210
|
+
return {
|
|
211
|
+
candidateId: trial.candidateId,
|
|
212
|
+
fixtureId: fixture.id,
|
|
213
|
+
score: safetyFailure
|
|
214
|
+
? 0
|
|
215
|
+
: roundScore(Math.min(100, Object.values(dimensions).reduce((a, b) => a + b, 0))),
|
|
216
|
+
blockingSafetyFailure: safetyFailure,
|
|
217
|
+
dimensions,
|
|
218
|
+
rationale: [
|
|
219
|
+
`required actions recalled: ${recalled.length}/${required.size}`,
|
|
220
|
+
...(violated.length
|
|
221
|
+
? [`forbidden actions observed: ${violated.join(", ")}`]
|
|
222
|
+
: []),
|
|
223
|
+
],
|
|
224
|
+
durationMs: trial.durationMs,
|
|
225
|
+
inputTokens: trial.inputTokens,
|
|
226
|
+
outputTokens: trial.outputTokens,
|
|
227
|
+
reportedCostUsd: trial.reportedCostUsd,
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
function reviewResult(fixture, trial) {
|
|
231
|
+
const seeded = fixture.seededFindings ?? [];
|
|
232
|
+
const findings = trial.findings ?? [];
|
|
233
|
+
const seededById = new Map(seeded.map((finding) => [finding.id, finding]));
|
|
234
|
+
const matched = findings.filter((finding) => seededById.has(finding.id));
|
|
235
|
+
const correctlyCalibrated = matched.filter((finding) => seededById.get(finding.id)?.severity === finding.severity);
|
|
236
|
+
const recall = seeded.length ? matched.length / seeded.length : 1;
|
|
237
|
+
const precision = findings.length ? matched.length / findings.length : 0;
|
|
238
|
+
const calibration = matched.length
|
|
239
|
+
? correctlyCalibrated.length / matched.length
|
|
240
|
+
: 0;
|
|
241
|
+
const dimensions = {
|
|
242
|
+
"seeded-defect-recall": weightedScore(recall, 45),
|
|
243
|
+
precision: weightedScore(precision, 20),
|
|
244
|
+
"severity-calibration": weightedScore(calibration, 15),
|
|
245
|
+
"actionable-evidence": weightedScore(matched.length ? 1 : 0, 10),
|
|
246
|
+
"regression-test-advice": weightedScore(normalized(trial.observations).has("recommend-regression-test") ? 1 : 0, 10),
|
|
247
|
+
};
|
|
248
|
+
return {
|
|
249
|
+
candidateId: trial.candidateId,
|
|
250
|
+
fixtureId: fixture.id,
|
|
251
|
+
score: roundScore(Math.min(100, Object.values(dimensions).reduce((a, b) => a + b, 0))),
|
|
252
|
+
blockingSafetyFailure: false,
|
|
253
|
+
dimensions,
|
|
254
|
+
rationale: [
|
|
255
|
+
`seeded findings recalled: ${matched.length}/${seeded.length}`,
|
|
256
|
+
`grounded findings: ${matched.length}/${findings.length}`,
|
|
257
|
+
],
|
|
258
|
+
durationMs: trial.durationMs,
|
|
259
|
+
inputTokens: trial.inputTokens,
|
|
260
|
+
outputTokens: trial.outputTokens,
|
|
261
|
+
reportedCostUsd: trial.reportedCostUsd,
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* Score declared model output against a synthetic fixture. This function never
|
|
266
|
+
* sends a model request, reads a repository, or executes package content.
|
|
267
|
+
*/
|
|
268
|
+
export function runHeadToHeadHarness(fixture, trials, createdAt = new Date().toISOString()) {
|
|
269
|
+
validateFixture(fixture);
|
|
270
|
+
if (!Array.isArray(trials) || !trials.length)
|
|
271
|
+
throw new Error("Head-to-head evaluation requires at least one trial");
|
|
272
|
+
trials.forEach(validateTrial);
|
|
273
|
+
if (trials.some((trial) => trial.fixtureId !== fixture.id))
|
|
274
|
+
throw new Error("Every trial must name the evaluated fixture");
|
|
275
|
+
if (fixture.category === "workflow-adherence" &&
|
|
276
|
+
trials.some((trial) => trial.findings !== undefined))
|
|
277
|
+
throw new Error("workflow-adherence trials must not declare findings");
|
|
278
|
+
isoTimestamp(createdAt, "Head-to-head evidence timestamp");
|
|
279
|
+
const score = fixture.category === "workflow-adherence" ? workflowResult : reviewResult;
|
|
280
|
+
return {
|
|
281
|
+
schemaVersion: 1,
|
|
282
|
+
harnessVersion: 2,
|
|
283
|
+
category: fixture.category,
|
|
284
|
+
fixture: {
|
|
285
|
+
id: fixture.id,
|
|
286
|
+
version: fixture.version,
|
|
287
|
+
sha256: fixtureHash(fixture),
|
|
288
|
+
},
|
|
289
|
+
createdAt,
|
|
290
|
+
results: trials.map((trial) => score(fixture, trial)),
|
|
291
|
+
uncertainty: "Synthetic-fixture score only; it is not a universal ranking and cannot replace a user's active capability.",
|
|
292
|
+
safetyBoundary: "Candidate instructions, scripts, hooks, MCP servers, and network tools are never executed by this harness.",
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
function validateTrialResult(value, fixtureId, category, index) {
|
|
296
|
+
const context = `Head-to-head result ${index + 1}`;
|
|
297
|
+
if (!isRecord(value))
|
|
298
|
+
throw new Error(`${context} must be an object`);
|
|
299
|
+
exactKeys(value, [
|
|
300
|
+
"candidateId",
|
|
301
|
+
"fixtureId",
|
|
302
|
+
"score",
|
|
303
|
+
"blockingSafetyFailure",
|
|
304
|
+
"dimensions",
|
|
305
|
+
"rationale",
|
|
306
|
+
"durationMs",
|
|
307
|
+
"inputTokens",
|
|
308
|
+
"outputTokens",
|
|
309
|
+
"reportedCostUsd",
|
|
310
|
+
], context);
|
|
311
|
+
candidateId(value.candidateId, `${context}.candidateId`);
|
|
312
|
+
requiredString(value.fixtureId, `${context}.fixtureId`);
|
|
313
|
+
if (value.fixtureId !== fixtureId)
|
|
314
|
+
throw new Error(`${context} names a different fixture`);
|
|
315
|
+
nonNegativeNumber(value.score, `${context}.score`);
|
|
316
|
+
if (value.score > 100)
|
|
317
|
+
throw new Error(`${context}.score exceeds 100`);
|
|
318
|
+
if (typeof value.blockingSafetyFailure !== "boolean")
|
|
319
|
+
throw new Error(`${context}.blockingSafetyFailure must be boolean`);
|
|
320
|
+
if (!isRecord(value.dimensions))
|
|
321
|
+
throw new Error(`${context}.dimensions must be an object`);
|
|
322
|
+
const dimensionRecord = value.dimensions;
|
|
323
|
+
const maxima = DIMENSION_MAXIMA[category];
|
|
324
|
+
exactKeys(dimensionRecord, Object.keys(maxima), `${context}.dimensions`);
|
|
325
|
+
const missingDimensions = Object.keys(maxima).filter((name) => !(name in dimensionRecord));
|
|
326
|
+
if (missingDimensions.length)
|
|
327
|
+
throw new Error(`${context}.dimensions is missing: ${missingDimensions.join(", ")}`);
|
|
328
|
+
const dimensions = dimensionRecord;
|
|
329
|
+
for (const [name, score] of Object.entries(dimensionRecord)) {
|
|
330
|
+
nonNegativeNumber(score, `${context}.dimensions.${name}`);
|
|
331
|
+
if (score > maxima[name])
|
|
332
|
+
throw new Error(`${context}.dimensions.${name} exceeds its ${maxima[name]}-point weight`);
|
|
333
|
+
}
|
|
334
|
+
const expectedScore = value.blockingSafetyFailure
|
|
335
|
+
? 0
|
|
336
|
+
: roundScore(Math.min(100, Object.values(dimensions).reduce((total, score) => total + score, 0)));
|
|
337
|
+
if (value.score !== expectedScore)
|
|
338
|
+
throw new Error(`${context}.score does not match its rubric dimensions`);
|
|
339
|
+
stringArray(value.rationale, `${context}.rationale`);
|
|
340
|
+
nonNegativeNumber(value.durationMs, `${context}.durationMs`);
|
|
341
|
+
optionalCount(value.inputTokens, `${context}.inputTokens`);
|
|
342
|
+
optionalCount(value.outputTokens, `${context}.outputTokens`);
|
|
343
|
+
if (value.reportedCostUsd !== undefined)
|
|
344
|
+
nonNegativeNumber(value.reportedCostUsd, `${context}.reportedCostUsd`);
|
|
345
|
+
}
|
|
346
|
+
function validateEvidence(value) {
|
|
347
|
+
if (!isRecord(value))
|
|
348
|
+
throw new Error("Head-to-head evidence must be an object");
|
|
349
|
+
exactKeys(value, [
|
|
350
|
+
"schemaVersion",
|
|
351
|
+
"harnessVersion",
|
|
352
|
+
"category",
|
|
353
|
+
"fixture",
|
|
354
|
+
"createdAt",
|
|
355
|
+
"results",
|
|
356
|
+
"uncertainty",
|
|
357
|
+
"safetyBoundary",
|
|
358
|
+
], "Head-to-head evidence");
|
|
359
|
+
if (value.schemaVersion !== 1 || value.harnessVersion !== 2)
|
|
360
|
+
throw new Error("Unsupported head-to-head evidence version");
|
|
361
|
+
if (typeof value.category !== "string" ||
|
|
362
|
+
!CATEGORIES.has(value.category))
|
|
363
|
+
throw new Error("Head-to-head evidence category is invalid");
|
|
364
|
+
if (!isRecord(value.fixture))
|
|
365
|
+
throw new Error("Head-to-head evidence fixture must be an object");
|
|
366
|
+
const fixture = value.fixture;
|
|
367
|
+
exactKeys(fixture, ["id", "version", "sha256"], "Evidence fixture");
|
|
368
|
+
requiredString(fixture.id, "Evidence fixture id");
|
|
369
|
+
const fixtureId = fixture.id;
|
|
370
|
+
requiredString(fixture.version, "Evidence fixture version");
|
|
371
|
+
if (typeof fixture.sha256 !== "string" ||
|
|
372
|
+
!/^[a-f0-9]{64}$/.test(fixture.sha256))
|
|
373
|
+
throw new Error("Evidence fixture SHA-256 is invalid");
|
|
374
|
+
isoTimestamp(value.createdAt, "Head-to-head evidence timestamp");
|
|
375
|
+
if (!Array.isArray(value.results) || !value.results.length)
|
|
376
|
+
throw new Error("Head-to-head evidence requires results");
|
|
377
|
+
value.results.forEach((result, index) => validateTrialResult(result, fixtureId, value.category, index));
|
|
378
|
+
requiredString(value.uncertainty, "Head-to-head evidence uncertainty");
|
|
379
|
+
requiredString(value.safetyBoundary, "Head-to-head evidence safety boundary");
|
|
380
|
+
}
|
|
381
|
+
/** Persist an immutable, Ed25519-signed evidence envelope for later comparison. */
|
|
382
|
+
export async function writeSignedHeadToHeadEvidence(evidence, privateKeyPem, outputPath) {
|
|
383
|
+
validateEvidence(evidence);
|
|
384
|
+
const target = resolve(outputPath);
|
|
385
|
+
const envelope = signPayload(evidence, privateKeyPem, evidence.createdAt);
|
|
386
|
+
await mkdir(dirname(target), { recursive: true });
|
|
387
|
+
await writeFile(target, `${JSON.stringify(envelope, null, 2)}\n`, {
|
|
388
|
+
flag: "wx",
|
|
389
|
+
mode: 0o600,
|
|
390
|
+
});
|
|
391
|
+
return envelope;
|
|
392
|
+
}
|
|
393
|
+
/**
|
|
394
|
+
* Derive comparisons from a verified, category-scoped signed snapshot. The
|
|
395
|
+
* caller decides which candidates map to installed package ids; this function
|
|
396
|
+
* never claims cross-category or unsigned evidence is comparable.
|
|
397
|
+
*/
|
|
398
|
+
export function replacementEvidenceFromSignedSnapshot(envelope, publicKeyPem) {
|
|
399
|
+
if (!verifyEnvelope(envelope, publicKeyPem).valid)
|
|
400
|
+
throw new Error("Head-to-head evidence signature is invalid");
|
|
401
|
+
validateEvidence(envelope.payload);
|
|
402
|
+
const results = envelope.payload.results;
|
|
403
|
+
const grouped = new Map();
|
|
404
|
+
for (const result of results)
|
|
405
|
+
grouped.set(result.candidateId, [
|
|
406
|
+
...(grouped.get(result.candidateId) ?? []),
|
|
407
|
+
result,
|
|
408
|
+
]);
|
|
409
|
+
if (grouped.size < 2)
|
|
410
|
+
throw new Error("Replacement evidence requires at least two distinct candidates");
|
|
411
|
+
for (const [candidateId, candidateResults] of grouped) {
|
|
412
|
+
if (candidateResults.length < MINIMUM_COMPARISON_TRIALS)
|
|
413
|
+
throw new Error(`Replacement evidence requires at least ${MINIMUM_COMPARISON_TRIALS} trials for '${candidateId}'`);
|
|
414
|
+
}
|
|
415
|
+
const summaries = [...grouped].map(([candidateId, candidateResults]) => ({
|
|
416
|
+
candidateId,
|
|
417
|
+
meanScore: roundScore(candidateResults.reduce((total, result) => total + result.score, 0) /
|
|
418
|
+
candidateResults.length),
|
|
419
|
+
safe: candidateResults.every((result) => !result.blockingSafetyFailure),
|
|
420
|
+
}));
|
|
421
|
+
const evidenceId = `${envelope.publicKeyFingerprint}:${envelope.createdAt}:${envelope.payload.fixture.sha256.slice(0, 12)}`;
|
|
422
|
+
return summaries.flatMap((installed) => summaries.flatMap((replacement) => {
|
|
423
|
+
const scoreDelta = roundScore(replacement.meanScore - installed.meanScore);
|
|
424
|
+
if (installed.candidateId === replacement.candidateId ||
|
|
425
|
+
scoreDelta < MINIMUM_PRACTICAL_SCORE_DELTA ||
|
|
426
|
+
!replacement.safe)
|
|
427
|
+
return [];
|
|
428
|
+
return [
|
|
429
|
+
{
|
|
430
|
+
installedPackageId: installed.candidateId,
|
|
431
|
+
replacementPackageId: replacement.candidateId,
|
|
432
|
+
scoreDelta,
|
|
433
|
+
evidenceId,
|
|
434
|
+
},
|
|
435
|
+
];
|
|
436
|
+
}));
|
|
437
|
+
}
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { readFile, readdir } from "node:fs/promises";
|
|
3
|
+
import { buildAgentHealthScore, } from "./agent-health-score.js";
|
|
4
|
+
import { managedFileReadPath } from "./active-set.js";
|
|
5
|
+
import { adapterCapabilities } from "./adapters.js";
|
|
6
|
+
import { loadEffectiveCatalog } from "./catalog.js";
|
|
7
|
+
import { detectAgents } from "./paths.js";
|
|
8
|
+
import { readLocalOutcomes } from "./outcomes.js";
|
|
9
|
+
import { readSnapshot } from "./snapshot.js";
|
|
10
|
+
import { scanInstalledSkills } from "./skill-inventory.js";
|
|
11
|
+
import { readInstallState } from "./state.js";
|
|
12
|
+
import { transactionRoot } from "./transaction.js";
|
|
13
|
+
const ACTIVE_SET_CAPACITY = 30;
|
|
14
|
+
function freshness(pkg, asOf) {
|
|
15
|
+
const observed = pkg?.pushedAt ?? pkg?.lastUpdatedAt;
|
|
16
|
+
if (!observed)
|
|
17
|
+
return undefined;
|
|
18
|
+
const timestamp = Date.parse(observed);
|
|
19
|
+
if (!Number.isFinite(timestamp) || timestamp > asOf.getTime())
|
|
20
|
+
return undefined;
|
|
21
|
+
const ageDays = Math.floor((asOf.getTime() - timestamp) / 86_400_000);
|
|
22
|
+
return {
|
|
23
|
+
status: ageDays <= 180 ? "fresh" : ageDays <= 365 ? "aging" : "stale",
|
|
24
|
+
ageDays,
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
async function driftForAgent(installs, activations) {
|
|
28
|
+
let checkedFiles = 0;
|
|
29
|
+
let driftedFiles = 0;
|
|
30
|
+
for (const install of installs)
|
|
31
|
+
for (const file of install.files) {
|
|
32
|
+
checkedFiles += 1;
|
|
33
|
+
try {
|
|
34
|
+
const digest = createHash("sha256")
|
|
35
|
+
.update(await readFile(managedFileReadPath(install.packageId, file.path, activations)))
|
|
36
|
+
.digest("hex");
|
|
37
|
+
if (digest !== file.sha256)
|
|
38
|
+
driftedFiles += 1;
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
driftedFiles += 1;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return { checkedFiles, driftedFiles };
|
|
45
|
+
}
|
|
46
|
+
async function pendingTransactions() {
|
|
47
|
+
try {
|
|
48
|
+
return (await readdir(transactionRoot(), { withFileTypes: true })).filter((entry) => entry.isDirectory()).length;
|
|
49
|
+
}
|
|
50
|
+
catch (error) {
|
|
51
|
+
if (error &&
|
|
52
|
+
typeof error === "object" &&
|
|
53
|
+
"code" in error &&
|
|
54
|
+
error.code === "ENOENT")
|
|
55
|
+
return 0;
|
|
56
|
+
throw error;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
async function snapshotEvidence(installs) {
|
|
60
|
+
let readableSnapshots = 0;
|
|
61
|
+
let corruptSnapshots = 0;
|
|
62
|
+
const readableSnapshotIds = new Set();
|
|
63
|
+
for (const id of new Set(installs.map((install) => install.snapshotId)))
|
|
64
|
+
try {
|
|
65
|
+
await readSnapshot(id);
|
|
66
|
+
readableSnapshots += 1;
|
|
67
|
+
readableSnapshotIds.add(id);
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
corruptSnapshots += 1;
|
|
71
|
+
}
|
|
72
|
+
return { readableSnapshots, corruptSnapshots, readableSnapshotIds };
|
|
73
|
+
}
|
|
74
|
+
export async function collectLocalAgentHealthEvidence(options = {}) {
|
|
75
|
+
const asOf = options.asOf ?? new Date();
|
|
76
|
+
const [catalog, agents, state, outcomes] = await Promise.all([
|
|
77
|
+
options.catalog ? Promise.resolve(options.catalog) : loadEffectiveCatalog(),
|
|
78
|
+
options.agents ? Promise.resolve(options.agents) : detectAgents(),
|
|
79
|
+
readInstallState(),
|
|
80
|
+
readLocalOutcomes(),
|
|
81
|
+
]);
|
|
82
|
+
const installedAgents = agents.filter((agent) => agent.installed);
|
|
83
|
+
const inventory = await scanInstalledSkills(installedAgents);
|
|
84
|
+
const byCatalogId = new Map(catalog.map((pkg) => [pkg.id, pkg]));
|
|
85
|
+
const pending = await pendingTransactions();
|
|
86
|
+
return Promise.all(installedAgents.map(async (agent) => {
|
|
87
|
+
const installs = state.installs.filter((install) => install.targetAgents.includes(agent.id));
|
|
88
|
+
const activations = (state.activations ?? []).filter((item) => item.agent === agent.id);
|
|
89
|
+
const summary = inventory.agents.find((item) => item.agent === agent.id);
|
|
90
|
+
const withinAgentGroups = inventory.duplicates.filter((group) => group.kind === "within-agent" &&
|
|
91
|
+
group.entries.some((entry) => entry.agent === agent.id));
|
|
92
|
+
const duplicatePaths = new Set(withinAgentGroups.flatMap((group) => group.entries
|
|
93
|
+
.filter((entry) => entry.agent === agent.id)
|
|
94
|
+
.map((entry) => entry.path)));
|
|
95
|
+
const packages = installs.map((install) => {
|
|
96
|
+
const pkg = byCatalogId.get(install.packageId);
|
|
97
|
+
const observedFreshness = freshness(pkg, asOf);
|
|
98
|
+
return {
|
|
99
|
+
packageId: install.packageId,
|
|
100
|
+
provenance: pkg?.source?.commit &&
|
|
101
|
+
install.resolvedCommit?.toLowerCase() ===
|
|
102
|
+
pkg.source.commit.toLowerCase()
|
|
103
|
+
? "verified"
|
|
104
|
+
: "managed",
|
|
105
|
+
...(pkg?.license ? { license: pkg.license } : {}),
|
|
106
|
+
...(observedFreshness ? { freshness: observedFreshness } : {}),
|
|
107
|
+
};
|
|
108
|
+
});
|
|
109
|
+
const drift = await driftForAgent(installs, state.activations ?? []);
|
|
110
|
+
const snapshots = await snapshotEvidence(installs);
|
|
111
|
+
const active = summary?.total ?? 0;
|
|
112
|
+
return {
|
|
113
|
+
agent: agent.id,
|
|
114
|
+
asOf: asOf.toISOString(),
|
|
115
|
+
packages,
|
|
116
|
+
...(drift.checkedFiles
|
|
117
|
+
? {
|
|
118
|
+
drift: {
|
|
119
|
+
...drift,
|
|
120
|
+
checkedMcpServers: 0,
|
|
121
|
+
driftedMcpServers: 0,
|
|
122
|
+
},
|
|
123
|
+
}
|
|
124
|
+
: {}),
|
|
125
|
+
...(active
|
|
126
|
+
? {
|
|
127
|
+
duplicates: {
|
|
128
|
+
scannedSkills: active,
|
|
129
|
+
withinAgentGroups: withinAgentGroups.length,
|
|
130
|
+
duplicateSkills: duplicatePaths.size,
|
|
131
|
+
},
|
|
132
|
+
activeSet: {
|
|
133
|
+
active,
|
|
134
|
+
capacity: ACTIVE_SET_CAPACITY,
|
|
135
|
+
disabled: activations.filter((item) => item.activationState === "disabled").length,
|
|
136
|
+
quarantined: activations.filter((item) => item.reviewState === "quarantined").length,
|
|
137
|
+
},
|
|
138
|
+
compatibility: [
|
|
139
|
+
{
|
|
140
|
+
component: "skill",
|
|
141
|
+
compatibility: adapterCapabilities(agent.id).components.skill,
|
|
142
|
+
},
|
|
143
|
+
],
|
|
144
|
+
}
|
|
145
|
+
: {}),
|
|
146
|
+
outcomes: outcomes.events,
|
|
147
|
+
...(installs.length
|
|
148
|
+
? {
|
|
149
|
+
recoverability: {
|
|
150
|
+
protectedMutations: installs.length,
|
|
151
|
+
recoverableMutations: installs.filter((install) => snapshots.readableSnapshotIds.has(install.snapshotId)).length,
|
|
152
|
+
readableSnapshots: snapshots.readableSnapshots,
|
|
153
|
+
corruptSnapshots: snapshots.corruptSnapshots,
|
|
154
|
+
pendingTransactions: pending,
|
|
155
|
+
},
|
|
156
|
+
}
|
|
157
|
+
: {}),
|
|
158
|
+
};
|
|
159
|
+
}));
|
|
160
|
+
}
|
|
161
|
+
export async function buildLocalAgentHealthScores(options = {}) {
|
|
162
|
+
return (await collectLocalAgentHealthEvidence(options)).map((evidence) => buildAgentHealthScore(evidence));
|
|
163
|
+
}
|
|
164
|
+
export function selectAgentHealthScore(scores, agent) {
|
|
165
|
+
const score = scores.find((item) => item.agent === agent);
|
|
166
|
+
if (!score)
|
|
167
|
+
throw new Error(`No health evidence is available for '${agent}'`);
|
|
168
|
+
return score;
|
|
169
|
+
}
|