@agentskit/harness 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +77 -0
- package/CODE_OF_CONDUCT.md +5 -0
- package/CONTRIBUTING.md +26 -0
- package/LICENSE +21 -0
- package/README.md +473 -0
- package/SECURITY.md +11 -0
- package/dist/cli.js +1308 -0
- package/dist/cli.js.map +1 -0
- package/dist/index.d.ts +968 -0
- package/dist/index.js +1828 -0
- package/dist/index.js.map +1 -0
- package/docs/ADR-0001-extensible-kernel.md +41 -0
- package/docs/ADR-0002-profiles-and-context.md +22 -0
- package/docs/ADR-0003-doc-bridge-context-binding.md +36 -0
- package/docs/ADR-0004-run-metrics.md +29 -0
- package/docs/ADR-0005-benchmark-manifest.md +27 -0
- package/docs/ADR-0006-agent-session-protocol.md +36 -0
- package/docs/ADR-0007-policy-gate.md +33 -0
- package/docs/ADR-0008-runtime-executor.md +33 -0
- package/docs/ADR-0009-process-runtime-boundary.md +34 -0
- package/docs/ADR-0010-docker-sandbox-runtime.md +32 -0
- package/docs/ADR-0011-runtime-attestation.md +30 -0
- package/docs/ADR-0012-controlled-baseline-observations.md +27 -0
- package/docs/ADR-0013-honest-benchmark-comparability.md +27 -0
- package/docs/ADR-0014-criterion-level-benchmark-evidence.md +24 -0
- package/docs/ADR-0015-directional-benchmark-outcomes.md +23 -0
- package/docs/ADR-0016-baseline-evidence-digests.md +21 -0
- package/docs/ADR-0017-event-log-integrity.md +25 -0
- package/docs/ADR-0018-verification-projection-attestation.md +23 -0
- package/docs/ADR-0019-human-decision-attestation.md +27 -0
- package/docs/ADR-0020-terminal-reconciliation.md +26 -0
- package/docs/ADR-0021-event-lock-recovery.md +25 -0
- package/docs/ADR-0022-signed-evidence-bundle.md +27 -0
- package/docs/ADR-0023-safe-action-recovery.md +30 -0
- package/docs/ADR-0024-controlled-completion-metrics.md +27 -0
- package/docs/ADR-0025-ci-dogfood.md +22 -0
- package/docs/ADR-0026-ci-evidence-artifact.md +22 -0
- package/docs/ADR-0027-portable-evidence.md +19 -0
- package/docs/ADR-0028-effective-metrics.md +20 -0
- package/docs/ADR-0029-honest-ci-preparation.md +20 -0
- package/docs/ADR-0030-agentskit-os-benchmark-bridge.md +20 -0
- package/docs/ADR-0031-real-provider-baseline.md +18 -0
- package/docs/ADR-0032-harness-equivalent-benchmark.md +25 -0
- package/docs/ADR-0033-portable-agent-gate.md +25 -0
- package/docs/ADR-0034-measurement-quality-gates.md +25 -0
- package/docs/ADR-0035-reproducible-benchmark-samples.md +20 -0
- package/docs/ADR-0036-comparable-baseline-samples.md +20 -0
- package/docs/ADR-0037-replicated-baseline-collection.md +27 -0
- package/docs/ADR-0038-end-to-end-benchmark-boundary.md +28 -0
- package/docs/ADR-0039-artifact-and-protocol-metrics.md +39 -0
- package/docs/ADR-0040-benchmark-corpus-surfaces.md +32 -0
- package/package.json +68 -0
package/dist/cli.js
ADDED
|
@@ -0,0 +1,1308 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { createPrivateKey, createPublicKey, sign, verify, createHash } from 'crypto';
|
|
3
|
+
import { readFileSync, mkdirSync, writeFileSync, existsSync, openSync, writeSync, appendFileSync, closeSync, unlinkSync, mkdtempSync, renameSync, rmSync, lstatSync, readdirSync } from 'fs';
|
|
4
|
+
import { Command } from 'commander';
|
|
5
|
+
import { resolve, dirname, relative, join, sep } from 'path';
|
|
6
|
+
import { execFile, spawn } from 'child_process';
|
|
7
|
+
import { promisify } from 'util';
|
|
8
|
+
import { tmpdir } from 'os';
|
|
9
|
+
|
|
10
|
+
// src/constants.ts
|
|
11
|
+
var STATES = [
|
|
12
|
+
"CLARIFYING",
|
|
13
|
+
"PLANNED",
|
|
14
|
+
"IMPLEMENTING",
|
|
15
|
+
"VERIFYING",
|
|
16
|
+
"AWAITING_HUMAN_APPROVAL",
|
|
17
|
+
"AWAITING_AUTHORIZATION",
|
|
18
|
+
"COMPLETE",
|
|
19
|
+
"BLOCKED",
|
|
20
|
+
"STALE",
|
|
21
|
+
"CANCELLED",
|
|
22
|
+
"SUPERSEDED"
|
|
23
|
+
];
|
|
24
|
+
var LEGAL_TRANSITIONS = {
|
|
25
|
+
CLARIFYING: ["PLANNED", "BLOCKED", "CANCELLED"],
|
|
26
|
+
PLANNED: ["IMPLEMENTING", "CLARIFYING", "STALE", "CANCELLED"],
|
|
27
|
+
IMPLEMENTING: ["VERIFYING", "CLARIFYING", "STALE", "CANCELLED"],
|
|
28
|
+
VERIFYING: ["AWAITING_HUMAN_APPROVAL", "BLOCKED", "STALE", "CANCELLED"],
|
|
29
|
+
AWAITING_HUMAN_APPROVAL: ["AWAITING_AUTHORIZATION", "COMPLETE", "BLOCKED", "IMPLEMENTING", "STALE", "CANCELLED"],
|
|
30
|
+
AWAITING_AUTHORIZATION: ["COMPLETE", "BLOCKED", "IMPLEMENTING", "STALE", "CANCELLED"],
|
|
31
|
+
COMPLETE: ["STALE", "SUPERSEDED"],
|
|
32
|
+
BLOCKED: ["SUPERSEDED"],
|
|
33
|
+
STALE: ["SUPERSEDED", "PLANNED"],
|
|
34
|
+
CANCELLED: ["SUPERSEDED"],
|
|
35
|
+
SUPERSEDED: []
|
|
36
|
+
};
|
|
37
|
+
var REAL_CATEGORIES = /* @__PURE__ */ new Set(["endpoint", "database", "cli", "mcp", "ui"]);
|
|
38
|
+
var DECISIONS = /* @__PURE__ */ new Set(["approved", "approve", "yes", "ok", "rejected", "reject", "no"]);
|
|
39
|
+
|
|
40
|
+
// src/errors.ts
|
|
41
|
+
var HarnessError = class extends Error {
|
|
42
|
+
code;
|
|
43
|
+
constructor(message, code = "HARNESS_ERROR") {
|
|
44
|
+
super(message);
|
|
45
|
+
this.name = "HarnessError";
|
|
46
|
+
this.code = code;
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
var fail = (message, code = "HARNESS_ERROR") => {
|
|
50
|
+
throw new HarnessError(message, code);
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
// src/profiles.ts
|
|
54
|
+
var isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
55
|
+
var record = (value, label) => {
|
|
56
|
+
if (!isRecord(value)) fail(`${label} must be an object.`, "INVALID_CONFIG");
|
|
57
|
+
return value;
|
|
58
|
+
};
|
|
59
|
+
var id = (value, label) => {
|
|
60
|
+
if (typeof value !== "string" || !value.trim()) fail(`${label} must be a non-empty string.`, "INVALID_CONFIG");
|
|
61
|
+
return value;
|
|
62
|
+
};
|
|
63
|
+
var parents = (value, label) => value === void 0 ? [] : Array.isArray(value) ? value.map((item, index2) => id(item, `${label}[${index2}]`)) : [id(value, label)];
|
|
64
|
+
var merge = (base, overlay) => {
|
|
65
|
+
const result = { ...base };
|
|
66
|
+
for (const key of ["surfaces", "budget", "cleanup"]) {
|
|
67
|
+
if (overlay[key] !== void 0) result[key] = { ...isRecord(result[key]) ? result[key] : {}, ...record(overlay[key], `profile.${key}`) };
|
|
68
|
+
}
|
|
69
|
+
if (overlay["checkOverrides"] !== void 0) {
|
|
70
|
+
if (!Array.isArray(overlay["checkOverrides"])) fail("profile.checkOverrides must be an array.", "INVALID_CONFIG");
|
|
71
|
+
const checks = Array.isArray(result["checks"]) ? [...result["checks"]] : [];
|
|
72
|
+
for (const [index2, value] of overlay["checkOverrides"].entries()) {
|
|
73
|
+
const override = record(value, `profile.checkOverrides[${index2}]`);
|
|
74
|
+
const checkId = id(override["id"], `profile.checkOverrides[${index2}].id`);
|
|
75
|
+
const checkIndex = checks.findIndex((check) => isRecord(check) && check["id"] === checkId);
|
|
76
|
+
if (checkIndex < 0) fail(`profile.checkOverrides references unknown check: ${checkId}.`, "INVALID_CONFIG");
|
|
77
|
+
checks[checkIndex] = { ...checks[checkIndex], ...override };
|
|
78
|
+
}
|
|
79
|
+
result["checks"] = checks;
|
|
80
|
+
}
|
|
81
|
+
return result;
|
|
82
|
+
};
|
|
83
|
+
var resolveProfile = (root) => {
|
|
84
|
+
if (root["profiles"] === void 0) return root;
|
|
85
|
+
const profileMap = record(root["profiles"], "profiles");
|
|
86
|
+
const selected = id(root["profile"], "profile");
|
|
87
|
+
const visiting = /* @__PURE__ */ new Set();
|
|
88
|
+
const visited = /* @__PURE__ */ new Map();
|
|
89
|
+
const resolve6 = (name) => {
|
|
90
|
+
const cached = visited.get(name);
|
|
91
|
+
if (cached) return cached;
|
|
92
|
+
if (visiting.has(name)) fail(`Profile inheritance cycle includes ${name}.`, "INVALID_CONFIG");
|
|
93
|
+
const definition = record(profileMap[name], `profiles.${name}`);
|
|
94
|
+
visiting.add(name);
|
|
95
|
+
let result = { ...root };
|
|
96
|
+
for (const parent of parents(definition["extends"], `profiles.${name}.extends`)) result = merge(result, resolve6(parent));
|
|
97
|
+
result = merge(result, definition);
|
|
98
|
+
visiting.delete(name);
|
|
99
|
+
visited.set(name, result);
|
|
100
|
+
return result;
|
|
101
|
+
};
|
|
102
|
+
return resolve6(selected);
|
|
103
|
+
};
|
|
104
|
+
var sha256 = (value) => createHash("sha256").update(value).digest("hex");
|
|
105
|
+
var hashJson = (value) => sha256(JSON.stringify(value));
|
|
106
|
+
var readJson = (path) => JSON.parse(readFileSync(path, "utf8"));
|
|
107
|
+
var writeJson = (path, value) => {
|
|
108
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
109
|
+
writeFileSync(path, `${JSON.stringify(value, null, 2)}
|
|
110
|
+
`, "utf8");
|
|
111
|
+
};
|
|
112
|
+
var pathInside = (root, candidate) => {
|
|
113
|
+
const rel = relative(resolve(root), resolve(candidate));
|
|
114
|
+
return rel === "" || !rel.startsWith(`..${sep}`) && rel !== ".." && !rel.startsWith(sep);
|
|
115
|
+
};
|
|
116
|
+
var latestPath = (stateDir) => join(stateDir, "latest.json");
|
|
117
|
+
var runPath = (stateDir, runId) => join(stateDir, "runs", runId, "run.json");
|
|
118
|
+
var saveRun = (stateDir, run) => writeJson(runPath(stateDir, run.runId), run);
|
|
119
|
+
var readRun = (stateDir, runId) => readJson(runPath(stateDir, runId));
|
|
120
|
+
var loadLatestRun = (stateDir) => {
|
|
121
|
+
if (!existsSync(latestPath(stateDir))) return null;
|
|
122
|
+
const pointer = readJson(latestPath(stateDir));
|
|
123
|
+
return readRun(stateDir, pointer.runId);
|
|
124
|
+
};
|
|
125
|
+
var setLatest = (stateDir, run) => writeJson(latestPath(stateDir), {
|
|
126
|
+
runId: run.runId,
|
|
127
|
+
path: relative(resolve(stateDir, "..", ".."), runPath(stateDir, run.runId)),
|
|
128
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
129
|
+
});
|
|
130
|
+
var cleanConfiguredArtifacts = (loaded) => {
|
|
131
|
+
const roots = loaded.config.cleanup?.roots ?? [];
|
|
132
|
+
for (const root of roots) {
|
|
133
|
+
const target = resolve(loaded.root, root);
|
|
134
|
+
if (!pathInside(loaded.root, target)) fail(`Cleanup root escapes project root: ${root}`, "INVALID_CONFIG");
|
|
135
|
+
if (existsSync(target)) for (const entry of readdirSync(target)) rmSync(join(target, entry), { recursive: true, force: true });
|
|
136
|
+
}
|
|
137
|
+
return { cleaned: roots };
|
|
138
|
+
};
|
|
139
|
+
var fileContents = (path) => readFileSync(path, "utf8");
|
|
140
|
+
|
|
141
|
+
// src/types.ts
|
|
142
|
+
var SURFACE_NAMES = ["logic", "endpoint", "database", "cli", "mcp", "ui", "docs"];
|
|
143
|
+
var CHECK_CATEGORIES = ["build", "test", "lint", ...SURFACE_NAMES, "custom"];
|
|
144
|
+
var RUN_STATES = [
|
|
145
|
+
"CLARIFYING",
|
|
146
|
+
"PLANNED",
|
|
147
|
+
"IMPLEMENTING",
|
|
148
|
+
"VERIFYING",
|
|
149
|
+
"AWAITING_HUMAN_APPROVAL",
|
|
150
|
+
"AWAITING_AUTHORIZATION",
|
|
151
|
+
"COMPLETE",
|
|
152
|
+
"BLOCKED",
|
|
153
|
+
"STALE",
|
|
154
|
+
"CANCELLED",
|
|
155
|
+
"SUPERSEDED"
|
|
156
|
+
];
|
|
157
|
+
|
|
158
|
+
// src/config.ts
|
|
159
|
+
var isRecord2 = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
160
|
+
var stringValue = (value, label) => {
|
|
161
|
+
if (typeof value !== "string") return fail(`${label} is required.`, "INVALID_CONFIG");
|
|
162
|
+
const result = value.trim();
|
|
163
|
+
if (!result) return fail(`${label} is required.`, "INVALID_CONFIG");
|
|
164
|
+
return result;
|
|
165
|
+
};
|
|
166
|
+
var stringArray = (value, label) => {
|
|
167
|
+
if (!Array.isArray(value)) fail(`${label} must be an array of non-empty strings.`, "INVALID_CONFIG");
|
|
168
|
+
const items = value;
|
|
169
|
+
if (!items.every((item) => typeof item === "string" && Boolean(item.trim()))) fail(`${label} must be an array of non-empty strings.`, "INVALID_CONFIG");
|
|
170
|
+
return items.map((item) => stringValue(item, label));
|
|
171
|
+
};
|
|
172
|
+
var asRecord = (value, label) => {
|
|
173
|
+
if (!isRecord2(value)) fail(`${label} must be an object.`, "INVALID_CONFIG");
|
|
174
|
+
return value;
|
|
175
|
+
};
|
|
176
|
+
var surface = (value, name) => {
|
|
177
|
+
if (typeof value === "boolean") return value ? { required: true } : { required: false, reason: `${name} is not applicable.` };
|
|
178
|
+
const record3 = asRecord(value, `surfaces.${name}`);
|
|
179
|
+
if (typeof record3["required"] !== "boolean") fail(`surfaces.${name}.required must be boolean.`, "INVALID_CONFIG");
|
|
180
|
+
if (!record3["required"] && typeof record3["reason"] !== "string") fail(`surfaces.${name}.reason is required when not applicable.`, "INVALID_CONFIG");
|
|
181
|
+
return { required: record3["required"], ...typeof record3["reason"] === "string" ? { reason: record3["reason"] } : {} };
|
|
182
|
+
};
|
|
183
|
+
var parseCheck = (value, index2) => {
|
|
184
|
+
const record3 = asRecord(value, `checks[${index2}]`);
|
|
185
|
+
const id2 = stringValue(record3["id"], `checks[${index2}].id`);
|
|
186
|
+
const category = stringValue(record3["category"], `checks[${index2}].category`);
|
|
187
|
+
if (!CHECK_CATEGORIES.includes(category)) fail(`checks[${index2}].category is invalid.`, "INVALID_CONFIG");
|
|
188
|
+
const command = stringValue(record3["command"], `checks[${index2}].command`);
|
|
189
|
+
if (REAL_CATEGORIES.has(category) && record3["execution"] !== "real") fail(`checks[${index2}] must declare execution: real.`, "INVALID_CONFIG");
|
|
190
|
+
if (record3["evidence"] !== "structured") fail(`checks[${index2}] must declare evidence: structured.`, "INVALID_CONFIG");
|
|
191
|
+
if (record3["capabilities"] !== void 0 && (!Array.isArray(record3["capabilities"]) || !record3["capabilities"].every((item) => typeof item === "string"))) fail(`checks[${index2}].capabilities must be strings.`, "INVALID_CONFIG");
|
|
192
|
+
const capabilities = Array.isArray(record3["capabilities"]) ? record3["capabilities"].filter((item) => typeof item === "string") : void 0;
|
|
193
|
+
if (category === "ui" && !capabilities?.includes("real-browser")) fail(`checks[${index2}] must declare real-browser.`, "INVALID_CONFIG");
|
|
194
|
+
if (category === "ui" && !capabilities?.includes("screenshot")) fail(`checks[${index2}] must declare screenshot.`, "INVALID_CONFIG");
|
|
195
|
+
if (record3["required"] !== void 0 && typeof record3["required"] !== "boolean") fail(`checks[${index2}].required must be boolean.`, "INVALID_CONFIG");
|
|
196
|
+
if (record3["timeoutMs"] !== void 0 && (!Number.isInteger(record3["timeoutMs"]) || typeof record3["timeoutMs"] !== "number" || record3["timeoutMs"] < 1)) fail(`checks[${index2}].timeoutMs must be positive.`, "INVALID_CONFIG");
|
|
197
|
+
return { id: id2, category, command, required: record3["required"] !== false, timeoutMs: typeof record3["timeoutMs"] === "number" ? record3["timeoutMs"] : 12e4, ...record3["execution"] === "real" ? { execution: "real" } : {}, ...capabilities ? { capabilities } : {}, evidence: "structured" };
|
|
198
|
+
};
|
|
199
|
+
var parseOutcome = (value, index2, checks) => {
|
|
200
|
+
const record3 = asRecord(value, `contract.outcomes[${index2}]`);
|
|
201
|
+
const id2 = stringValue(record3["id"], `contract.outcomes[${index2}].id`);
|
|
202
|
+
const statement = stringValue(record3["statement"], `contract.outcomes[${index2}].statement`);
|
|
203
|
+
const ids = stringArray(record3["checks"], `contract.outcomes[${index2}].checks`);
|
|
204
|
+
if (ids.some((checkId) => !checks.some((check) => check.id === checkId))) fail(`contract.outcomes[${index2}] references an unknown check.`, "INVALID_CONFIG");
|
|
205
|
+
return { id: id2, statement, checks: [...new Set(ids)] };
|
|
206
|
+
};
|
|
207
|
+
var validateConfig = (rawValue) => {
|
|
208
|
+
const raw = resolveProfile(asRecord(rawValue, "verification config"));
|
|
209
|
+
if (raw["schemaVersion"] !== 1) fail("verification config schemaVersion must be 1.", "INVALID_CONFIG");
|
|
210
|
+
const project = stringValue(raw["project"], "verification config project");
|
|
211
|
+
const contractRaw = asRecord(raw["contract"], "contract");
|
|
212
|
+
const rawChecks = raw["checks"];
|
|
213
|
+
const checks = Array.isArray(rawChecks) ? rawChecks.map(parseCheck) : fail("checks must be a non-empty array.", "INVALID_CONFIG");
|
|
214
|
+
if (!checks.length || new Set(checks.map((check) => check.id)).size !== checks.length) fail("check ids must be unique.", "INVALID_CONFIG");
|
|
215
|
+
const scopeRaw = asRecord(contractRaw["scope"], "contract.scope");
|
|
216
|
+
const scope = { inScope: stringArray(scopeRaw["inScope"], "contract.scope.inScope"), outOfScope: stringArray(scopeRaw["outOfScope"], "contract.scope.outOfScope") };
|
|
217
|
+
const ambiguities = stringArray(contractRaw["ambiguities"], "contract.ambiguities");
|
|
218
|
+
const rawOutcomes = contractRaw["outcomes"];
|
|
219
|
+
const outcomes = Array.isArray(rawOutcomes) ? rawOutcomes.map((outcome, index2) => parseOutcome(outcome, index2, checks)) : fail("contract.outcomes must be a non-empty array.", "INVALID_CONFIG");
|
|
220
|
+
if (!outcomes.length || new Set(outcomes.map((outcome) => outcome.id)).size !== outcomes.length) fail("outcome ids must be unique.", "INVALID_CONFIG");
|
|
221
|
+
const mapped = new Set(outcomes.flatMap((outcome) => outcome.checks));
|
|
222
|
+
if (checks.some((check) => check.required && !mapped.has(check.id))) fail("every required check must map to an outcome.", "INVALID_CONFIG");
|
|
223
|
+
const rawSurfaces = isRecord2(raw["surfaces"]) ? raw["surfaces"] : void 0;
|
|
224
|
+
const surfaces = Object.fromEntries(SURFACE_NAMES.map((name) => [name, surface(rawSurfaces?.[name] ?? name === "logic", name)]));
|
|
225
|
+
for (const name of SURFACE_NAMES) if (surfaces[name].required && !checks.some((check) => check.required && check.category === name)) fail(`required surface ${name} has no required check.`, "INVALID_CONFIG");
|
|
226
|
+
const trackingRaw = isRecord2(raw["tracking"]) ? raw["tracking"] : { required: false, reason: "tracking is not configured for this run." };
|
|
227
|
+
if (trackingRaw["required"] === true && typeof trackingRaw["target"] !== "string") fail("tracking.target is required when tracking is enabled.", "INVALID_CONFIG");
|
|
228
|
+
if (trackingRaw["required"] !== true && typeof trackingRaw["reason"] !== "string") fail("tracking.reason is required when tracking is disabled.", "INVALID_CONFIG");
|
|
229
|
+
const budgetRaw = raw["budget"] === void 0 ? void 0 : asRecord(raw["budget"], "budget");
|
|
230
|
+
if (budgetRaw && budgetRaw["maxDurationMs"] !== void 0 && (!Number.isInteger(budgetRaw["maxDurationMs"]) || typeof budgetRaw["maxDurationMs"] !== "number" || budgetRaw["maxDurationMs"] < 1)) fail("budget.maxDurationMs must be positive.", "INVALID_CONFIG");
|
|
231
|
+
const cleanupRaw = raw["cleanup"] === void 0 ? void 0 : asRecord(raw["cleanup"], "cleanup");
|
|
232
|
+
const cleanup = cleanupRaw ? { roots: cleanupRaw["roots"] === void 0 ? void 0 : stringArray(cleanupRaw["roots"], "cleanup.roots") } : void 0;
|
|
233
|
+
const benchmarkRaw = raw["benchmark"] === void 0 ? void 0 : asRecord(raw["benchmark"], "benchmark");
|
|
234
|
+
const benchmark2 = benchmarkRaw ? { suiteId: stringValue(benchmarkRaw["suiteId"], "benchmark.suiteId"), taskId: stringValue(benchmarkRaw["taskId"], "benchmark.taskId"), mode: benchmarkRaw["mode"] === "harness" ? "harness" : fail("benchmark.mode must be harness.", "INVALID_CONFIG") } : void 0;
|
|
235
|
+
const contract = { intent: stringValue(contractRaw["intent"], "contract.intent"), scope, ambiguities, outcomes };
|
|
236
|
+
const tracking = { required: trackingRaw["required"] === true, ...typeof trackingRaw["target"] === "string" ? { target: trackingRaw["target"] } : {}, ...typeof trackingRaw["reason"] === "string" ? { reason: trackingRaw["reason"] } : {} };
|
|
237
|
+
return { schemaVersion: 1, project, ...typeof raw["root"] === "string" ? { root: raw["root"] } : {}, ...typeof raw["stateDir"] === "string" ? { stateDir: raw["stateDir"] } : {}, profile: typeof raw["profile"] === "string" ? raw["profile"] : "strict", contract, surfaces, checks, tracking, ...budgetRaw ? { budget: { maxDurationMs: budgetRaw["maxDurationMs"] } } : {}, ...cleanup ? { cleanup } : {}, ...benchmark2 ? { benchmark: benchmark2 } : {} };
|
|
238
|
+
};
|
|
239
|
+
var loadConfig = (configPath = ".codex/verification.json") => {
|
|
240
|
+
const absolute = resolve(configPath);
|
|
241
|
+
const raw = readJson(absolute);
|
|
242
|
+
const rawRecord = asRecord(raw, "verification config");
|
|
243
|
+
const root = resolve(dirname(absolute), typeof rawRecord["root"] === "string" ? rawRecord["root"] : ".");
|
|
244
|
+
const stateDir = resolve(root, typeof rawRecord["stateDir"] === "string" ? rawRecord["stateDir"] : ".codex/verification");
|
|
245
|
+
if (!pathInside(root, stateDir)) fail("stateDir must be inside the project root.", "INVALID_CONFIG");
|
|
246
|
+
const config = validateConfig(raw);
|
|
247
|
+
return { absolute, root, stateDir, config, configHash: hashJson(config) };
|
|
248
|
+
};
|
|
249
|
+
|
|
250
|
+
// src/state-machine.ts
|
|
251
|
+
var transition = (run, to, reason, actor = "harness") => {
|
|
252
|
+
if (!STATES.includes(to)) fail(`Unknown state ${to}.`, "INVALID_STATE");
|
|
253
|
+
if (run.state !== to && !LEGAL_TRANSITIONS[run.state].some((state) => state === to)) fail(`Illegal transition ${run.state} -> ${to}.`, "INVALID_STATE");
|
|
254
|
+
const event = { from: run.state, to, at: (/* @__PURE__ */ new Date()).toISOString(), actor, ...reason ? { reason } : {} };
|
|
255
|
+
return { ...run, state: to, transitions: [...run.transitions, event] };
|
|
256
|
+
};
|
|
257
|
+
var assertHuman = (actor) => {
|
|
258
|
+
if (actor !== "human") fail("This action requires --by human.", "HUMAN_APPROVAL_REQUIRED");
|
|
259
|
+
};
|
|
260
|
+
var approvedDecision = (decision) => {
|
|
261
|
+
if (!DECISIONS.has(decision)) fail("Decision must be approved or rejected.", "INVALID_INPUT");
|
|
262
|
+
return ["approved", "approve", "yes", "ok"].includes(decision);
|
|
263
|
+
};
|
|
264
|
+
var HARNESS_EVENT_SCHEMA_VERSION = 1;
|
|
265
|
+
var EVENT_LOG_GENESIS = "GENESIS";
|
|
266
|
+
var HARNESS_EVENT_TYPES = ["run.created", "state.transitioned", "context.attached", "verification.completed", "approval.recorded", "authorization.recorded", "session.started", "session.resumed", "agent.turn.started", "policy.evaluated", "tool.approval.requested", "tool.approval.recorded", "tool.requested", "tool.execution.started", "tool.recovery.recorded", "tool.blocked", "tool.completed", "tool.failed", "session.ended"];
|
|
267
|
+
var SESSION_EVENT_TYPES = /* @__PURE__ */ new Set(["session.started", "session.resumed", "agent.turn.started", "policy.evaluated", "tool.approval.requested", "tool.approval.recorded", "tool.requested", "tool.execution.started", "tool.recovery.recorded", "tool.blocked", "tool.completed", "tool.failed", "session.ended"]);
|
|
268
|
+
var eventPath = (stateDir, runId) => join(stateDir, "runs", runId, "events.ndjson");
|
|
269
|
+
var lockPath = (stateDir, runId) => `${eventPath(stateDir, runId)}.lock`;
|
|
270
|
+
var parseLock = (value) => {
|
|
271
|
+
try {
|
|
272
|
+
const record3 = JSON.parse(value);
|
|
273
|
+
if (!Number.isInteger(record3["pid"]) || record3["pid"] <= 0 || typeof record3["at"] !== "string" || !Number.isFinite(Date.parse(record3["at"]))) fail("Event log lock metadata is invalid.", "HARNESS_ERROR");
|
|
274
|
+
return { pid: record3["pid"], at: record3["at"] };
|
|
275
|
+
} catch (error) {
|
|
276
|
+
if (error instanceof SyntaxError) fail("Event log lock metadata is invalid.", "HARNESS_ERROR");
|
|
277
|
+
throw error;
|
|
278
|
+
}
|
|
279
|
+
};
|
|
280
|
+
var readLock = (stateDir, runId) => {
|
|
281
|
+
const path = lockPath(stateDir, runId);
|
|
282
|
+
return existsSync(path) ? parseLock(readFileSync(path, "utf8")) : null;
|
|
283
|
+
};
|
|
284
|
+
var isEventType = (value) => typeof value === "string" && HARNESS_EVENT_TYPES.includes(value);
|
|
285
|
+
var digest = (value) => /^[a-f0-9]{64}$/.test(value);
|
|
286
|
+
var eventBody = (event) => {
|
|
287
|
+
const { eventHash: _eventHash, ...body2 } = event;
|
|
288
|
+
return body2;
|
|
289
|
+
};
|
|
290
|
+
var eventDigest = (event) => sha256(JSON.stringify(eventBody(event)));
|
|
291
|
+
var parseEvent = (value, expectedSequence) => {
|
|
292
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) fail("Event log contains a non-object record.", "HARNESS_ERROR");
|
|
293
|
+
const record3 = value;
|
|
294
|
+
if (record3["schemaVersion"] !== HARNESS_EVENT_SCHEMA_VERSION || typeof record3["runId"] !== "string" || typeof record3["sequence"] !== "number" || record3["sequence"] !== expectedSequence || typeof record3["at"] !== "string" || typeof record3["sourceRevision"] !== "string" || typeof record3["configHash"] !== "string" || !isEventType(record3["type"]) || typeof record3["payload"] !== "object" || record3["payload"] === null || record3["sessionId"] !== void 0 && (typeof record3["sessionId"] !== "string" || !record3["sessionId"].trim()) || SESSION_EVENT_TYPES.has(record3["type"]) && typeof record3["sessionId"] !== "string") fail("Event log is invalid or out of order.", "HARNESS_ERROR");
|
|
295
|
+
const hasPreviousHash = record3["previousHash"] !== void 0;
|
|
296
|
+
const hasEventHash = record3["eventHash"] !== void 0;
|
|
297
|
+
if (hasPreviousHash !== hasEventHash || hasPreviousHash && (typeof record3["previousHash"] !== "string" || record3["previousHash"] !== EVENT_LOG_GENESIS && !digest(record3["previousHash"]) || typeof record3["eventHash"] !== "string" || !digest(record3["eventHash"]))) fail("Event log integrity metadata is invalid.", "HARNESS_ERROR");
|
|
298
|
+
return record3;
|
|
299
|
+
};
|
|
300
|
+
var validateChain = (events2) => {
|
|
301
|
+
const current = events2.filter((event) => event.eventHash !== void 0);
|
|
302
|
+
if (!current.length) return { status: "legacy", eventCount: events2.length };
|
|
303
|
+
if (current.length !== events2.length) fail("Event log mixes legacy and hashed records.", "HARNESS_ERROR");
|
|
304
|
+
let previous = EVENT_LOG_GENESIS;
|
|
305
|
+
for (const event of events2) {
|
|
306
|
+
const eventHash = event.eventHash ?? fail("Event log hash chain is invalid.", "HARNESS_ERROR");
|
|
307
|
+
if (event.previousHash !== previous || eventHash !== eventDigest(event)) fail("Event log hash chain is invalid.", "HARNESS_ERROR");
|
|
308
|
+
previous = eventHash;
|
|
309
|
+
}
|
|
310
|
+
return { status: "verified", eventCount: events2.length, ...events2.length ? { headHash: previous } : {} };
|
|
311
|
+
};
|
|
312
|
+
var FileEventStore = class {
|
|
313
|
+
constructor(stateDir) {
|
|
314
|
+
this.stateDir = stateDir;
|
|
315
|
+
}
|
|
316
|
+
stateDir;
|
|
317
|
+
append(event) {
|
|
318
|
+
if (!event.runId.trim()) fail("Event runId is required.", "INVALID_INPUT");
|
|
319
|
+
if (!event.sourceRevision.trim() || !event.configHash.trim()) fail("Event sourceRevision and configHash are required.", "INVALID_INPUT");
|
|
320
|
+
if (!isEventType(event.type)) fail("Event type is invalid.", "INVALID_INPUT");
|
|
321
|
+
if (SESSION_EVENT_TYPES.has(event.type) && (!event.sessionId || !event.sessionId.trim())) fail("Session events require a sessionId.", "INVALID_INPUT");
|
|
322
|
+
if (event.sessionId !== void 0 && !event.sessionId.trim()) fail("Event sessionId cannot be empty.", "INVALID_INPUT");
|
|
323
|
+
const path = eventPath(this.stateDir, event.runId);
|
|
324
|
+
const lock = lockPath(this.stateDir, event.runId);
|
|
325
|
+
mkdirSync(join(this.stateDir, "runs", event.runId), { recursive: true });
|
|
326
|
+
let lockFd;
|
|
327
|
+
try {
|
|
328
|
+
lockFd = openSync(lock, "wx");
|
|
329
|
+
writeSync(lockFd, JSON.stringify({ pid: process.pid, at: (/* @__PURE__ */ new Date()).toISOString() }));
|
|
330
|
+
} catch (error) {
|
|
331
|
+
if (error.code === "EEXIST") fail("Event log is busy; retry the operation.", "HARNESS_ERROR");
|
|
332
|
+
throw error;
|
|
333
|
+
}
|
|
334
|
+
try {
|
|
335
|
+
const events2 = this.readUnlocked(event.runId);
|
|
336
|
+
const previous = events2.at(-1);
|
|
337
|
+
const body2 = { schemaVersion: HARNESS_EVENT_SCHEMA_VERSION, sequence: events2.length + 1, at: (/* @__PURE__ */ new Date()).toISOString(), runId: event.runId, sourceRevision: event.sourceRevision, configHash: event.configHash, ...event.sessionId ? { sessionId: event.sessionId } : {}, ...previous?.eventHash ? { previousHash: previous.eventHash } : events2.length ? {} : { previousHash: EVENT_LOG_GENESIS }, type: event.type, payload: event.payload };
|
|
338
|
+
const record3 = events2.length && !previous?.eventHash ? body2 : { ...body2, eventHash: eventDigest(body2) };
|
|
339
|
+
appendFileSync(path, `${JSON.stringify(record3)}
|
|
340
|
+
`, "utf8");
|
|
341
|
+
return record3;
|
|
342
|
+
} finally {
|
|
343
|
+
closeSync(lockFd);
|
|
344
|
+
unlinkSync(lock);
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
readUnlocked(runId) {
|
|
348
|
+
const path = eventPath(this.stateDir, runId);
|
|
349
|
+
if (!existsSync(path)) return [];
|
|
350
|
+
const events2 = readFileSync(path, "utf8").split(/\r?\n/).map((line) => line.trim()).filter(Boolean).map((line, index2) => {
|
|
351
|
+
try {
|
|
352
|
+
return parseEvent(JSON.parse(line), index2 + 1);
|
|
353
|
+
} catch (error) {
|
|
354
|
+
if (error instanceof SyntaxError) fail("Event log contains invalid JSON.", "HARNESS_ERROR");
|
|
355
|
+
throw error;
|
|
356
|
+
}
|
|
357
|
+
});
|
|
358
|
+
validateChain(events2);
|
|
359
|
+
return events2;
|
|
360
|
+
}
|
|
361
|
+
read(runId) {
|
|
362
|
+
if (existsSync(lockPath(this.stateDir, runId))) fail("Event log is busy; retry the operation.", "HARNESS_ERROR");
|
|
363
|
+
return this.readUnlocked(runId);
|
|
364
|
+
}
|
|
365
|
+
verify(runId) {
|
|
366
|
+
return validateChain(this.read(runId));
|
|
367
|
+
}
|
|
368
|
+
};
|
|
369
|
+
var inspectEventLogLock = (stateDir, runId) => {
|
|
370
|
+
const path = lockPath(stateDir, runId);
|
|
371
|
+
const lock = readLock(stateDir, runId);
|
|
372
|
+
return lock ? { status: "locked", path, lock } : { status: "unlocked", path };
|
|
373
|
+
};
|
|
374
|
+
var recoverEventLogLock = ({ stateDir, runId, actor, maxAgeMs = 3e5 }) => {
|
|
375
|
+
if (actor !== "human") fail("Event log lock recovery requires a human actor.", "HUMAN_APPROVAL_REQUIRED");
|
|
376
|
+
if (!Number.isInteger(maxAgeMs) || maxAgeMs < 0) fail("maxAgeMs must be a non-negative integer.", "INVALID_INPUT");
|
|
377
|
+
const path = lockPath(stateDir, runId);
|
|
378
|
+
const lock = readLock(stateDir, runId);
|
|
379
|
+
if (!lock) return { status: "unlocked", path };
|
|
380
|
+
const ageMs = Date.now() - Date.parse(lock.at);
|
|
381
|
+
if (ageMs < maxAgeMs) fail("Event log lock is not old enough to recover.", "HARNESS_ERROR");
|
|
382
|
+
try {
|
|
383
|
+
process.kill(lock.pid, 0);
|
|
384
|
+
} catch (error) {
|
|
385
|
+
if (error.code !== "ESRCH") fail("Event log lock owner cannot be proven dead.", "HARNESS_ERROR");
|
|
386
|
+
unlinkSync(path);
|
|
387
|
+
return { status: "recovered", path, lock };
|
|
388
|
+
}
|
|
389
|
+
return fail("Event log lock owner is still alive.", "HARNESS_ERROR");
|
|
390
|
+
};
|
|
391
|
+
|
|
392
|
+
// src/plugins.ts
|
|
393
|
+
var createPluginSlot = (id2) => {
|
|
394
|
+
if (!id2.trim()) fail("Plugin slot id is required.", "INVALID_INPUT");
|
|
395
|
+
return { id: id2 };
|
|
396
|
+
};
|
|
397
|
+
|
|
398
|
+
// src/context.ts
|
|
399
|
+
var hashContextSnapshot = ({ providerId, query, references, sourceHash: sourceHash2 }) => hashJson({ providerId, query, references, sourceHash: sourceHash2 });
|
|
400
|
+
var hashContextSnapshots = (snapshots) => hashJson(snapshots.map(({ providerId, query, references, sourceHash: sourceHash2, snapshotHash }) => ({ providerId, query, references, sourceHash: sourceHash2, snapshotHash })));
|
|
401
|
+
var record2 = (value, label) => {
|
|
402
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${label} must be an object.`, "INVALID_INPUT");
|
|
403
|
+
return value;
|
|
404
|
+
};
|
|
405
|
+
var requiredString = (value, label) => {
|
|
406
|
+
if (typeof value !== "string" || !value.trim()) fail(`${label} must be a non-empty string.`, "INVALID_INPUT");
|
|
407
|
+
return value;
|
|
408
|
+
};
|
|
409
|
+
var validateContextSnapshot = (value, index2 = 0) => {
|
|
410
|
+
const raw = record2(value, `context snapshot ${index2}`);
|
|
411
|
+
const rawQuery = record2(raw["query"], `context snapshot ${index2}.query`);
|
|
412
|
+
const rawReferences = raw["references"];
|
|
413
|
+
if (!Array.isArray(rawReferences)) fail(`context snapshot ${index2}.references must be an array.`, "INVALID_INPUT");
|
|
414
|
+
const references = rawReferences.map((reference, referenceIndex) => {
|
|
415
|
+
const rawReference = record2(reference, `context snapshot ${index2}.references[${referenceIndex}]`);
|
|
416
|
+
return {
|
|
417
|
+
id: requiredString(rawReference["id"], `context snapshot ${index2}.references[${referenceIndex}].id`),
|
|
418
|
+
uri: requiredString(rawReference["uri"], `context snapshot ${index2}.references[${referenceIndex}].uri`),
|
|
419
|
+
...typeof rawReference["title"] === "string" ? { title: rawReference["title"] } : {},
|
|
420
|
+
...typeof rawReference["version"] === "string" ? { version: rawReference["version"] } : {},
|
|
421
|
+
...typeof rawReference["contentHash"] === "string" ? { contentHash: rawReference["contentHash"] } : {}
|
|
422
|
+
};
|
|
423
|
+
});
|
|
424
|
+
const scope = rawQuery["scope"] === void 0 ? void 0 : Array.isArray(rawQuery["scope"]) && rawQuery["scope"].every((item) => typeof item === "string") ? rawQuery["scope"] : fail(`context snapshot ${index2}.query.scope must be an array of strings.`, "INVALID_INPUT");
|
|
425
|
+
const snapshot = {
|
|
426
|
+
providerId: requiredString(raw["providerId"], `context snapshot ${index2}.providerId`),
|
|
427
|
+
query: { query: requiredString(rawQuery["query"], `context snapshot ${index2}.query.query`), ...scope ? { scope } : {}, ...typeof rawQuery["sourceRevision"] === "string" ? { sourceRevision: rawQuery["sourceRevision"] } : {} },
|
|
428
|
+
references,
|
|
429
|
+
sourceHash: requiredString(raw["sourceHash"], `context snapshot ${index2}.sourceHash`),
|
|
430
|
+
snapshotHash: requiredString(raw["snapshotHash"], `context snapshot ${index2}.snapshotHash`),
|
|
431
|
+
resolvedAt: requiredString(raw["resolvedAt"], `context snapshot ${index2}.resolvedAt`)
|
|
432
|
+
};
|
|
433
|
+
if (snapshot.snapshotHash !== hashContextSnapshot(snapshot)) fail(`context snapshot ${index2}.snapshotHash does not match its contents.`, "INVALID_INPUT");
|
|
434
|
+
return snapshot;
|
|
435
|
+
};
|
|
436
|
+
var readContextSnapshots = (path) => {
|
|
437
|
+
const value = JSON.parse(readFileSync(path, "utf8"));
|
|
438
|
+
return (Array.isArray(value) ? value : [value]).map((snapshot, index2) => validateContextSnapshot(snapshot, index2));
|
|
439
|
+
};
|
|
440
|
+
var validateContextSnapshots = (snapshots) => snapshots.map((snapshot, index2) => validateContextSnapshot(snapshot, index2));
|
|
441
|
+
createPluginSlot("context.provider");
|
|
442
|
+
|
|
443
|
+
// src/runs.ts
|
|
444
|
+
var now = () => (/* @__PURE__ */ new Date()).toISOString();
|
|
445
|
+
var newRunId = () => `${Date.now()}-${process.pid}-${Math.random().toString(36).slice(2, 8)}`;
|
|
446
|
+
var saveRun2 = (stateDir, run) => {
|
|
447
|
+
saveRun(stateDir, run);
|
|
448
|
+
const store = new FileEventStore(stateDir);
|
|
449
|
+
const events2 = store.read(run.runId);
|
|
450
|
+
if (!events2.some((event) => event.type === "run.created")) store.append({ runId: run.runId, sourceRevision: run.sourceRevision, configHash: run.configHash, type: "run.created", payload: { project: run.project, baselineRevision: run.baseline.revision, baselineStatusHash: run.baseline.statusHash } });
|
|
451
|
+
const loggedTransitions = new Set(events2.filter((event) => event.type === "state.transitioned").map((event) => event.payload.transitionIndex));
|
|
452
|
+
run.transitions.forEach((transition2, transitionIndex) => {
|
|
453
|
+
if (loggedTransitions.has(transitionIndex)) return;
|
|
454
|
+
store.append({ runId: run.runId, sourceRevision: run.sourceRevision, configHash: run.configHash, type: "state.transitioned", payload: { from: transition2.from, to: transition2.to, actor: transition2.actor ?? "harness", ...transition2.reason ? { reason: transition2.reason } : {}, transitionIndex } });
|
|
455
|
+
});
|
|
456
|
+
const loggedSnapshots = new Set(events2.filter((event) => event.type === "context.attached").map((event) => event.payload.snapshotHash));
|
|
457
|
+
for (const snapshot of run.contextSnapshots ?? []) {
|
|
458
|
+
if (loggedSnapshots.has(snapshot.snapshotHash)) continue;
|
|
459
|
+
store.append({ runId: run.runId, sourceRevision: run.sourceRevision, configHash: run.configHash, type: "context.attached", payload: { providerId: snapshot.providerId, sourceHash: snapshot.sourceHash, snapshotHash: snapshot.snapshotHash, query: snapshot.query } });
|
|
460
|
+
}
|
|
461
|
+
};
|
|
462
|
+
var createRun = async ({ loaded, baseline, supersedes, dirtyBaselineAuthorized, contextSnapshots = [], planner = "human" }) => {
|
|
463
|
+
const contractHash = hashJson(loaded.config.contract);
|
|
464
|
+
const run = {
|
|
465
|
+
type: "agentskit-harness-run",
|
|
466
|
+
schemaVersion: 1,
|
|
467
|
+
runId: newRunId(),
|
|
468
|
+
project: loaded.config.project,
|
|
469
|
+
state: "PLANNED",
|
|
470
|
+
configHash: loaded.configHash,
|
|
471
|
+
contractHash,
|
|
472
|
+
sourceRevision: baseline.revision,
|
|
473
|
+
sourceStatusHash: baseline.statusHash,
|
|
474
|
+
baseline,
|
|
475
|
+
...planner === "human" ? { contractApproval: { actor: "human", at: now(), contractHash } } : { contractPreparation: { actor: "ci", at: now(), contractHash } },
|
|
476
|
+
checks: loaded.config.checks.map(({ id: id2, category }) => ({ id: id2, category, status: "pending" })),
|
|
477
|
+
contextSnapshots,
|
|
478
|
+
...contextSnapshots.length ? { contextHash: hashContextSnapshots(contextSnapshots) } : {},
|
|
479
|
+
...loaded.config.benchmark ? { benchmark: loaded.config.benchmark } : {},
|
|
480
|
+
outcomes: loaded.config.contract.outcomes.map(({ id: id2, statement, checks }) => ({ id: id2, statement, checks, status: "pending" })),
|
|
481
|
+
transitions: [{ from: null, to: "PLANNED", at: now(), actor: planner }],
|
|
482
|
+
evidenceReferences: [],
|
|
483
|
+
...supersedes ? { supersedes } : {},
|
|
484
|
+
...dirtyBaselineAuthorized ? { dirtyBaselineAuthorized: true } : {}
|
|
485
|
+
};
|
|
486
|
+
saveRun2(loaded.stateDir, run);
|
|
487
|
+
setLatest(loaded.stateDir, run);
|
|
488
|
+
return run;
|
|
489
|
+
};
|
|
490
|
+
var parseStructuredEvidence = (stdout) => {
|
|
491
|
+
for (const line of stdout.split(/\r?\n/).map((item) => item.trim()).filter(Boolean).reverse()) {
|
|
492
|
+
try {
|
|
493
|
+
const value = JSON.parse(line);
|
|
494
|
+
if (typeof value === "object" && value !== null && !Array.isArray(value) && typeof value["status"] === "string") return value;
|
|
495
|
+
} catch {
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
return null;
|
|
499
|
+
};
|
|
500
|
+
var isRecord3 = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
501
|
+
var viewportValid = (viewport) => typeof viewport === "string" || isRecord3(viewport) && typeof viewport["width"] === "number" && viewport["width"] > 0 && typeof viewport["height"] === "number" && viewport["height"] > 0;
|
|
502
|
+
var validateEvidence = (root, check, evidence, outcomeIds) => {
|
|
503
|
+
if (!evidence || evidence.status !== "passed") return ["structured evidence did not pass"];
|
|
504
|
+
if (!Array.isArray(evidence.criteria) || !evidence.criteria.every((id2) => typeof id2 === "string") || outcomeIds.some((id2) => !evidence.criteria.includes(id2))) return [`evidence must map criteria: ${outcomeIds.join(", ")}`];
|
|
505
|
+
const failures = [];
|
|
506
|
+
if (check.category === "ui") {
|
|
507
|
+
if (evidence.capability !== "real-browser") failures.push("UI evidence must declare capability real-browser");
|
|
508
|
+
if (!Array.isArray(evidence.artifacts) || evidence.artifacts.length === 0) failures.push("UI evidence requires screenshot artifacts");
|
|
509
|
+
}
|
|
510
|
+
const artifacts = Array.isArray(evidence.artifacts) ? evidence.artifacts : [];
|
|
511
|
+
for (const artifactValue of artifacts) {
|
|
512
|
+
if (!isRecord3(artifactValue) || typeof artifactValue["path"] !== "string" || typeof artifactValue["sha256"] !== "string") {
|
|
513
|
+
failures.push("artifact requires string path and sha256");
|
|
514
|
+
continue;
|
|
515
|
+
}
|
|
516
|
+
const artifact = artifactValue;
|
|
517
|
+
const artifactPath = join(root, artifact.path);
|
|
518
|
+
if (!pathInside(root, artifactPath)) failures.push(`artifact path escapes project root: ${artifact.path}`);
|
|
519
|
+
else if (!existsSync(artifactPath) || sha256(readFileSync(artifactPath)) !== artifact.sha256) failures.push(`artifact hash mismatch: ${artifact.path}`);
|
|
520
|
+
if (check.category === "ui" && (artifact.type !== "screenshot" || !viewportValid(artifact.viewport))) failures.push(`UI artifact requires type=screenshot and viewport: ${artifact.path}`);
|
|
521
|
+
}
|
|
522
|
+
return failures;
|
|
523
|
+
};
|
|
524
|
+
var execFileAsync = promisify(execFile);
|
|
525
|
+
var git = async (root, args) => {
|
|
526
|
+
try {
|
|
527
|
+
return (await execFileAsync("git", ["-C", root, ...args], { encoding: "utf8" })).stdout.trim();
|
|
528
|
+
} catch {
|
|
529
|
+
return "";
|
|
530
|
+
}
|
|
531
|
+
};
|
|
532
|
+
var sourceSnapshot = async (root, stateDir) => {
|
|
533
|
+
const revision = await git(root, ["rev-parse", "HEAD"]);
|
|
534
|
+
const stateRelative = relative(root, stateDir).replaceAll("\\", "/");
|
|
535
|
+
const pathspec = ["--", "."];
|
|
536
|
+
if (stateRelative && stateRelative !== ".." && !stateRelative.startsWith("../")) pathspec.push(`:(exclude)${stateRelative}`);
|
|
537
|
+
const status = await git(root, ["status", "--porcelain=v1", "--untracked-files=all", ...pathspec]);
|
|
538
|
+
const diff = await git(root, ["diff", "--no-ext-diff", "--binary", "HEAD", ...pathspec]);
|
|
539
|
+
const untrackedPaths = (await git(root, ["ls-files", "--others", "--exclude-standard", "-z"])).split("\0").filter(Boolean).filter((path) => !stateRelative || path !== stateRelative && !path.startsWith(`${stateRelative}/`));
|
|
540
|
+
const untracked = untrackedPaths.flatMap((path) => {
|
|
541
|
+
const absolute = resolve(root, path);
|
|
542
|
+
try {
|
|
543
|
+
return lstatSync(absolute).isFile() ? [{ path, hash: sha256(readFileSync(absolute)) }] : [];
|
|
544
|
+
} catch {
|
|
545
|
+
return [];
|
|
546
|
+
}
|
|
547
|
+
});
|
|
548
|
+
const fingerprint = { revision, status, diff, untracked };
|
|
549
|
+
return { revision: revision || `content:${hashJson(fingerprint)}`, status, statusHash: hashJson(fingerprint) };
|
|
550
|
+
};
|
|
551
|
+
|
|
552
|
+
// src/verification.ts
|
|
553
|
+
var now2 = () => (/* @__PURE__ */ new Date()).toISOString();
|
|
554
|
+
var requireRun = (run) => run ?? fail("No verification run exists.", "NO_RUN");
|
|
555
|
+
var verificationProjection = (run) => ({ checks: run.checks, outcomes: run.outcomes, metrics: run.metrics });
|
|
556
|
+
var verificationDigest = (run) => hashJson(verificationProjection(run));
|
|
557
|
+
var runCommand = (check, cwd) => new Promise((resolveResult) => {
|
|
558
|
+
const started = Date.now();
|
|
559
|
+
const child = spawn(check.command, { cwd, shell: true, env: process.env });
|
|
560
|
+
let stdout = "";
|
|
561
|
+
let stderr = "";
|
|
562
|
+
let timedOut = false;
|
|
563
|
+
const timer = setTimeout(() => {
|
|
564
|
+
timedOut = true;
|
|
565
|
+
child.kill("SIGTERM");
|
|
566
|
+
}, check.timeoutMs);
|
|
567
|
+
child.stdout.on("data", (chunk) => {
|
|
568
|
+
stdout += chunk.toString();
|
|
569
|
+
});
|
|
570
|
+
child.stderr.on("data", (chunk) => {
|
|
571
|
+
stderr += chunk.toString();
|
|
572
|
+
});
|
|
573
|
+
child.on("close", (exitCode) => {
|
|
574
|
+
clearTimeout(timer);
|
|
575
|
+
resolveResult({ exitCode: exitCode ?? 1, timedOut, stdout, stderr, durationMs: Date.now() - started });
|
|
576
|
+
});
|
|
577
|
+
});
|
|
578
|
+
var currentBinding = async (loaded) => ({ source: await sourceSnapshot(loaded.root, loaded.stateDir), configHash: loaded.configHash });
|
|
579
|
+
var staleRun = (loaded, run, reason) => {
|
|
580
|
+
const stale = transition(run, "STALE", reason);
|
|
581
|
+
saveRun2(loaded.stateDir, stale);
|
|
582
|
+
setLatest(loaded.stateDir, stale);
|
|
583
|
+
return fail(reason, "STALE");
|
|
584
|
+
};
|
|
585
|
+
var isFresh = async (loaded, run) => {
|
|
586
|
+
const current = await currentBinding(loaded);
|
|
587
|
+
return current.configHash === run.configHash && current.source.revision === run.sourceRevision && current.source.statusHash === run.sourceStatusHash;
|
|
588
|
+
};
|
|
589
|
+
var planRun = async ({ configPath, decision, actor = "human", allowDirty = false, contextSnapshots = [] }) => {
|
|
590
|
+
const automatedPreparation = actor === "ci" && decision === "prepared";
|
|
591
|
+
if (!automatedPreparation) {
|
|
592
|
+
assertHuman(actor);
|
|
593
|
+
if (!approvedDecision(decision)) fail("Contract was not approved.", "CLARIFYING");
|
|
594
|
+
}
|
|
595
|
+
const loaded = loadConfig(configPath);
|
|
596
|
+
if (loaded.config.contract.ambiguities.length) fail(`Unresolved ambiguities remain: ${loaded.config.contract.ambiguities.join(" | ")}`, "CLARIFYING");
|
|
597
|
+
const validatedContextSnapshots = validateContextSnapshots(contextSnapshots);
|
|
598
|
+
const baseline = await sourceSnapshot(loaded.root, loaded.stateDir);
|
|
599
|
+
const configRelative = relative(loaded.root, loaded.absolute);
|
|
600
|
+
const meaningful = baseline.status.split("\n").filter(Boolean).filter((line) => !line.endsWith(` ${configRelative}`) && !line.endsWith(` ${configRelative.replaceAll("/", "\\")}`));
|
|
601
|
+
if (meaningful.length && !allowDirty) fail(`Worktree is dirty before planning:
|
|
602
|
+
${meaningful.join("\n")}
|
|
603
|
+
Use --allow-dirty only with explicit human authorization.`, "WORKTREE_DIRTY");
|
|
604
|
+
const previous = loadLatestRun(loaded.stateDir);
|
|
605
|
+
if (previous && !["STALE", "SUPERSEDED"].includes(previous.state)) fail(`An active run already exists: ${previous.runId} (${previous.state}).`, "ACTIVE_RUN");
|
|
606
|
+
return createRun({ loaded, baseline, supersedes: previous?.runId, dirtyBaselineAuthorized: allowDirty, contextSnapshots: validatedContextSnapshots, planner: automatedPreparation ? "ci" : "human" });
|
|
607
|
+
};
|
|
608
|
+
var startRun = (loaded) => {
|
|
609
|
+
const run = requireRun(loadLatestRun(loaded.stateDir));
|
|
610
|
+
const next = transition(run, "IMPLEMENTING", "Implementation started.", "agent");
|
|
611
|
+
saveRun2(loaded.stateDir, next);
|
|
612
|
+
setLatest(loaded.stateDir, next);
|
|
613
|
+
return next;
|
|
614
|
+
};
|
|
615
|
+
var cancelRun = async ({ configPath, runId, reason = "Run cancelled by a human.", actor = "human" }) => {
|
|
616
|
+
assertHuman(actor);
|
|
617
|
+
const loaded = loadConfig(configPath);
|
|
618
|
+
const run = requireRun(runId ? readRun(loaded.stateDir, runId) : loadLatestRun(loaded.stateDir));
|
|
619
|
+
const next = transition(run, "CANCELLED", reason, "human");
|
|
620
|
+
saveRun2(loaded.stateDir, next);
|
|
621
|
+
setLatest(loaded.stateDir, next);
|
|
622
|
+
return next;
|
|
623
|
+
};
|
|
624
|
+
var verifyRun = async ({ configPath }) => {
|
|
625
|
+
const loaded = loadConfig(configPath);
|
|
626
|
+
const run = requireRun(loadLatestRun(loaded.stateDir));
|
|
627
|
+
if (!["IMPLEMENTING", "VERIFYING"].includes(run.state)) {
|
|
628
|
+
if (["AWAITING_HUMAN_APPROVAL", "AWAITING_AUTHORIZATION", "COMPLETE"].includes(run.state) && !await isFresh(loaded, run)) staleRun(loaded, run, "Run is stale because source or contract changed.");
|
|
629
|
+
fail(`Cannot verify from ${run.state}.`, "INVALID_STATE");
|
|
630
|
+
}
|
|
631
|
+
if (run.configHash !== loaded.configHash) staleRun(loaded, run, "Run is stale because the verification contract changed.");
|
|
632
|
+
const binding = await currentBinding(loaded);
|
|
633
|
+
let current = { ...transition(run, "VERIFYING", "Verification started.", "agent"), sourceRevision: binding.source.revision, sourceStatusHash: binding.source.statusHash };
|
|
634
|
+
saveRun2(loaded.stateDir, current);
|
|
635
|
+
const checkDir = join(loaded.stateDir, "runs", current.runId, "checks");
|
|
636
|
+
mkdirSync(checkDir, { recursive: true });
|
|
637
|
+
const outcomesByCheck = new Map(loaded.config.checks.map((check) => [check.id, loaded.config.contract.outcomes.filter((outcome) => outcome.checks.includes(check.id)).map((outcome) => outcome.id)]));
|
|
638
|
+
let totalDurationMs = 0;
|
|
639
|
+
for (const check of loaded.config.checks) {
|
|
640
|
+
const result = await runCommand(check, loaded.root);
|
|
641
|
+
totalDurationMs += result.durationMs;
|
|
642
|
+
const stdoutPath = join(checkDir, `${check.id}.stdout`);
|
|
643
|
+
const stderrPath = join(checkDir, `${check.id}.stderr`);
|
|
644
|
+
writeFileSync(stdoutPath, result.stdout, "utf8");
|
|
645
|
+
writeFileSync(stderrPath, result.stderr, "utf8");
|
|
646
|
+
const evidence = parseStructuredEvidence(result.stdout);
|
|
647
|
+
const failures = result.exitCode === 0 && !result.timedOut && evidence ? validateEvidence(loaded.root, check, evidence, outcomesByCheck.get(check.id) ?? []) : [result.timedOut ? "check timed out" : result.exitCode !== 0 ? `exit code ${result.exitCode}` : "missing final structured evidence"];
|
|
648
|
+
const nextCheck = { id: check.id, category: check.category, status: failures.length ? "failed" : "passed", exitCode: result.exitCode, durationMs: result.durationMs, ...evidence ? { evidence } : {}, ...failures.length ? { failures } : {} };
|
|
649
|
+
current = { ...current, evidenceReferences: [...current.evidenceReferences, { checkId: check.id, stdout: relative(loaded.stateDir, stdoutPath), stderr: relative(loaded.stateDir, stderrPath) }], checks: current.checks.map((item) => item.id === check.id ? nextCheck : item) };
|
|
650
|
+
saveRun2(loaded.stateDir, current);
|
|
651
|
+
}
|
|
652
|
+
const statuses = new Map(current.checks.map((check) => [check.id, check.status]));
|
|
653
|
+
const budgetExceeded = loaded.config.budget?.maxDurationMs !== void 0 && totalDurationMs > loaded.config.budget.maxDurationMs;
|
|
654
|
+
const allPassed = loaded.config.checks.every((check) => statuses.get(check.id) === "passed") && !budgetExceeded;
|
|
655
|
+
current = { ...current, outcomes: current.outcomes.map((outcome) => ({ ...outcome, status: outcome.checks.every((id2) => statuses.get(id2) === "passed") ? "passed" : "failed" })), metrics: { totalDurationMs, budgetExceeded } };
|
|
656
|
+
const digest2 = verificationDigest(current);
|
|
657
|
+
current = { ...current, verificationDigest: digest2 };
|
|
658
|
+
saveRun2(loaded.stateDir, current);
|
|
659
|
+
new FileEventStore(loaded.stateDir).append({ runId: current.runId, sourceRevision: current.sourceRevision, configHash: current.configHash, type: "verification.completed", payload: { verificationDigest: digest2, checkCount: current.checks.length, outcomeCount: current.outcomes.length, totalDurationMs, budgetExceeded } });
|
|
660
|
+
const nextState = allPassed ? "AWAITING_HUMAN_APPROVAL" : "BLOCKED";
|
|
661
|
+
current = { ...transition(current, nextState, allPassed ? "All configured checks passed; human approval is required." : budgetExceeded ? "Verification budget was exceeded." : "A configured check failed or lacked structured evidence.", "harness") };
|
|
662
|
+
saveRun2(loaded.stateDir, current);
|
|
663
|
+
setLatest(loaded.stateDir, current);
|
|
664
|
+
return current;
|
|
665
|
+
};
|
|
666
|
+
var assertFresh = async (loaded, run) => {
|
|
667
|
+
if (!await isFresh(loaded, run)) staleRun(loaded, run, "Run is stale because source or worktree changed after verification.");
|
|
668
|
+
};
|
|
669
|
+
var assertVerificationAttestation = (loaded, run) => {
|
|
670
|
+
const expected = verificationDigest(run);
|
|
671
|
+
if (run.verificationDigest !== expected) fail("Verification projection attestation does not match run.json.", "HARNESS_ERROR");
|
|
672
|
+
const event = new FileEventStore(loaded.stateDir).read(run.runId).filter((item) => item.type === "verification.completed").at(-1);
|
|
673
|
+
if (!event || event.payload.verificationDigest !== expected || event.sourceRevision !== run.sourceRevision || event.configHash !== run.configHash) fail("Verification projection attestation is missing from the event log.", "HARNESS_ERROR");
|
|
674
|
+
};
|
|
675
|
+
var plannerForRetry = (loaded, run) => {
|
|
676
|
+
if (run.contractPreparation) return "ci";
|
|
677
|
+
if (!run.supersedes) return "human";
|
|
678
|
+
return plannerForRetry(loaded, readRun(loaded.stateDir, run.supersedes));
|
|
679
|
+
};
|
|
680
|
+
var recordDecision = (loaded, run, type, payload) => {
|
|
681
|
+
new FileEventStore(loaded.stateDir).append({ runId: run.runId, sourceRevision: run.sourceRevision, configHash: run.configHash, type, payload: type === "authorization.recorded" ? { ...payload, target: payload.target ?? fail("tracking.target is required when authorizing.", "INVALID_CONFIG") } : payload });
|
|
682
|
+
};
|
|
683
|
+
var assertDecisionProjection = (run, decision, expectedState) => {
|
|
684
|
+
if (decision.decision !== "approved" || decision.resultingState !== expectedState || decision.verificationDigest !== run.verificationDigest || decision.sourceRevision !== run.sourceRevision || decision.contractHash !== run.contractHash) fail("Terminal decision attestation does not match the run projection.", "HARNESS_ERROR");
|
|
685
|
+
};
|
|
686
|
+
var reconcileRun = async ({ configPath, runId }) => {
|
|
687
|
+
const loaded = loadConfig(configPath);
|
|
688
|
+
const run = requireRun(runId ? readRun(loaded.stateDir, runId) : loadLatestRun(loaded.stateDir));
|
|
689
|
+
await assertFresh(loaded, run);
|
|
690
|
+
const store = new FileEventStore(loaded.stateDir);
|
|
691
|
+
const eventLog = store.verify(run.runId);
|
|
692
|
+
const events2 = store.read(run.runId);
|
|
693
|
+
if (events2.some((event) => event.runId !== run.runId || event.configHash !== run.configHash)) fail("Run event log is not bound to the current run projection.", "HARNESS_ERROR");
|
|
694
|
+
const requiresVerification = ["AWAITING_HUMAN_APPROVAL", "AWAITING_AUTHORIZATION", "COMPLETE"].includes(run.state);
|
|
695
|
+
if (requiresVerification) {
|
|
696
|
+
if (eventLog.status !== "verified") fail("Terminal run requires a verified event log.", "HARNESS_ERROR");
|
|
697
|
+
assertVerificationAttestation(loaded, run);
|
|
698
|
+
}
|
|
699
|
+
if (run.state === "AWAITING_AUTHORIZATION" || run.state === "COMPLETE") {
|
|
700
|
+
const approval = events2.filter((event) => event.type === "approval.recorded").at(-1) ?? fail("Terminal run is missing its human approval event.", "HARNESS_ERROR");
|
|
701
|
+
assertDecisionProjection(run, approval.payload, run.state === "COMPLETE" && !loaded.config.tracking.required ? "COMPLETE" : "AWAITING_AUTHORIZATION");
|
|
702
|
+
if (!run.humanApproval || run.humanApproval.actor !== "human" || run.humanApproval.verificationDigest !== run.verificationDigest || run.humanApproval.sourceRevision !== run.sourceRevision || run.humanApproval.contractHash !== run.contractHash) fail("Human approval projection is inconsistent with its audit event.", "HARNESS_ERROR");
|
|
703
|
+
}
|
|
704
|
+
if (run.state === "COMPLETE" && loaded.config.tracking.required) {
|
|
705
|
+
const authorization = events2.filter((event) => event.type === "authorization.recorded").at(-1) ?? fail("Complete tracked run is missing its authorization event.", "HARNESS_ERROR");
|
|
706
|
+
assertDecisionProjection(run, authorization.payload, "COMPLETE");
|
|
707
|
+
if (!run.authorization || run.authorization.actor !== "human" || run.authorization.verificationDigest !== run.verificationDigest || run.authorization.target !== authorization.payload.target || run.authorization.sourceRevision !== run.sourceRevision || run.authorization.contractHash !== run.contractHash) fail("Authorization projection is inconsistent with its audit event.", "HARNESS_ERROR");
|
|
708
|
+
}
|
|
709
|
+
return { status: "verified", runId: run.runId, state: run.state, eventCount: eventLog.eventCount, ...eventLog.headHash ? { headHash: eventLog.headHash } : {}, ...run.verificationDigest ? { verificationDigest: run.verificationDigest } : {} };
|
|
710
|
+
};
|
|
711
|
+
var approveRun = async ({ configPath, runId, decision, actor = "human" }) => {
|
|
712
|
+
assertHuman(actor);
|
|
713
|
+
const loaded = loadConfig(configPath);
|
|
714
|
+
const run = requireRun(runId ? readRun(loaded.stateDir, runId) : loadLatestRun(loaded.stateDir));
|
|
715
|
+
if (run.state !== "AWAITING_HUMAN_APPROVAL") fail(`Cannot approve from ${run.state}.`, "INVALID_STATE");
|
|
716
|
+
await assertFresh(loaded, run);
|
|
717
|
+
assertVerificationAttestation(loaded, run);
|
|
718
|
+
if (!approvedDecision(decision)) {
|
|
719
|
+
const blocked = transition(run, "BLOCKED", "Human rejected the verification result.", "human");
|
|
720
|
+
saveRun2(loaded.stateDir, blocked);
|
|
721
|
+
recordDecision(loaded, run, "approval.recorded", { decision: "rejected", resultingState: blocked.state, verificationDigest: run.verificationDigest, actor: "human", sourceRevision: run.sourceRevision, contractHash: run.contractHash });
|
|
722
|
+
setLatest(loaded.stateDir, blocked);
|
|
723
|
+
return blocked;
|
|
724
|
+
}
|
|
725
|
+
const nextState = loaded.config.tracking.required ? "AWAITING_AUTHORIZATION" : "COMPLETE";
|
|
726
|
+
const next = { ...transition(run, nextState, "Human approved the verification result.", "human"), humanApproval: { actor: "human", at: now2(), sourceRevision: run.sourceRevision, contractHash: run.contractHash, verificationDigest: run.verificationDigest } };
|
|
727
|
+
saveRun2(loaded.stateDir, next);
|
|
728
|
+
recordDecision(loaded, run, "approval.recorded", { decision: "approved", resultingState: nextState, verificationDigest: run.verificationDigest, actor: "human", sourceRevision: run.sourceRevision, contractHash: run.contractHash });
|
|
729
|
+
setLatest(loaded.stateDir, next);
|
|
730
|
+
return next;
|
|
731
|
+
};
|
|
732
|
+
var authorizeRun = async ({ configPath, runId, decision, actor = "human" }) => {
|
|
733
|
+
assertHuman(actor);
|
|
734
|
+
const loaded = loadConfig(configPath);
|
|
735
|
+
const run = requireRun(runId ? readRun(loaded.stateDir, runId) : loadLatestRun(loaded.stateDir));
|
|
736
|
+
if (run.state !== "AWAITING_AUTHORIZATION") fail(`Cannot authorize from ${run.state}.`, "INVALID_STATE");
|
|
737
|
+
await assertFresh(loaded, run);
|
|
738
|
+
assertVerificationAttestation(loaded, run);
|
|
739
|
+
if (!approvedDecision(decision)) {
|
|
740
|
+
const blocked = transition(run, "BLOCKED", "Human rejected external tracking authorization.", "human");
|
|
741
|
+
saveRun2(loaded.stateDir, blocked);
|
|
742
|
+
recordDecision(loaded, run, "authorization.recorded", { decision: "rejected", resultingState: blocked.state, verificationDigest: run.verificationDigest, actor: "human", target: loaded.config.tracking.target ?? "", sourceRevision: run.sourceRevision, contractHash: run.contractHash });
|
|
743
|
+
setLatest(loaded.stateDir, blocked);
|
|
744
|
+
return blocked;
|
|
745
|
+
}
|
|
746
|
+
if (!loaded.config.tracking.target) fail("tracking.target is required when authorizing.", "INVALID_CONFIG");
|
|
747
|
+
const next = { ...transition(run, "COMPLETE", "External tracking was authorized.", "human"), authorization: { actor: "human", at: now2(), target: loaded.config.tracking.target, sourceRevision: run.sourceRevision, contractHash: run.contractHash, verificationDigest: run.verificationDigest } };
|
|
748
|
+
saveRun2(loaded.stateDir, next);
|
|
749
|
+
recordDecision(loaded, run, "authorization.recorded", { decision: "approved", resultingState: "COMPLETE", verificationDigest: run.verificationDigest, actor: "human", target: loaded.config.tracking.target, sourceRevision: run.sourceRevision, contractHash: run.contractHash });
|
|
750
|
+
setLatest(loaded.stateDir, next);
|
|
751
|
+
return next;
|
|
752
|
+
};
|
|
753
|
+
var retryRun = async ({ configPath }) => {
|
|
754
|
+
const loaded = loadConfig(configPath);
|
|
755
|
+
const previous = loadLatestRun(loaded.stateDir);
|
|
756
|
+
const previousRun = requireRun(previous);
|
|
757
|
+
if (!["BLOCKED", "STALE", "CANCELLED"].includes(previousRun.state)) fail(`Cannot retry from ${previousRun.state}.`, "INVALID_STATE");
|
|
758
|
+
const baseline = await sourceSnapshot(loaded.root, loaded.stateDir);
|
|
759
|
+
const superseded = transition(previousRun, "SUPERSEDED", "Retry superseded the previous run.", "harness");
|
|
760
|
+
saveRun2(loaded.stateDir, superseded);
|
|
761
|
+
const run = await createRun({ loaded, baseline, supersedes: previousRun.runId, dirtyBaselineAuthorized: previousRun.dirtyBaselineAuthorized, planner: plannerForRetry(loaded, previousRun) });
|
|
762
|
+
const next = transition(run, "IMPLEMENTING", "Retry started after a previous attempt.", "agent");
|
|
763
|
+
saveRun2(loaded.stateDir, next);
|
|
764
|
+
setLatest(loaded.stateDir, next);
|
|
765
|
+
return next;
|
|
766
|
+
};
|
|
767
|
+
var cleanTaskArtifacts = (configPath) => cleanConfiguredArtifacts(loadConfig(configPath));
|
|
768
|
+
var index = (root, indexPath) => JSON.parse(readFileSync(resolve(root, indexPath), "utf8"));
|
|
769
|
+
var text = (entry) => [entry.id, entry.type, entry.title, entry.path, entry.description, entry.body, ...Array.isArray(entry.tags) ? entry.tags : []].filter((value) => typeof value === "string").join(" ").toLowerCase();
|
|
770
|
+
var sourceHash = (document) => typeof document.contentHash === "string" && document.contentHash.length > 0 ? document.contentHash : hashJson(document);
|
|
771
|
+
var matches = (entry, query) => {
|
|
772
|
+
const needle = query.query.trim().toLowerCase();
|
|
773
|
+
const scopes = query.scope?.map((scope) => scope.toLowerCase()) ?? [];
|
|
774
|
+
const value = text(entry);
|
|
775
|
+
return Boolean(needle && value.includes(needle) && (scopes.length === 0 || scopes.some((scope) => value.includes(scope))));
|
|
776
|
+
};
|
|
777
|
+
var createDocBridgeContextProvider = ({ root, indexPath = ".doc-bridge/index.json" }) => ({
|
|
778
|
+
id: "doc-bridge",
|
|
779
|
+
version: "1.0.0",
|
|
780
|
+
resolve: async (query) => {
|
|
781
|
+
const document = index(root, indexPath);
|
|
782
|
+
const contentHash = sourceHash(document);
|
|
783
|
+
const entries = Array.isArray(document.knowledge) ? document.knowledge.filter((value) => typeof value === "object" && value !== null && !Array.isArray(value)).filter((entry) => matches(entry, query)).sort((left, right) => String(left.id ?? "").localeCompare(String(right.id ?? ""))).slice(0, 8) : [];
|
|
784
|
+
const references = entries.flatMap((entry) => typeof entry.id === "string" && typeof entry.path === "string" ? [{ id: entry.id, uri: `doc-bridge://${entry.path}`, ...typeof entry.title === "string" ? { title: entry.title } : {}, contentHash }] : []);
|
|
785
|
+
return { providerId: "doc-bridge", query, references, sourceHash: contentHash, snapshotHash: hashContextSnapshot({ providerId: "doc-bridge", query, references, sourceHash: contentHash }), resolvedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
786
|
+
}
|
|
787
|
+
});
|
|
788
|
+
var BENCHMARK_SCHEMA_VERSION = 1;
|
|
789
|
+
var percentage = (part, total) => total ? Number((part / total).toFixed(4)) : null;
|
|
790
|
+
var DEFAULT_POLICY = { minComparableTasks: 3, maxDurationRegressionRate: 0.2, minCompletedRunsPerTask: 3, minBaselineSamplesPerTask: 3, requireZeroEscapedIncomplete: true };
|
|
791
|
+
var improvementRate = (baseline, current) => baseline === void 0 || current === void 0 || baseline === 0 ? null : Number(((baseline - current) / baseline).toFixed(4));
|
|
792
|
+
var increaseRate = (baseline, current) => baseline === void 0 || current === void 0 || baseline === 0 ? null : Number(((current - baseline) / baseline).toFixed(4));
|
|
793
|
+
var increaseDelta = (baseline, current) => baseline === void 0 || current === null ? null : Number((current - baseline).toFixed(4));
|
|
794
|
+
var increaseDirection = (rate2, delta) => delta === null ? improvementDirection(rate2) : delta > 0 ? "improved" : delta < 0 ? "regressed" : "unchanged";
|
|
795
|
+
var improvementDirection = (rate2) => rate2 === null ? "unavailable" : rate2 > 0 ? "improved" : rate2 < 0 ? "regressed" : "unchanged";
|
|
796
|
+
var count = (items, predicate) => items.filter(predicate).length;
|
|
797
|
+
var median = (values) => {
|
|
798
|
+
if (!values.length) return null;
|
|
799
|
+
const sorted = [...values].sort((left, right) => left - right);
|
|
800
|
+
const middle = Math.floor(sorted.length / 2);
|
|
801
|
+
return sorted.length % 2 ? sorted[middle] ?? null : ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2;
|
|
802
|
+
};
|
|
803
|
+
var reviewMinutes = (run) => {
|
|
804
|
+
if (!run.humanApproval) return void 0;
|
|
805
|
+
const reviewStart = run.transitions.find((transition2) => transition2.to === "AWAITING_HUMAN_APPROVAL")?.at;
|
|
806
|
+
if (!reviewStart) return void 0;
|
|
807
|
+
const elapsed = Date.parse(run.humanApproval.at) - Date.parse(reviewStart);
|
|
808
|
+
return Number.isFinite(elapsed) && elapsed >= 0 ? Number((elapsed / 6e4).toFixed(2)) : void 0;
|
|
809
|
+
};
|
|
810
|
+
var confidence = (comparable, completedRuns, policy) => !comparable ? "insufficient" : completedRuns >= policy.minCompletedRunsPerTask ? "reliable" : "directional";
|
|
811
|
+
var artifactAcceptanceRate = (run) => {
|
|
812
|
+
const rates = run.checks.flatMap((check) => {
|
|
813
|
+
const evidence = check.evidence;
|
|
814
|
+
if (!evidence) return [];
|
|
815
|
+
const direct = typeof evidence["artifactAcceptanceRate"] === "number" ? [evidence["artifactAcceptanceRate"]] : [];
|
|
816
|
+
const reports = Array.isArray(evidence["reports"]) ? evidence["reports"].flatMap((report) => typeof report === "object" && report !== null && typeof report["artifactAcceptanceRate"] === "number" ? [report["artifactAcceptanceRate"]] : []) : [];
|
|
817
|
+
return [...direct, ...reports].filter((rate2) => Number.isFinite(rate2) && rate2 >= 0 && rate2 <= 1);
|
|
818
|
+
});
|
|
819
|
+
return rates.length ? Number((rates.reduce((total, rate2) => total + rate2, 0) / rates.length).toFixed(4)) : void 0;
|
|
820
|
+
};
|
|
821
|
+
var projectRun = (run) => {
|
|
822
|
+
const checks = { total: run.checks.length, passed: count(run.checks, (check) => check.status === "passed"), failed: count(run.checks, (check) => check.status === "failed") };
|
|
823
|
+
const outcomes = { total: run.outcomes.length, passed: count(run.outcomes, (outcome) => outcome.status === "passed"), failed: count(run.outcomes, (outcome) => outcome.status === "failed") };
|
|
824
|
+
const evidence = { total: run.checks.length, attached: count(run.checks, (check) => check.evidence !== void 0) };
|
|
825
|
+
const acceptanceRate = artifactAcceptanceRate(run);
|
|
826
|
+
const humanReviewMinutes = reviewMinutes(run);
|
|
827
|
+
const escapedIncomplete = run.state === "COMPLETE" && checks.failed === 0 && outcomes.failed === 0 && evidence.attached === evidence.total ? 0 : void 0;
|
|
828
|
+
return { runId: run.runId, state: run.state, sourceRevision: run.sourceRevision, configHash: run.configHash, contractHash: run.contractHash, ...run.supersedes ? { supersedes: run.supersedes } : {}, ...run.metrics ? { durationMs: run.metrics.totalDurationMs } : {}, checks, outcomes, evidence, ...acceptanceRate === void 0 ? {} : { artifactAcceptanceRate: acceptanceRate }, ...escapedIncomplete === void 0 ? {} : { escapedIncomplete }, humanApproved: run.humanApproval !== void 0, ...humanReviewMinutes === void 0 ? {} : { humanReviewMinutes }, authorized: run.authorization !== void 0, ...run.benchmark ? { benchmark: run.benchmark } : {} };
|
|
829
|
+
};
|
|
830
|
+
var summarize = (runs) => {
|
|
831
|
+
const stateCounts = Object.fromEntries(RUN_STATES.map((state) => [state, count(runs, (run) => run.state === state)]));
|
|
832
|
+
const checksTotal = runs.reduce((total, run) => total + run.checks.total, 0);
|
|
833
|
+
const checksPassed = runs.reduce((total, run) => total + run.checks.passed, 0);
|
|
834
|
+
const outcomesTotal = runs.reduce((total, run) => total + run.outcomes.total, 0);
|
|
835
|
+
const outcomesPassed = runs.reduce((total, run) => total + run.outcomes.passed, 0);
|
|
836
|
+
const evidenceTotal = runs.reduce((total, run) => total + run.evidence.total, 0);
|
|
837
|
+
const evidenceAttached = runs.reduce((total, run) => total + run.evidence.attached, 0);
|
|
838
|
+
const firstAttempts = runs.filter((run) => !run.supersedes);
|
|
839
|
+
const superseded = new Set(runs.flatMap((run) => run.supersedes ? [run.supersedes] : []));
|
|
840
|
+
const effectiveRuns = runs.filter((run) => !superseded.has(run.runId));
|
|
841
|
+
const effectiveChecksTotal = effectiveRuns.reduce((total, run) => total + run.checks.total, 0);
|
|
842
|
+
const effectiveChecksPassed = effectiveRuns.reduce((total, run) => total + run.checks.passed, 0);
|
|
843
|
+
const effectiveOutcomesTotal = effectiveRuns.reduce((total, run) => total + run.outcomes.total, 0);
|
|
844
|
+
const effectiveOutcomesPassed = effectiveRuns.reduce((total, run) => total + run.outcomes.passed, 0);
|
|
845
|
+
const effectiveEvidenceTotal = effectiveRuns.reduce((total, run) => total + run.evidence.total, 0);
|
|
846
|
+
const effectiveEvidenceAttached = effectiveRuns.reduce((total, run) => total + run.evidence.attached, 0);
|
|
847
|
+
const durations = runs.flatMap((run) => run.durationMs === void 0 ? [] : [run.durationMs]);
|
|
848
|
+
return {
|
|
849
|
+
totalRuns: runs.length,
|
|
850
|
+
stateCounts,
|
|
851
|
+
completeRuns: stateCounts.COMPLETE,
|
|
852
|
+
retriedRuns: count(runs, (run) => run.supersedes !== void 0),
|
|
853
|
+
staleRuns: stateCounts.STALE,
|
|
854
|
+
firstAttemptRuns: firstAttempts.length,
|
|
855
|
+
humanApprovedRuns: count(runs, (run) => run.humanApproved),
|
|
856
|
+
authorizedRuns: count(runs, (run) => run.authorized),
|
|
857
|
+
effectiveRunCount: effectiveRuns.length,
|
|
858
|
+
effectiveCompleteRuns: count(effectiveRuns, (run) => run.state === "COMPLETE"),
|
|
859
|
+
effectiveCompletionRate: percentage(count(effectiveRuns, (run) => run.state === "COMPLETE"), effectiveRuns.length),
|
|
860
|
+
effectiveCheckPassRate: percentage(effectiveChecksPassed, effectiveChecksTotal),
|
|
861
|
+
effectiveOutcomePassRate: percentage(effectiveOutcomesPassed, effectiveOutcomesTotal),
|
|
862
|
+
effectiveEvidenceCoverageRate: percentage(effectiveEvidenceAttached, effectiveEvidenceTotal),
|
|
863
|
+
checkPassRate: percentage(checksPassed, checksTotal),
|
|
864
|
+
outcomePassRate: percentage(outcomesPassed, outcomesTotal),
|
|
865
|
+
evidenceCoverageRate: percentage(evidenceAttached, evidenceTotal),
|
|
866
|
+
firstAttemptApprovalRate: percentage(count(firstAttempts, (run) => run.humanApproved), firstAttempts.length),
|
|
867
|
+
retryRate: percentage(count(runs, (run) => run.supersedes !== void 0), runs.length),
|
|
868
|
+
staleRate: percentage(stateCounts.STALE, runs.length),
|
|
869
|
+
averageDurationMs: durations.length ? Math.round(durations.reduce((total, duration) => total + duration, 0) / durations.length) : null,
|
|
870
|
+
medianDurationMs: median(durations)
|
|
871
|
+
};
|
|
872
|
+
};
|
|
873
|
+
var readRuns = (stateDir) => {
|
|
874
|
+
const runsDir = join(stateDir, "runs");
|
|
875
|
+
if (!existsSync(runsDir)) return [];
|
|
876
|
+
return readdirSync(runsDir, { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => {
|
|
877
|
+
try {
|
|
878
|
+
const candidate = readJson(join(runsDir, entry.name, "run.json"));
|
|
879
|
+
return candidate.type === "agentskit-harness-run" ? readRun(stateDir, entry.name) : void 0;
|
|
880
|
+
} catch (error) {
|
|
881
|
+
return fail(`Benchmark could not read run ${entry.name}: ${error instanceof Error ? error.message : String(error)}`, "HARNESS_ERROR");
|
|
882
|
+
}
|
|
883
|
+
}).filter((run) => run !== void 0);
|
|
884
|
+
};
|
|
885
|
+
var nonEmptyString = (value, label) => {
|
|
886
|
+
if (typeof value !== "string") return fail(`${label} must be a non-empty string.`, "INVALID_CONFIG");
|
|
887
|
+
const result = value.trim();
|
|
888
|
+
if (!result) return fail(`${label} must be a non-empty string.`, "INVALID_CONFIG");
|
|
889
|
+
return result;
|
|
890
|
+
};
|
|
891
|
+
var sha2562 = (value, label) => {
|
|
892
|
+
if (value === void 0) return void 0;
|
|
893
|
+
const result = nonEmptyString(value, label);
|
|
894
|
+
if (!/^[a-f0-9]{64}$/.test(result)) return fail(`${label} must be a lowercase SHA-256 digest.`, "INVALID_CONFIG");
|
|
895
|
+
return result;
|
|
896
|
+
};
|
|
897
|
+
var stringList = (value, label) => {
|
|
898
|
+
if (!Array.isArray(value)) return fail(`${label} must be a non-empty string array.`, "INVALID_CONFIG");
|
|
899
|
+
const items = value;
|
|
900
|
+
if (!items.length || !items.every((item) => typeof item === "string" && Boolean(item.trim()))) return fail(`${label} must be a non-empty string array.`, "INVALID_CONFIG");
|
|
901
|
+
return items.map((item) => String(item).trim());
|
|
902
|
+
};
|
|
903
|
+
var nonNegativeNumber = (value, label) => {
|
|
904
|
+
if (value === void 0) return void 0;
|
|
905
|
+
if (typeof value !== "number") return fail(`${label} must be a non-negative number.`, "INVALID_CONFIG");
|
|
906
|
+
if (!Number.isFinite(value) || value < 0) return fail(`${label} must be a non-negative number.`, "INVALID_CONFIG");
|
|
907
|
+
const result = value;
|
|
908
|
+
return result;
|
|
909
|
+
};
|
|
910
|
+
var nonNegativeInteger = (value, label) => {
|
|
911
|
+
const result = nonNegativeNumber(value, label);
|
|
912
|
+
if (result !== void 0 && !Number.isInteger(result)) return fail(`${label} must be an integer.`, "INVALID_CONFIG");
|
|
913
|
+
return result;
|
|
914
|
+
};
|
|
915
|
+
var rate = (value, label) => {
|
|
916
|
+
const result = nonNegativeNumber(value, label);
|
|
917
|
+
if (result !== void 0 && result > 1) return fail(`${label} must be between 0 and 1.`, "INVALID_CONFIG");
|
|
918
|
+
return result;
|
|
919
|
+
};
|
|
920
|
+
var timestamp = (value, label) => {
|
|
921
|
+
const result = nonEmptyString(value, label);
|
|
922
|
+
if (!Number.isFinite(Date.parse(result))) return fail(`${label} must be a valid timestamp.`, "INVALID_CONFIG");
|
|
923
|
+
return result;
|
|
924
|
+
};
|
|
925
|
+
var relativePath = (value, label) => {
|
|
926
|
+
const result = nonEmptyString(value, label);
|
|
927
|
+
if (result.startsWith("/") || result.split("/").includes("..")) return fail(`${label} must be a repository-relative path.`, "INVALID_CONFIG");
|
|
928
|
+
return result;
|
|
929
|
+
};
|
|
930
|
+
var taskFile = (value, label) => {
|
|
931
|
+
if (value === void 0) return void 0;
|
|
932
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return fail(`${label} must be an object.`, "INVALID_CONFIG");
|
|
933
|
+
const raw = value;
|
|
934
|
+
return { path: relativePath(raw["path"], `${label}.path`), sha256: sha2562(raw["sha256"], `${label}.sha256`) ?? fail(`${label}.sha256 is required.`, "INVALID_CONFIG") };
|
|
935
|
+
};
|
|
936
|
+
var taskSource = (value, label) => {
|
|
937
|
+
if (value === void 0) return void 0;
|
|
938
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return fail(`${label} must be an object.`, "INVALID_CONFIG");
|
|
939
|
+
const raw = value;
|
|
940
|
+
return { repository: nonEmptyString(raw["repository"], `${label}.repository`), path: relativePath(raw["path"], `${label}.path`), revision: nonEmptyString(raw["revision"], `${label}.revision`) };
|
|
941
|
+
};
|
|
942
|
+
var suiteSource = (value, label) => {
|
|
943
|
+
if (value === void 0) return void 0;
|
|
944
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return fail(`${label} must be an object.`, "INVALID_CONFIG");
|
|
945
|
+
const raw = value;
|
|
946
|
+
return { repository: nonEmptyString(raw["repository"], `${label}.repository`), revision: nonEmptyString(raw["revision"], `${label}.revision`), taskDefinition: relativePath(raw["taskDefinition"], `${label}.taskDefinition`) };
|
|
947
|
+
};
|
|
948
|
+
var taskScope = (value, label) => {
|
|
949
|
+
if (value === void 0) return void 0;
|
|
950
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return fail(`${label} must be an object.`, "INVALID_CONFIG");
|
|
951
|
+
const raw = value;
|
|
952
|
+
return { read: stringList(raw["read"], `${label}.read`), write: stringList(raw["write"], `${label}.write`) };
|
|
953
|
+
};
|
|
954
|
+
var taskSurfaces = (value, label) => {
|
|
955
|
+
if (value === void 0) return void 0;
|
|
956
|
+
if (!Array.isArray(value) || !value.length) return fail(`${label} must be a non-empty array.`, "INVALID_CONFIG");
|
|
957
|
+
const surfaces = value.map((item, index2) => nonEmptyString(item, `${label}[${index2}]`));
|
|
958
|
+
if (surfaces.some((surface2) => !SURFACE_NAMES.includes(surface2))) fail(`${label} contains an unknown surface.`, "INVALID_CONFIG");
|
|
959
|
+
if (new Set(surfaces).size !== surfaces.length) fail(`${label} must contain unique surfaces.`, "INVALID_CONFIG");
|
|
960
|
+
return surfaces;
|
|
961
|
+
};
|
|
962
|
+
var benchmarkPolicy = (value) => {
|
|
963
|
+
if (value === void 0) return void 0;
|
|
964
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return fail("benchmark.policy must be an object.", "INVALID_CONFIG");
|
|
965
|
+
const raw = value;
|
|
966
|
+
const minComparableTasks = nonNegativeInteger(raw["minComparableTasks"], "benchmark.policy.minComparableTasks");
|
|
967
|
+
const maxDurationRegressionRate = nonNegativeNumber(raw["maxDurationRegressionRate"], "benchmark.policy.maxDurationRegressionRate");
|
|
968
|
+
const minCompletedRunsPerTask = nonNegativeInteger(raw["minCompletedRunsPerTask"], "benchmark.policy.minCompletedRunsPerTask");
|
|
969
|
+
const minBaselineSamplesPerTask = nonNegativeInteger(raw["minBaselineSamplesPerTask"] ?? 1, "benchmark.policy.minBaselineSamplesPerTask");
|
|
970
|
+
if (minComparableTasks === void 0 || minComparableTasks < 1) return fail("benchmark.policy.minComparableTasks must be at least 1.", "INVALID_CONFIG");
|
|
971
|
+
if (maxDurationRegressionRate === void 0 || maxDurationRegressionRate > 1) return fail("benchmark.policy.maxDurationRegressionRate must be between 0 and 1.", "INVALID_CONFIG");
|
|
972
|
+
if (minCompletedRunsPerTask === void 0 || minCompletedRunsPerTask < 1) return fail("benchmark.policy.minCompletedRunsPerTask must be at least 1.", "INVALID_CONFIG");
|
|
973
|
+
if (minBaselineSamplesPerTask === void 0 || minBaselineSamplesPerTask < 1) return fail("benchmark.policy.minBaselineSamplesPerTask must be at least 1.", "INVALID_CONFIG");
|
|
974
|
+
if (typeof raw["requireZeroEscapedIncomplete"] !== "boolean") return fail("benchmark.policy.requireZeroEscapedIncomplete must be boolean.", "INVALID_CONFIG");
|
|
975
|
+
return { minComparableTasks, maxDurationRegressionRate, minCompletedRunsPerTask, minBaselineSamplesPerTask, requireZeroEscapedIncomplete: raw["requireZeroEscapedIncomplete"] };
|
|
976
|
+
};
|
|
977
|
+
var validateBenchmarkManifest = (value) => {
|
|
978
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) fail("benchmark manifest must be an object.", "INVALID_CONFIG");
|
|
979
|
+
const raw = value;
|
|
980
|
+
if (raw["type"] !== "agentskit-harness-benchmark-manifest" || raw["schemaVersion"] !== BENCHMARK_SCHEMA_VERSION) fail("benchmark manifest type or schemaVersion is invalid.", "INVALID_CONFIG");
|
|
981
|
+
const rawTasks = Array.isArray(raw["tasks"]) && raw["tasks"].length ? raw["tasks"] : fail("benchmark manifest tasks must be non-empty.", "INVALID_CONFIG");
|
|
982
|
+
const tasks = rawTasks.map((item, index2) => {
|
|
983
|
+
if (typeof item !== "object" || item === null || Array.isArray(item)) fail(`benchmark.tasks[${index2}] must be an object.`, "INVALID_CONFIG");
|
|
984
|
+
const task = item;
|
|
985
|
+
const kind = task["kind"] === void 0 ? void 0 : nonEmptyString(task["kind"], `benchmark.tasks[${index2}].kind`);
|
|
986
|
+
const prompt = taskFile(task["prompt"], `benchmark.tasks[${index2}].prompt`);
|
|
987
|
+
const source = taskSource(task["source"], `benchmark.tasks[${index2}].source`);
|
|
988
|
+
const scope = taskScope(task["scope"], `benchmark.tasks[${index2}].scope`);
|
|
989
|
+
const surfaces = taskSurfaces(task["surfaces"], `benchmark.tasks[${index2}].surfaces`);
|
|
990
|
+
return { id: nonEmptyString(task["id"], `benchmark.tasks[${index2}].id`), title: nonEmptyString(task["title"], `benchmark.tasks[${index2}].title`), acceptanceCriteria: stringList(task["acceptanceCriteria"], `benchmark.tasks[${index2}].acceptanceCriteria`), ...surfaces === void 0 ? {} : { surfaces }, ...kind === void 0 ? {} : { kind }, ...prompt === void 0 ? {} : { prompt }, ...source === void 0 ? {} : { source }, ...scope === void 0 ? {} : { scope } };
|
|
991
|
+
});
|
|
992
|
+
if (new Set(tasks.map((task) => task.id)).size !== tasks.length) fail("benchmark task ids must be unique.", "INVALID_CONFIG");
|
|
993
|
+
const taskIds = new Set(tasks.map((task) => task.id));
|
|
994
|
+
const rawObservations = raw["observations"] === void 0 ? [] : Array.isArray(raw["observations"]) ? raw["observations"] : fail("benchmark.observations must be an array.", "INVALID_CONFIG");
|
|
995
|
+
const observations = rawObservations.map((item, index2) => {
|
|
996
|
+
if (typeof item !== "object" || item === null || Array.isArray(item)) fail(`benchmark.observations[${index2}] must be an object.`, "INVALID_CONFIG");
|
|
997
|
+
const observation = item;
|
|
998
|
+
const status = observation["status"];
|
|
999
|
+
if (!["passed", "failed", "blocked", "not-run"].includes(String(status))) fail(`benchmark.observations[${index2}].status is invalid.`, "INVALID_CONFIG");
|
|
1000
|
+
const taskId = nonEmptyString(observation["taskId"], `benchmark.observations[${index2}].taskId`);
|
|
1001
|
+
if (!taskIds.has(taskId)) fail(`benchmark observation references unknown task: ${taskId}.`, "INVALID_CONFIG");
|
|
1002
|
+
const task = tasks.find((candidate) => candidate.id === taskId) ?? fail(`benchmark observation references unknown task: ${taskId}.`, "INVALID_CONFIG");
|
|
1003
|
+
const attempts = nonNegativeInteger(observation["attempts"], `benchmark.observations[${index2}].attempts`);
|
|
1004
|
+
const durationMs = nonNegativeNumber(observation["durationMs"], `benchmark.observations[${index2}].durationMs`);
|
|
1005
|
+
const durationSamplesMs = observation["durationSamplesMs"] === void 0 ? void 0 : Array.isArray(observation["durationSamplesMs"]) ? observation["durationSamplesMs"].map((sample, sampleIndex) => nonNegativeNumber(sample, `benchmark.observations[${index2}].durationSamplesMs[${sampleIndex}]`) ?? fail(`benchmark.observations[${index2}].durationSamplesMs must contain numbers.`, "INVALID_CONFIG")) : fail(`benchmark.observations[${index2}].durationSamplesMs must be an array.`, "INVALID_CONFIG");
|
|
1006
|
+
if (durationSamplesMs && !durationSamplesMs.length) fail(`benchmark.observations[${index2}].durationSamplesMs must not be empty.`, "INVALID_CONFIG");
|
|
1007
|
+
const artifactAcceptanceRate2 = rate(observation["artifactAcceptanceRate"], `benchmark.observations[${index2}].artifactAcceptanceRate`);
|
|
1008
|
+
const protocolCompletionRate = rate(observation["protocolCompletionRate"], `benchmark.observations[${index2}].protocolCompletionRate`);
|
|
1009
|
+
const reviewMinutes2 = nonNegativeNumber(observation["reviewMinutes"], `benchmark.observations[${index2}].reviewMinutes`);
|
|
1010
|
+
const escapedIncomplete = nonNegativeInteger(observation["escapedIncomplete"], `benchmark.observations[${index2}].escapedIncomplete`);
|
|
1011
|
+
const evidenceDigest = sha2562(observation["evidenceDigest"], `benchmark.observations[${index2}].evidenceDigest`);
|
|
1012
|
+
const rawEvidence = observation["evidence"] === void 0 ? void 0 : Array.isArray(observation["evidence"]) ? observation["evidence"] : fail(`benchmark.observations[${index2}].evidence must be an array.`, "INVALID_CONFIG");
|
|
1013
|
+
const evidence = rawEvidence?.map((item2, evidenceIndex) => {
|
|
1014
|
+
if (typeof item2 !== "object" || item2 === null || Array.isArray(item2)) fail(`benchmark.observations[${index2}].evidence[${evidenceIndex}] must be an object.`, "INVALID_CONFIG");
|
|
1015
|
+
const entry = item2;
|
|
1016
|
+
const criterion = nonEmptyString(entry["criterion"], `benchmark.observations[${index2}].evidence[${evidenceIndex}].criterion`);
|
|
1017
|
+
if (!task.acceptanceCriteria.includes(criterion)) fail(`benchmark evidence references unknown criterion: ${criterion}.`, "INVALID_CONFIG");
|
|
1018
|
+
const evidenceStatus = entry["status"];
|
|
1019
|
+
if (!["passed", "failed", "blocked", "not-run"].includes(String(evidenceStatus))) fail(`benchmark.observations[${index2}].evidence[${evidenceIndex}].status is invalid.`, "INVALID_CONFIG");
|
|
1020
|
+
return { criterion, status: evidenceStatus, source: nonEmptyString(entry["source"], `benchmark.observations[${index2}].evidence[${evidenceIndex}].source`) };
|
|
1021
|
+
});
|
|
1022
|
+
if (evidence && new Set(evidence.map((entry) => entry.criterion)).size !== evidence.length) fail(`benchmark.observations[${index2}].evidence criteria must be unique.`, "INVALID_CONFIG");
|
|
1023
|
+
return { taskId, mode: "baseline", status, source: nonEmptyString(observation["source"], `benchmark.observations[${index2}].source`), recordedAt: timestamp(observation["recordedAt"], `benchmark.observations[${index2}].recordedAt`), ...attempts === void 0 ? {} : { attempts }, ...durationMs === void 0 ? {} : { durationMs }, ...durationSamplesMs === void 0 ? {} : { durationSamplesMs }, ...artifactAcceptanceRate2 === void 0 ? {} : { artifactAcceptanceRate: artifactAcceptanceRate2 }, ...protocolCompletionRate === void 0 ? {} : { protocolCompletionRate }, ...reviewMinutes2 === void 0 ? {} : { reviewMinutes: reviewMinutes2 }, ...escapedIncomplete === void 0 ? {} : { escapedIncomplete }, ...evidence === void 0 ? {} : { evidence }, ...evidenceDigest === void 0 ? {} : { evidenceDigest } };
|
|
1024
|
+
});
|
|
1025
|
+
if (new Set(observations.map((observation) => observation.taskId)).size !== observations.length) fail("benchmark allows at most one baseline observation per task.", "INVALID_CONFIG");
|
|
1026
|
+
const provenance = suiteSource(raw["provenance"], "benchmark.provenance");
|
|
1027
|
+
const policy = benchmarkPolicy(raw["policy"]);
|
|
1028
|
+
return { type: "agentskit-harness-benchmark-manifest", schemaVersion: BENCHMARK_SCHEMA_VERSION, suiteId: nonEmptyString(raw["suiteId"], "benchmark.suiteId"), name: nonEmptyString(raw["name"], "benchmark.name"), ...provenance === void 0 ? {} : { provenance }, tasks, observations, ...policy === void 0 ? {} : { policy } };
|
|
1029
|
+
};
|
|
1030
|
+
var loadBenchmarkManifest = (path) => {
|
|
1031
|
+
try {
|
|
1032
|
+
return validateBenchmarkManifest(JSON.parse(readFileSync(path, "utf8")));
|
|
1033
|
+
} catch (error) {
|
|
1034
|
+
if (error instanceof SyntaxError) return fail(`Invalid benchmark manifest JSON: ${error.message}`, "INVALID_CONFIG");
|
|
1035
|
+
throw error;
|
|
1036
|
+
}
|
|
1037
|
+
};
|
|
1038
|
+
var recordBenchmarkObservation = (path, input) => {
|
|
1039
|
+
const originalContent = readFileSync(path, "utf8");
|
|
1040
|
+
const manifest = loadBenchmarkManifest(path);
|
|
1041
|
+
const taskId = nonEmptyString(input.taskId, "benchmark observation.taskId");
|
|
1042
|
+
if (!manifest.tasks.some((task) => task.id === taskId)) fail(`benchmark observation references unknown task: ${taskId}.`, "INVALID_INPUT");
|
|
1043
|
+
if (manifest.observations.some((observation2) => observation2.taskId === taskId)) fail(`benchmark already has an observation for task: ${taskId}.`, "INVALID_INPUT");
|
|
1044
|
+
const observation = validateBenchmarkManifest({
|
|
1045
|
+
...manifest,
|
|
1046
|
+
observations: [...manifest.observations, {
|
|
1047
|
+
taskId,
|
|
1048
|
+
mode: "baseline",
|
|
1049
|
+
status: input.status,
|
|
1050
|
+
source: input.source,
|
|
1051
|
+
recordedAt: input.recordedAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
1052
|
+
...input.attempts === void 0 ? {} : { attempts: input.attempts },
|
|
1053
|
+
...input.durationMs === void 0 ? {} : { durationMs: input.durationMs },
|
|
1054
|
+
...input.durationSamplesMs === void 0 ? {} : { durationSamplesMs: input.durationSamplesMs },
|
|
1055
|
+
...input.artifactAcceptanceRate === void 0 ? {} : { artifactAcceptanceRate: input.artifactAcceptanceRate },
|
|
1056
|
+
...input.protocolCompletionRate === void 0 ? {} : { protocolCompletionRate: input.protocolCompletionRate },
|
|
1057
|
+
...input.reviewMinutes === void 0 ? {} : { reviewMinutes: input.reviewMinutes },
|
|
1058
|
+
...input.escapedIncomplete === void 0 ? {} : { escapedIncomplete: input.escapedIncomplete },
|
|
1059
|
+
...input.evidence === void 0 ? {} : { evidence: input.evidence },
|
|
1060
|
+
...input.evidenceDigest === void 0 ? {} : { evidenceDigest: input.evidenceDigest }
|
|
1061
|
+
}]
|
|
1062
|
+
});
|
|
1063
|
+
const temporaryRoot = mkdtempSync(join(tmpdir(), "agentskit-harness-baseline-"));
|
|
1064
|
+
const temporaryPath = join(temporaryRoot, "manifest.json");
|
|
1065
|
+
try {
|
|
1066
|
+
writeFileSync(temporaryPath, `${JSON.stringify(observation, null, 2)}
|
|
1067
|
+
`, "utf8");
|
|
1068
|
+
if (readFileSync(path, "utf8") !== originalContent) fail("benchmark manifest changed while recording an observation.", "STALE");
|
|
1069
|
+
renameSync(temporaryPath, path);
|
|
1070
|
+
} finally {
|
|
1071
|
+
rmSync(temporaryRoot, { recursive: true, force: true });
|
|
1072
|
+
}
|
|
1073
|
+
return observation;
|
|
1074
|
+
};
|
|
1075
|
+
var comparisons = (runs, manifest, policy) => manifest.tasks.map((task) => {
|
|
1076
|
+
const taskRuns = runs.filter((run) => run.benchmark?.suiteId === manifest.suiteId && run.benchmark.taskId === task.id);
|
|
1077
|
+
const latest = taskRuns.at(-1);
|
|
1078
|
+
const baseline = manifest.observations.find((observation) => observation.taskId === task.id);
|
|
1079
|
+
const coveredCriteria = new Set((baseline?.evidence ?? []).map((entry) => entry.criterion));
|
|
1080
|
+
const baselineEvidenceCoverageRate = baseline ? percentage(coveredCriteria.size, task.acceptanceCriteria.length) : null;
|
|
1081
|
+
const baselineEvidenceComplete = baselineEvidenceCoverageRate === 1;
|
|
1082
|
+
const baselineEvidencePassed = baselineEvidenceComplete && (baseline?.evidence?.every((entry) => entry.status === "passed") ?? false);
|
|
1083
|
+
const baselineDeliveryComplete = baseline?.status === "passed" && baselineEvidencePassed;
|
|
1084
|
+
const baselineDurationSamples = baseline?.durationSamplesMs ?? (baseline?.durationMs === void 0 ? [] : [baseline.durationMs]);
|
|
1085
|
+
const baselineMedianDurationMs = median(baselineDurationSamples);
|
|
1086
|
+
const baselineSamplesSufficient = baselineDurationSamples.length >= policy.minBaselineSamplesPerTask;
|
|
1087
|
+
const comparable = baseline !== void 0 && baseline.status !== "not-run" && baselineEvidenceComplete && baselineDeliveryComplete && baselineSamplesSufficient && latest?.state === "COMPLETE";
|
|
1088
|
+
const comparability = comparable ? "comparable" : baseline === void 0 ? "missing-baseline" : baseline.status === "not-run" ? "baseline-not-run" : !baselineEvidenceComplete ? "baseline-evidence-missing" : latest === void 0 ? "harness-not-run" : latest.state !== "COMPLETE" ? "harness-not-complete" : !baselineDeliveryComplete ? "baseline-incomplete" : "baseline-samples-insufficient";
|
|
1089
|
+
const completedTaskRuns = taskRuns.filter((run) => run.state === "COMPLETE");
|
|
1090
|
+
const durationSamplesMs = completedTaskRuns.flatMap((run) => run.durationMs === void 0 ? [] : [run.durationMs]);
|
|
1091
|
+
const medianDurationMs = median(durationSamplesMs);
|
|
1092
|
+
const retryCount = count(taskRuns, (run) => run.supersedes !== void 0);
|
|
1093
|
+
const attempts = retryCount + (taskRuns.length ? 1 : 0);
|
|
1094
|
+
const durationRate = comparable ? improvementRate(baselineMedianDurationMs ?? void 0, medianDurationMs ?? void 0) : null;
|
|
1095
|
+
const attemptsRate = comparable ? improvementRate(baseline?.attempts, attempts) : null;
|
|
1096
|
+
const reviewRate = comparable ? improvementRate(baseline?.reviewMinutes, latest?.humanReviewMinutes) : null;
|
|
1097
|
+
const acceptanceSamples = taskRuns.flatMap((run) => run.artifactAcceptanceRate === void 0 ? [] : [run.artifactAcceptanceRate]);
|
|
1098
|
+
const harnessArtifactAcceptanceRate = acceptanceSamples.length ? Number((acceptanceSamples.reduce((total, rate2) => total + rate2, 0) / acceptanceSamples.length).toFixed(4)) : null;
|
|
1099
|
+
const artifactAcceptanceImprovementRate = increaseRate(baseline?.artifactAcceptanceRate, harnessArtifactAcceptanceRate ?? void 0);
|
|
1100
|
+
const artifactAcceptanceDelta = increaseDelta(baseline?.artifactAcceptanceRate, harnessArtifactAcceptanceRate);
|
|
1101
|
+
const completedRuns = completedTaskRuns.length;
|
|
1102
|
+
const harnessProtocolCompletionRate = taskRuns.length ? percentage(completedRuns, taskRuns.length) : null;
|
|
1103
|
+
const protocolCompletionImprovementRate = increaseRate(baseline?.protocolCompletionRate, harnessProtocolCompletionRate ?? void 0);
|
|
1104
|
+
const protocolCompletionDelta = increaseDelta(baseline?.protocolCompletionRate, harnessProtocolCompletionRate);
|
|
1105
|
+
const escapedIncompleteRate = improvementRate(baseline?.escapedIncomplete, latest?.escapedIncomplete);
|
|
1106
|
+
return { taskId: task.id, title: task.title, comparability, comparable, baselineDeliveryComplete, baselineEvidenceCoverageRate, baselineSampleCount: baselineDurationSamples.length, baselineArtifactAcceptanceRate: baseline?.artifactAcceptanceRate ?? null, baselineProtocolCompletionRate: baseline?.protocolCompletionRate ?? null, ...baselineMedianDurationMs === null ? {} : { baselineMedianDurationMs }, improvement: { durationRate, duration: improvementDirection(durationRate), attemptsRate, attempts: improvementDirection(attemptsRate), reviewRate, review: improvementDirection(reviewRate), artifactAcceptanceRate: artifactAcceptanceImprovementRate, artifactAcceptance: increaseDirection(artifactAcceptanceImprovementRate, artifactAcceptanceDelta), artifactAcceptanceDelta, protocolCompletionRate: protocolCompletionImprovementRate, protocolCompletion: increaseDirection(protocolCompletionImprovementRate, protocolCompletionDelta), protocolCompletionDelta, escapedIncompleteRate, escapedIncomplete: improvementDirection(escapedIncompleteRate) }, ...baseline ? { baseline } : {}, harness: { attempts, retryCount, completedRuns, durationSamplesMs, ...medianDurationMs === null ? {} : { medianDurationMs }, ...harnessArtifactAcceptanceRate === null ? {} : { artifactAcceptanceRate: harnessArtifactAcceptanceRate }, artifactAcceptanceSampleCount: acceptanceSamples.length, protocolCompletionRate: harnessProtocolCompletionRate, protocolCompletionSampleCount: taskRuns.length, latestState: latest?.state ?? "NOT_RUN", ...latest ? { latestRunId: latest.runId } : {}, ...latest?.durationMs === void 0 ? {} : { latestDurationMs: latest.durationMs }, checkPassRate: latest ? percentage(latest.checks.passed, latest.checks.total) : null, outcomePassRate: latest ? percentage(latest.outcomes.passed, latest.outcomes.total) : null, evidenceCoverageRate: latest ? percentage(latest.evidence.attached, latest.evidence.total) : null, ...latest?.escapedIncomplete === void 0 ? {} : { escapedIncomplete: latest.escapedIncomplete }, ...latest?.humanReviewMinutes === void 0 ? {} : { humanReviewMinutes: latest.humanReviewMinutes }, humanApproved: latest?.humanApproved ?? false }, confidence: confidence(comparable, completedRuns, policy), ...comparable && baselineMedianDurationMs !== null && medianDurationMs !== null ? { durationDeltaMs: medianDurationMs - baselineMedianDurationMs } : {}, ...comparable && baseline?.attempts !== void 0 ? { attemptDelta: attempts - baseline.attempts } : {}, ...comparable && baseline?.reviewMinutes !== void 0 && latest?.humanReviewMinutes !== void 0 ? { reviewDeltaMinutes: latest.humanReviewMinutes - baseline.reviewMinutes } : {}, ...baseline?.escapedIncomplete !== void 0 && latest?.escapedIncomplete !== void 0 ? { escapedIncompleteDelta: latest.escapedIncomplete - baseline.escapedIncomplete } : {} };
|
|
1107
|
+
});
|
|
1108
|
+
var benchmarkRuns = (stateDir, manifest) => {
|
|
1109
|
+
const runs = readRuns(stateDir).map(projectRun).sort((left, right) => left.runId.localeCompare(right.runId));
|
|
1110
|
+
const policy = manifest?.policy ?? DEFAULT_POLICY;
|
|
1111
|
+
const reportComparisons = manifest ? comparisons(runs, manifest, policy) : [];
|
|
1112
|
+
const comparable = reportComparisons.filter((comparison) => comparison.comparable);
|
|
1113
|
+
const durationRegressionTaskIds = comparable.filter((comparison) => (comparison.improvement.durationRate ?? 0) < -policy.maxDurationRegressionRate).map((comparison) => comparison.taskId);
|
|
1114
|
+
const escapedIncompleteTaskIds = comparable.filter((comparison) => comparison.harness.escapedIncomplete !== 0).map((comparison) => comparison.taskId);
|
|
1115
|
+
const reasons = [];
|
|
1116
|
+
if (comparable.length < policy.minComparableTasks) reasons.push(`requires at least ${policy.minComparableTasks} comparable tasks`);
|
|
1117
|
+
const incompleteBaselineTaskIds = reportComparisons.filter((comparison) => comparison.comparability === "baseline-incomplete").map((comparison) => comparison.taskId);
|
|
1118
|
+
if (incompleteBaselineTaskIds.length) reasons.push(`baseline delivery incomplete: ${incompleteBaselineTaskIds.join(", ")}`);
|
|
1119
|
+
const baselineSampleGaps = reportComparisons.filter((comparison) => comparison.baselineSampleCount < policy.minBaselineSamplesPerTask).map((comparison) => `${comparison.taskId} (${comparison.baselineSampleCount}/${policy.minBaselineSamplesPerTask})`);
|
|
1120
|
+
if (baselineSampleGaps.length) reasons.push(`requires ${policy.minBaselineSamplesPerTask} baseline samples per task: ${baselineSampleGaps.join(", ")}`);
|
|
1121
|
+
if (durationRegressionTaskIds.length) reasons.push(`duration regression exceeds ${policy.maxDurationRegressionRate * 100}%: ${durationRegressionTaskIds.join(", ")}`);
|
|
1122
|
+
if (policy.requireZeroEscapedIncomplete && escapedIncompleteTaskIds.length) reasons.push(`escaped incomplete delivery: ${escapedIncompleteTaskIds.join(", ")}`);
|
|
1123
|
+
const confidenceLevel = comparable.length < policy.minComparableTasks ? "insufficient" : comparable.every((comparison) => comparison.confidence === "reliable") ? "reliable" : "directional";
|
|
1124
|
+
const qualityGate = { status: comparable.length < policy.minComparableTasks ? "insufficient-data" : reasons.length ? "failed" : "passed", confidence: confidenceLevel, comparableTaskCount: comparable.length, policy, durationRegressionTaskIds, escapedIncompleteTaskIds, reasons };
|
|
1125
|
+
return { type: "agentskit-harness-benchmark", schemaVersion: BENCHMARK_SCHEMA_VERSION, stateDir, generatedAt: (/* @__PURE__ */ new Date()).toISOString(), runs, summary: summarize(runs), comparisons: reportComparisons, qualityGate, ...manifest ? { manifest: { suiteId: manifest.suiteId, taskCount: manifest.tasks.length, baselineCount: manifest.observations.length, comparableTaskCount: comparable.length } } : {} };
|
|
1126
|
+
};
|
|
1127
|
+
var EVIDENCE_BUNDLE_SCHEMA_VERSION = 1;
|
|
1128
|
+
var body = (bundle) => {
|
|
1129
|
+
const { payloadHash: _payloadHash, signature: _signature, ...unsigned } = bundle;
|
|
1130
|
+
return unsigned;
|
|
1131
|
+
};
|
|
1132
|
+
var parseBundle = (path) => {
|
|
1133
|
+
try {
|
|
1134
|
+
return JSON.parse(fileContents(path));
|
|
1135
|
+
} catch (error) {
|
|
1136
|
+
return fail(`Invalid evidence bundle JSON: ${error instanceof Error ? error.message : String(error)}`, "INVALID_INPUT");
|
|
1137
|
+
}
|
|
1138
|
+
};
|
|
1139
|
+
var validDigest = (value) => typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
|
|
1140
|
+
var validKeyId = (value) => typeof value === "string" && /^[A-Za-z0-9._:-]{1,128}$/.test(value);
|
|
1141
|
+
var requireRun2 = (run) => run ?? fail("No verification run exists.", "NO_RUN");
|
|
1142
|
+
var bundleFile = (stateDir, path) => {
|
|
1143
|
+
const absolute = resolve(stateDir, path);
|
|
1144
|
+
if (!pathInside(stateDir, absolute)) fail(`Evidence path escapes state directory: ${path}`, "HARNESS_ERROR");
|
|
1145
|
+
const content = readFileSync(absolute);
|
|
1146
|
+
return { path: relative(stateDir, absolute).split(sep).join("/"), sha256: sha256(content), contentBase64: content.toString("base64") };
|
|
1147
|
+
};
|
|
1148
|
+
var exportEvidenceBundle = async ({ configPath, runId, outputPath, privateKeyPath, keyId }) => {
|
|
1149
|
+
if (!validKeyId(keyId)) fail("keyId must contain only letters, numbers, dot, underscore, colon, or hyphen.", "INVALID_INPUT");
|
|
1150
|
+
const loaded = loadConfig(configPath);
|
|
1151
|
+
const run = requireRun2(runId ? readJson(join(loaded.stateDir, "runs", runId, "run.json")) : loadLatestRun(loaded.stateDir));
|
|
1152
|
+
const reconciliation = await reconcileRun({ configPath, runId: run.runId });
|
|
1153
|
+
const digest2 = run.verificationDigest ?? fail("Only a reconciled COMPLETE run can be exported.", "INVALID_STATE");
|
|
1154
|
+
if (reconciliation.state !== "COMPLETE") fail("Only a reconciled COMPLETE run can be exported.", "INVALID_STATE");
|
|
1155
|
+
const eventLog = new FileEventStore(loaded.stateDir);
|
|
1156
|
+
eventLog.read(run.runId);
|
|
1157
|
+
const eventVerification = eventLog.verify(run.runId);
|
|
1158
|
+
const paths = /* @__PURE__ */ new Set(["runs/" + run.runId + "/run.json", "runs/" + run.runId + "/events.ndjson"]);
|
|
1159
|
+
for (const reference of run.evidenceReferences) {
|
|
1160
|
+
paths.add(reference.stdout);
|
|
1161
|
+
paths.add(reference.stderr);
|
|
1162
|
+
}
|
|
1163
|
+
const files = [...paths].map((path) => {
|
|
1164
|
+
if (!existsSync(resolve(loaded.stateDir, path))) fail(`Evidence file is missing: ${path}`, "HARNESS_ERROR");
|
|
1165
|
+
return bundleFile(loaded.stateDir, path);
|
|
1166
|
+
});
|
|
1167
|
+
const privateKey = createPrivateKey(readFileSync(privateKeyPath));
|
|
1168
|
+
const unsigned = { type: "agentskit-harness-evidence-bundle", schemaVersion: EVIDENCE_BUNDLE_SCHEMA_VERSION, runId: run.runId, signerKeyId: keyId, sourceRevision: run.sourceRevision, configHash: run.configHash, contractHash: run.contractHash, verificationDigest: digest2, eventLog: eventVerification, files };
|
|
1169
|
+
const payloadHash = sha256(JSON.stringify(unsigned));
|
|
1170
|
+
const publicKeyPem = createPublicKey(privateKey).export({ type: "spki", format: "pem" }).toString();
|
|
1171
|
+
const bundle = { ...unsigned, payloadHash, signature: { algorithm: "ed25519", keyId, publicKeyPem, signatureBase64: sign(null, Buffer.from(payloadHash), privateKey).toString("base64") } };
|
|
1172
|
+
writeFileSync(outputPath, `${JSON.stringify(bundle, null, 2)}
|
|
1173
|
+
`, "utf8");
|
|
1174
|
+
return bundle;
|
|
1175
|
+
};
|
|
1176
|
+
var verifyEvidenceBundle = (path, { trustedKeys = [] } = {}) => {
|
|
1177
|
+
const bundle = parseBundle(path);
|
|
1178
|
+
if (bundle.type !== "agentskit-harness-evidence-bundle" || bundle.schemaVersion !== EVIDENCE_BUNDLE_SCHEMA_VERSION || !bundle.runId || !validKeyId(bundle.signerKeyId) || !validDigest(bundle.payloadHash) || bundle.signature?.algorithm !== "ed25519" || bundle.signature.keyId !== bundle.signerKeyId || typeof bundle.signature.publicKeyPem !== "string" || typeof bundle.signature.signatureBase64 !== "string" || !Array.isArray(bundle.files)) fail("Evidence bundle metadata is invalid.", "HARNESS_ERROR");
|
|
1179
|
+
if (trustedKeys.length) {
|
|
1180
|
+
const trusted = trustedKeys.find((key) => key.keyId === bundle.signerKeyId);
|
|
1181
|
+
if (!trusted) return fail(`Evidence bundle key is not trusted: ${bundle.signerKeyId}`, "HARNESS_ERROR");
|
|
1182
|
+
if (trusted.status === "revoked") fail(`Evidence bundle key is revoked: ${bundle.signerKeyId}`, "HARNESS_ERROR");
|
|
1183
|
+
if (trusted.publicKeyPem !== bundle.signature.publicKeyPem) fail(`Evidence bundle key does not match trust store: ${bundle.signerKeyId}`, "HARNESS_ERROR");
|
|
1184
|
+
}
|
|
1185
|
+
const paths = /* @__PURE__ */ new Set();
|
|
1186
|
+
for (const file of bundle.files) {
|
|
1187
|
+
if (!file || typeof file.path !== "string" || paths.has(file.path) || !validDigest(file.sha256) || typeof file.contentBase64 !== "string") fail("Evidence bundle file metadata is invalid.", "HARNESS_ERROR");
|
|
1188
|
+
paths.add(file.path);
|
|
1189
|
+
const content = Buffer.from(file.contentBase64, "base64");
|
|
1190
|
+
if (sha256(content) !== file.sha256) fail(`Evidence bundle file hash mismatch: ${file.path}`, "HARNESS_ERROR");
|
|
1191
|
+
}
|
|
1192
|
+
if (!paths.has(`runs/${bundle.runId}/run.json`) || !paths.has(`runs/${bundle.runId}/events.ndjson`)) fail("Evidence bundle is missing the run projection or event log.", "HARNESS_ERROR");
|
|
1193
|
+
if (sha256(JSON.stringify(body(bundle))) !== bundle.payloadHash) fail("Evidence bundle payload hash mismatch.", "HARNESS_ERROR");
|
|
1194
|
+
let valid = false;
|
|
1195
|
+
try {
|
|
1196
|
+
valid = verify(null, Buffer.from(bundle.payloadHash), createPublicKey(bundle.signature.publicKeyPem), Buffer.from(bundle.signature.signatureBase64, "base64"));
|
|
1197
|
+
} catch {
|
|
1198
|
+
valid = false;
|
|
1199
|
+
}
|
|
1200
|
+
if (!valid) fail("Evidence bundle signature is invalid.", "HARNESS_ERROR");
|
|
1201
|
+
return { status: "verified", runId: bundle.runId, payloadHash: bundle.payloadHash, fileCount: bundle.files.length, signed: true };
|
|
1202
|
+
};
|
|
1203
|
+
var readEvidenceTrustStore = (path) => {
|
|
1204
|
+
const value = readJson(path);
|
|
1205
|
+
if (value.schemaVersion !== 1 || !Array.isArray(value.keys)) fail("Evidence trust store must contain schemaVersion 1 and a keys array.", "INVALID_INPUT");
|
|
1206
|
+
return value.keys.map((key, index2) => {
|
|
1207
|
+
if (typeof key !== "object" || key === null || Array.isArray(key) || !validKeyId(key["keyId"]) || typeof key["publicKeyPem"] !== "string" || key["status"] !== "active" && key["status"] !== "revoked") fail(`Invalid evidence trust store key at index ${index2}.`, "INVALID_INPUT");
|
|
1208
|
+
return key;
|
|
1209
|
+
});
|
|
1210
|
+
};
|
|
1211
|
+
|
|
1212
|
+
// src/cli.ts
|
|
1213
|
+
var packageJson = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8"));
|
|
1214
|
+
var program = new Command();
|
|
1215
|
+
program.name("ak-harness").description("Portable, evidence-backed development harness for coding agents.").version(packageJson.version).option("-c, --config <path>", "verification contract path", ".codex/verification.json").option("--json", "emit machine-readable output");
|
|
1216
|
+
var options = () => program.opts();
|
|
1217
|
+
var print = (value) => {
|
|
1218
|
+
if (options().json) console.log(JSON.stringify(value));
|
|
1219
|
+
else console.log(typeof value === "string" ? value : JSON.stringify(value, null, 2));
|
|
1220
|
+
};
|
|
1221
|
+
var readBenchmarkEvidence = (path) => {
|
|
1222
|
+
try {
|
|
1223
|
+
const content = readFileSync(path, "utf8");
|
|
1224
|
+
const raw = JSON.parse(content);
|
|
1225
|
+
const evidence = Array.isArray(raw) ? raw : typeof raw === "object" && raw !== null ? raw.evidence : void 0;
|
|
1226
|
+
if (Array.isArray(evidence)) return { evidence, digest: createHash("sha256").update(content).digest("hex") };
|
|
1227
|
+
} catch (error) {
|
|
1228
|
+
fail(`Invalid benchmark evidence JSON: ${error instanceof Error ? error.message : String(error)}`, "INVALID_INPUT");
|
|
1229
|
+
}
|
|
1230
|
+
return fail("benchmark evidence file must contain an array or an object with an evidence array.", "INVALID_INPUT");
|
|
1231
|
+
};
|
|
1232
|
+
var decisionArgs = (first, second) => {
|
|
1233
|
+
const decisions = /* @__PURE__ */ new Set(["approved", "approve", "yes", "ok", "rejected", "reject", "no"]);
|
|
1234
|
+
return decisions.has(first) ? { decision: first, ...second ? { runId: second } : {} } : { decision: second ?? "", runId: first };
|
|
1235
|
+
};
|
|
1236
|
+
program.command("doctor").description("Validate the contract without starting a run.").action(() => print({ status: "passed", criteria: ["package"], config: loadConfig(options().config).config }));
|
|
1237
|
+
program.command("plan <decision>").description("Prepare or approve the frozen task contract and create a planned run.").option("--by <actor>", "planner: human approval or ci preparation", "human").option("--allow-dirty", "allow a human-authorized dirty worktree").option("--context-file <path>", "attach a context snapshot JSON file").action(async (decision, command) => print(await planRun({ configPath: options().config, decision, actor: command.by, allowDirty: command.allowDirty ?? false, contextSnapshots: command.contextFile ? readContextSnapshots(command.contextFile) : [] })));
|
|
1238
|
+
var context = program.command("context").description("Resolve portable, provenance-bearing context snapshots.");
|
|
1239
|
+
context.command("resolve <query>").description("Resolve a Doc Bridge snapshot from the local index.").option("--provider <provider>", "context provider", "doc-bridge").option("--scope <scope...>", "optional search scopes").option("--index <path>", "Doc Bridge index path", ".doc-bridge/index.json").action(async (query, command) => {
|
|
1240
|
+
if (command.provider !== "doc-bridge") fail(`Unsupported context provider: ${command.provider}`, "INVALID_INPUT");
|
|
1241
|
+
const loaded = loadConfig(options().config);
|
|
1242
|
+
print(await createDocBridgeContextProvider({ root: loaded.root, indexPath: command.index }).resolve({ query, ...command.scope?.length ? { scope: command.scope } : {} }));
|
|
1243
|
+
});
|
|
1244
|
+
program.command("start").description("Move a planned run into implementation.").action(() => print(startRun(loadConfig(options().config))));
|
|
1245
|
+
program.command("verify").description("Execute every configured check and record evidence.").action(async () => print(await verifyRun({ configPath: options().config })));
|
|
1246
|
+
program.command("run").description("Alias for verify, compatible with the common protocol.").action(async () => print(await verifyRun({ configPath: options().config })));
|
|
1247
|
+
program.command("approve <run-id-or-decision> [decision-or-run-id]").description("Record human approval or rejection. Accepts <run-id> <decision> or <decision> <run-id>.").option("--by <actor>", "approval actor", "human").action(async (first, second, command) => {
|
|
1248
|
+
const args = decisionArgs(first, second);
|
|
1249
|
+
print(await approveRun({ configPath: options().config, ...args, actor: command.by }));
|
|
1250
|
+
});
|
|
1251
|
+
program.command("authorize <run-id-or-decision> [decision-or-run-id]").description("Authorize or reject declared external tracking. Accepts <run-id> <decision> or <decision> <run-id>.").option("--by <actor>", "approval actor", "human").action(async (first, second, command) => {
|
|
1252
|
+
const args = decisionArgs(first, second);
|
|
1253
|
+
print(await authorizeRun({ configPath: options().config, ...args, actor: command.by }));
|
|
1254
|
+
});
|
|
1255
|
+
program.command("retry").description("Create a new implementation attempt after a blocked or stale run.").action(async () => print(await retryRun({ configPath: options().config })));
|
|
1256
|
+
program.command("cancel [run-id]").description("Cancel an active run.").option("--by <actor>", "cancellation actor", "human").option("--reason <reason>", "cancellation reason", "Run cancelled by a human.").action(async (runId, command) => print(await cancelRun({ configPath: options().config, runId, reason: command.reason, actor: command.by })));
|
|
1257
|
+
program.command("status").description("Show the latest run after reconciling its audit evidence.").action(async () => {
|
|
1258
|
+
const loaded = loadConfig(options().config);
|
|
1259
|
+
print(loadLatestRun(loaded.stateDir) ? await reconcileRun({ configPath: options().config }) : { state: "CLARIFYING", message: "No run exists." });
|
|
1260
|
+
});
|
|
1261
|
+
program.command("audit [run-id]").description("Reconcile a run projection with its verified lifecycle decisions.").action(async (runId) => print(await reconcileRun({ configPath: options().config, runId })));
|
|
1262
|
+
var events = program.command("events").description("Inspect the lifecycle audit log.");
|
|
1263
|
+
events.command("verify [run-id]").description("Verify the latest or selected event log hash chain.").action((runId) => {
|
|
1264
|
+
const loaded = loadConfig(options().config);
|
|
1265
|
+
const run = runId ? { runId } : loadLatestRun(loaded.stateDir);
|
|
1266
|
+
const selectedRunId = run?.runId ?? fail("No verification run exists.", "NO_RUN");
|
|
1267
|
+
print(new FileEventStore(loaded.stateDir).verify(selectedRunId));
|
|
1268
|
+
});
|
|
1269
|
+
events.command("lock [run-id]").description("Inspect the latest or selected event-log lock.").action((runId) => {
|
|
1270
|
+
const loaded = loadConfig(options().config);
|
|
1271
|
+
const run = runId ? { runId } : loadLatestRun(loaded.stateDir);
|
|
1272
|
+
const selectedRunId = run?.runId ?? fail("No verification run exists.", "NO_RUN");
|
|
1273
|
+
print(inspectEventLogLock(loaded.stateDir, selectedRunId));
|
|
1274
|
+
});
|
|
1275
|
+
events.command("unlock [run-id]").description("Recover an old event-log lock after confirming its owner is dead.").option("--by <actor>", "recovery actor", "human").option("--max-age-ms <milliseconds>", "minimum lock age", (value) => Number(value), 3e5).action((runId, command) => {
|
|
1276
|
+
const loaded = loadConfig(options().config);
|
|
1277
|
+
const run = runId ? { runId } : loadLatestRun(loaded.stateDir);
|
|
1278
|
+
const selectedRunId = run?.runId ?? fail("No verification run exists.", "NO_RUN");
|
|
1279
|
+
print(recoverEventLogLock({ stateDir: loaded.stateDir, runId: selectedRunId, actor: command.by, maxAgeMs: command.maxAgeMs }));
|
|
1280
|
+
});
|
|
1281
|
+
events.command("export [run-id]").description("Export a reconciled COMPLETE run as a signed evidence bundle.").requiredOption("--output <path>", "bundle output path").requiredOption("--private-key <path>", "Ed25519 private key path").requiredOption("--key-id <id>", "stable signing key identity").action(async (runId, command) => print(await exportEvidenceBundle({ configPath: options().config, runId, outputPath: command.output, privateKeyPath: command.privateKey, keyId: command.keyId })));
|
|
1282
|
+
events.command("verify-bundle <path>").description("Verify an exported signed evidence bundle independently.").option("--trusted-key-store <path>", "JSON trust store with active or revoked public keys").action((path, command) => print(verifyEvidenceBundle(path, { trustedKeys: command.trustedKeyStore ? readEvidenceTrustStore(command.trustedKeyStore) : [] })));
|
|
1283
|
+
var benchmark = program.command("benchmark").description("Aggregate reproducible metrics from historical runs.").option("--manifest <path>", "benchmark manifest for baseline comparison").action((command) => {
|
|
1284
|
+
const loaded = loadConfig(options().config);
|
|
1285
|
+
print(benchmarkRuns(loaded.stateDir, command.manifest ? loadBenchmarkManifest(command.manifest) : void 0));
|
|
1286
|
+
});
|
|
1287
|
+
benchmark.command("baseline <taskId>").description("Record one controlled baseline observation in a benchmark manifest.").option("--manifest <path>", "benchmark manifest path").requiredOption("--status <status>", "passed, failed, blocked, or not-run").requiredOption("--source <source>", "baseline source or run reference").option("--evidence-file <path>", "JSON file with criterion-level baseline evidence").option("--recorded-at <timestamp>", "ISO-8601 timestamp").option("--attempts <count>", "attempt count", (value) => Number(value)).option("--duration-ms <milliseconds>", "duration in milliseconds", (value) => Number(value)).option("--review-minutes <minutes>", "human review time in minutes", (value) => Number(value)).option("--escaped-incomplete <count>", "incomplete deliveries discovered after handoff", (value) => Number(value)).action((taskId, command, cliCommand) => {
|
|
1288
|
+
const manifest = command.manifest ?? cliCommand.parent?.opts().manifest;
|
|
1289
|
+
const manifestPath = manifest ?? fail("baseline requires --manifest <path>.", "INVALID_INPUT");
|
|
1290
|
+
const status = ["passed", "failed", "blocked", "not-run"].includes(command.status) ? command.status : fail("status must be passed, failed, blocked, or not-run.", "INVALID_INPUT");
|
|
1291
|
+
const evidence = command.evidenceFile ? readBenchmarkEvidence(command.evidenceFile) : void 0;
|
|
1292
|
+
print(recordBenchmarkObservation(manifestPath, { taskId, status, source: command.source, ...evidence ? { evidence: evidence.evidence, evidenceDigest: evidence.digest } : {}, ...command.recordedAt ? { recordedAt: command.recordedAt } : {}, ...command.attempts === void 0 ? {} : { attempts: command.attempts }, ...command.durationMs === void 0 ? {} : { durationMs: command.durationMs }, ...command.reviewMinutes === void 0 ? {} : { reviewMinutes: command.reviewMinutes }, ...command.escapedIncomplete === void 0 ? {} : { escapedIncomplete: command.escapedIncomplete } }));
|
|
1293
|
+
});
|
|
1294
|
+
program.command("clean").description("Remove only configured task-owned temporary artifacts.").action(() => print(cleanTaskArtifacts(options().config)));
|
|
1295
|
+
process.on("SIGINT", () => {
|
|
1296
|
+
process.stderr.write("Cancelled.\n");
|
|
1297
|
+
process.exitCode = 130;
|
|
1298
|
+
});
|
|
1299
|
+
try {
|
|
1300
|
+
await program.parseAsync(process.argv);
|
|
1301
|
+
} catch (error) {
|
|
1302
|
+
const value = error instanceof Error ? error : new Error(String(error));
|
|
1303
|
+
process.stderr.write(`${"code" in value ? String(value.code) : "HARNESS_ERROR"}: ${value.message}
|
|
1304
|
+
`);
|
|
1305
|
+
process.exitCode = "code" in value && value.code === "INVALID_INPUT" ? 2 : 1;
|
|
1306
|
+
}
|
|
1307
|
+
//# sourceMappingURL=cli.js.map
|
|
1308
|
+
//# sourceMappingURL=cli.js.map
|