@agentskit/harness 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/CHANGELOG.md +77 -0
  2. package/CODE_OF_CONDUCT.md +5 -0
  3. package/CONTRIBUTING.md +26 -0
  4. package/LICENSE +21 -0
  5. package/README.md +473 -0
  6. package/SECURITY.md +11 -0
  7. package/dist/cli.js +1308 -0
  8. package/dist/cli.js.map +1 -0
  9. package/dist/index.d.ts +968 -0
  10. package/dist/index.js +1828 -0
  11. package/dist/index.js.map +1 -0
  12. package/docs/ADR-0001-extensible-kernel.md +41 -0
  13. package/docs/ADR-0002-profiles-and-context.md +22 -0
  14. package/docs/ADR-0003-doc-bridge-context-binding.md +36 -0
  15. package/docs/ADR-0004-run-metrics.md +29 -0
  16. package/docs/ADR-0005-benchmark-manifest.md +27 -0
  17. package/docs/ADR-0006-agent-session-protocol.md +36 -0
  18. package/docs/ADR-0007-policy-gate.md +33 -0
  19. package/docs/ADR-0008-runtime-executor.md +33 -0
  20. package/docs/ADR-0009-process-runtime-boundary.md +34 -0
  21. package/docs/ADR-0010-docker-sandbox-runtime.md +32 -0
  22. package/docs/ADR-0011-runtime-attestation.md +30 -0
  23. package/docs/ADR-0012-controlled-baseline-observations.md +27 -0
  24. package/docs/ADR-0013-honest-benchmark-comparability.md +27 -0
  25. package/docs/ADR-0014-criterion-level-benchmark-evidence.md +24 -0
  26. package/docs/ADR-0015-directional-benchmark-outcomes.md +23 -0
  27. package/docs/ADR-0016-baseline-evidence-digests.md +21 -0
  28. package/docs/ADR-0017-event-log-integrity.md +25 -0
  29. package/docs/ADR-0018-verification-projection-attestation.md +23 -0
  30. package/docs/ADR-0019-human-decision-attestation.md +27 -0
  31. package/docs/ADR-0020-terminal-reconciliation.md +26 -0
  32. package/docs/ADR-0021-event-lock-recovery.md +25 -0
  33. package/docs/ADR-0022-signed-evidence-bundle.md +27 -0
  34. package/docs/ADR-0023-safe-action-recovery.md +30 -0
  35. package/docs/ADR-0024-controlled-completion-metrics.md +27 -0
  36. package/docs/ADR-0025-ci-dogfood.md +22 -0
  37. package/docs/ADR-0026-ci-evidence-artifact.md +22 -0
  38. package/docs/ADR-0027-portable-evidence.md +19 -0
  39. package/docs/ADR-0028-effective-metrics.md +20 -0
  40. package/docs/ADR-0029-honest-ci-preparation.md +20 -0
  41. package/docs/ADR-0030-agentskit-os-benchmark-bridge.md +20 -0
  42. package/docs/ADR-0031-real-provider-baseline.md +18 -0
  43. package/docs/ADR-0032-harness-equivalent-benchmark.md +25 -0
  44. package/docs/ADR-0033-portable-agent-gate.md +25 -0
  45. package/docs/ADR-0034-measurement-quality-gates.md +25 -0
  46. package/docs/ADR-0035-reproducible-benchmark-samples.md +20 -0
  47. package/docs/ADR-0036-comparable-baseline-samples.md +20 -0
  48. package/docs/ADR-0037-replicated-baseline-collection.md +27 -0
  49. package/docs/ADR-0038-end-to-end-benchmark-boundary.md +28 -0
  50. package/docs/ADR-0039-artifact-and-protocol-metrics.md +39 -0
  51. package/docs/ADR-0040-benchmark-corpus-surfaces.md +32 -0
  52. package/package.json +68 -0
package/dist/index.js ADDED
@@ -0,0 +1,1828 @@
1
+ import { resolve, dirname, join, relative, sep } from 'path';
2
+ import { randomUUID, createPrivateKey, createPublicKey, sign, verify, createHash } from 'crypto';
3
+ import { existsSync, mkdirSync, openSync, writeSync, appendFileSync, closeSync, unlinkSync, readFileSync, writeFileSync, mkdtempSync, renameSync, rmSync, lstatSync, readdirSync } from 'fs';
4
+ import { execFile, spawn } from 'child_process';
5
+ import { promisify } from 'util';
6
+ import { tmpdir } from 'os';
7
+
8
+ // src/constants.ts
9
+ var STATES = [
10
+ "CLARIFYING",
11
+ "PLANNED",
12
+ "IMPLEMENTING",
13
+ "VERIFYING",
14
+ "AWAITING_HUMAN_APPROVAL",
15
+ "AWAITING_AUTHORIZATION",
16
+ "COMPLETE",
17
+ "BLOCKED",
18
+ "STALE",
19
+ "CANCELLED",
20
+ "SUPERSEDED"
21
+ ];
22
+ var LEGAL_TRANSITIONS = {
23
+ CLARIFYING: ["PLANNED", "BLOCKED", "CANCELLED"],
24
+ PLANNED: ["IMPLEMENTING", "CLARIFYING", "STALE", "CANCELLED"],
25
+ IMPLEMENTING: ["VERIFYING", "CLARIFYING", "STALE", "CANCELLED"],
26
+ VERIFYING: ["AWAITING_HUMAN_APPROVAL", "BLOCKED", "STALE", "CANCELLED"],
27
+ AWAITING_HUMAN_APPROVAL: ["AWAITING_AUTHORIZATION", "COMPLETE", "BLOCKED", "IMPLEMENTING", "STALE", "CANCELLED"],
28
+ AWAITING_AUTHORIZATION: ["COMPLETE", "BLOCKED", "IMPLEMENTING", "STALE", "CANCELLED"],
29
+ COMPLETE: ["STALE", "SUPERSEDED"],
30
+ BLOCKED: ["SUPERSEDED"],
31
+ STALE: ["SUPERSEDED", "PLANNED"],
32
+ CANCELLED: ["SUPERSEDED"],
33
+ SUPERSEDED: []
34
+ };
35
+ var REAL_CATEGORIES = /* @__PURE__ */ new Set(["endpoint", "database", "cli", "mcp", "ui"]);
36
+ var DECISIONS = /* @__PURE__ */ new Set(["approved", "approve", "yes", "ok", "rejected", "reject", "no"]);
37
+
38
+ // src/errors.ts
39
+ var HarnessError = class extends Error {
40
+ code;
41
+ constructor(message, code = "HARNESS_ERROR") {
42
+ super(message);
43
+ this.name = "HarnessError";
44
+ this.code = code;
45
+ }
46
+ };
47
+ var fail = (message, code = "HARNESS_ERROR") => {
48
+ throw new HarnessError(message, code);
49
+ };
50
+
51
+ // src/profiles.ts
52
+ var isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
53
+ var record = (value, label) => {
54
+ if (!isRecord(value)) fail(`${label} must be an object.`, "INVALID_CONFIG");
55
+ return value;
56
+ };
57
+ var id = (value, label) => {
58
+ if (typeof value !== "string" || !value.trim()) fail(`${label} must be a non-empty string.`, "INVALID_CONFIG");
59
+ return value;
60
+ };
61
+ var parents = (value, label) => value === void 0 ? [] : Array.isArray(value) ? value.map((item, index2) => id(item, `${label}[${index2}]`)) : [id(value, label)];
62
+ var merge = (base, overlay) => {
63
+ const result = { ...base };
64
+ for (const key of ["surfaces", "budget", "cleanup"]) {
65
+ if (overlay[key] !== void 0) result[key] = { ...isRecord(result[key]) ? result[key] : {}, ...record(overlay[key], `profile.${key}`) };
66
+ }
67
+ if (overlay["checkOverrides"] !== void 0) {
68
+ if (!Array.isArray(overlay["checkOverrides"])) fail("profile.checkOverrides must be an array.", "INVALID_CONFIG");
69
+ const checks = Array.isArray(result["checks"]) ? [...result["checks"]] : [];
70
+ for (const [index2, value] of overlay["checkOverrides"].entries()) {
71
+ const override = record(value, `profile.checkOverrides[${index2}]`);
72
+ const checkId = id(override["id"], `profile.checkOverrides[${index2}].id`);
73
+ const checkIndex = checks.findIndex((check) => isRecord(check) && check["id"] === checkId);
74
+ if (checkIndex < 0) fail(`profile.checkOverrides references unknown check: ${checkId}.`, "INVALID_CONFIG");
75
+ checks[checkIndex] = { ...checks[checkIndex], ...override };
76
+ }
77
+ result["checks"] = checks;
78
+ }
79
+ return result;
80
+ };
81
+ var resolveProfile = (root) => {
82
+ if (root["profiles"] === void 0) return root;
83
+ const profileMap = record(root["profiles"], "profiles");
84
+ const selected = id(root["profile"], "profile");
85
+ const visiting = /* @__PURE__ */ new Set();
86
+ const visited = /* @__PURE__ */ new Map();
87
+ const resolve6 = (name) => {
88
+ const cached = visited.get(name);
89
+ if (cached) return cached;
90
+ if (visiting.has(name)) fail(`Profile inheritance cycle includes ${name}.`, "INVALID_CONFIG");
91
+ const definition = record(profileMap[name], `profiles.${name}`);
92
+ visiting.add(name);
93
+ let result = { ...root };
94
+ for (const parent of parents(definition["extends"], `profiles.${name}.extends`)) result = merge(result, resolve6(parent));
95
+ result = merge(result, definition);
96
+ visiting.delete(name);
97
+ visited.set(name, result);
98
+ return result;
99
+ };
100
+ return resolve6(selected);
101
+ };
102
+ var sha256 = (value) => createHash("sha256").update(value).digest("hex");
103
+ var hashJson = (value) => sha256(JSON.stringify(value));
104
+ var readJson = (path) => JSON.parse(readFileSync(path, "utf8"));
105
+ var writeJson = (path, value) => {
106
+ mkdirSync(dirname(path), { recursive: true });
107
+ writeFileSync(path, `${JSON.stringify(value, null, 2)}
108
+ `, "utf8");
109
+ };
110
+ var pathInside = (root, candidate) => {
111
+ const rel = relative(resolve(root), resolve(candidate));
112
+ return rel === "" || !rel.startsWith(`..${sep}`) && rel !== ".." && !rel.startsWith(sep);
113
+ };
114
+ var latestPath = (stateDir) => join(stateDir, "latest.json");
115
+ var runPath = (stateDir, runId) => join(stateDir, "runs", runId, "run.json");
116
+ var saveRun = (stateDir, run) => writeJson(runPath(stateDir, run.runId), run);
117
+ var readRun = (stateDir, runId) => readJson(runPath(stateDir, runId));
118
+ var loadLatestRun = (stateDir) => {
119
+ if (!existsSync(latestPath(stateDir))) return null;
120
+ const pointer = readJson(latestPath(stateDir));
121
+ return readRun(stateDir, pointer.runId);
122
+ };
123
+ var setLatest = (stateDir, run) => writeJson(latestPath(stateDir), {
124
+ runId: run.runId,
125
+ path: relative(resolve(stateDir, "..", ".."), runPath(stateDir, run.runId)),
126
+ updatedAt: (/* @__PURE__ */ new Date()).toISOString()
127
+ });
128
+ var cleanConfiguredArtifacts = (loaded) => {
129
+ const roots = loaded.config.cleanup?.roots ?? [];
130
+ for (const root of roots) {
131
+ const target = resolve(loaded.root, root);
132
+ if (!pathInside(loaded.root, target)) fail(`Cleanup root escapes project root: ${root}`, "INVALID_CONFIG");
133
+ if (existsSync(target)) for (const entry of readdirSync(target)) rmSync(join(target, entry), { recursive: true, force: true });
134
+ }
135
+ return { cleaned: roots };
136
+ };
137
+ var fileContents = (path) => readFileSync(path, "utf8");
138
+
139
+ // src/types.ts
140
+ var SURFACE_NAMES = ["logic", "endpoint", "database", "cli", "mcp", "ui", "docs"];
141
+ var CHECK_CATEGORIES = ["build", "test", "lint", ...SURFACE_NAMES, "custom"];
142
+ var RUN_STATES = [
143
+ "CLARIFYING",
144
+ "PLANNED",
145
+ "IMPLEMENTING",
146
+ "VERIFYING",
147
+ "AWAITING_HUMAN_APPROVAL",
148
+ "AWAITING_AUTHORIZATION",
149
+ "COMPLETE",
150
+ "BLOCKED",
151
+ "STALE",
152
+ "CANCELLED",
153
+ "SUPERSEDED"
154
+ ];
155
+
156
+ // src/config.ts
157
+ var isRecord2 = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
158
+ var stringValue = (value, label) => {
159
+ if (typeof value !== "string") return fail(`${label} is required.`, "INVALID_CONFIG");
160
+ const result = value.trim();
161
+ if (!result) return fail(`${label} is required.`, "INVALID_CONFIG");
162
+ return result;
163
+ };
164
+ var stringArray = (value, label) => {
165
+ if (!Array.isArray(value)) fail(`${label} must be an array of non-empty strings.`, "INVALID_CONFIG");
166
+ const items = value;
167
+ if (!items.every((item) => typeof item === "string" && Boolean(item.trim()))) fail(`${label} must be an array of non-empty strings.`, "INVALID_CONFIG");
168
+ return items.map((item) => stringValue(item, label));
169
+ };
170
+ var asRecord = (value, label) => {
171
+ if (!isRecord2(value)) fail(`${label} must be an object.`, "INVALID_CONFIG");
172
+ return value;
173
+ };
174
+ var surface = (value, name) => {
175
+ if (typeof value === "boolean") return value ? { required: true } : { required: false, reason: `${name} is not applicable.` };
176
+ const record3 = asRecord(value, `surfaces.${name}`);
177
+ if (typeof record3["required"] !== "boolean") fail(`surfaces.${name}.required must be boolean.`, "INVALID_CONFIG");
178
+ if (!record3["required"] && typeof record3["reason"] !== "string") fail(`surfaces.${name}.reason is required when not applicable.`, "INVALID_CONFIG");
179
+ return { required: record3["required"], ...typeof record3["reason"] === "string" ? { reason: record3["reason"] } : {} };
180
+ };
181
+ var parseCheck = (value, index2) => {
182
+ const record3 = asRecord(value, `checks[${index2}]`);
183
+ const id2 = stringValue(record3["id"], `checks[${index2}].id`);
184
+ const category = stringValue(record3["category"], `checks[${index2}].category`);
185
+ if (!CHECK_CATEGORIES.includes(category)) fail(`checks[${index2}].category is invalid.`, "INVALID_CONFIG");
186
+ const command = stringValue(record3["command"], `checks[${index2}].command`);
187
+ if (REAL_CATEGORIES.has(category) && record3["execution"] !== "real") fail(`checks[${index2}] must declare execution: real.`, "INVALID_CONFIG");
188
+ if (record3["evidence"] !== "structured") fail(`checks[${index2}] must declare evidence: structured.`, "INVALID_CONFIG");
189
+ if (record3["capabilities"] !== void 0 && (!Array.isArray(record3["capabilities"]) || !record3["capabilities"].every((item) => typeof item === "string"))) fail(`checks[${index2}].capabilities must be strings.`, "INVALID_CONFIG");
190
+ const capabilities = Array.isArray(record3["capabilities"]) ? record3["capabilities"].filter((item) => typeof item === "string") : void 0;
191
+ if (category === "ui" && !capabilities?.includes("real-browser")) fail(`checks[${index2}] must declare real-browser.`, "INVALID_CONFIG");
192
+ if (category === "ui" && !capabilities?.includes("screenshot")) fail(`checks[${index2}] must declare screenshot.`, "INVALID_CONFIG");
193
+ if (record3["required"] !== void 0 && typeof record3["required"] !== "boolean") fail(`checks[${index2}].required must be boolean.`, "INVALID_CONFIG");
194
+ if (record3["timeoutMs"] !== void 0 && (!Number.isInteger(record3["timeoutMs"]) || typeof record3["timeoutMs"] !== "number" || record3["timeoutMs"] < 1)) fail(`checks[${index2}].timeoutMs must be positive.`, "INVALID_CONFIG");
195
+ return { id: id2, category, command, required: record3["required"] !== false, timeoutMs: typeof record3["timeoutMs"] === "number" ? record3["timeoutMs"] : 12e4, ...record3["execution"] === "real" ? { execution: "real" } : {}, ...capabilities ? { capabilities } : {}, evidence: "structured" };
196
+ };
197
+ var parseOutcome = (value, index2, checks) => {
198
+ const record3 = asRecord(value, `contract.outcomes[${index2}]`);
199
+ const id2 = stringValue(record3["id"], `contract.outcomes[${index2}].id`);
200
+ const statement = stringValue(record3["statement"], `contract.outcomes[${index2}].statement`);
201
+ const ids = stringArray(record3["checks"], `contract.outcomes[${index2}].checks`);
202
+ if (ids.some((checkId) => !checks.some((check) => check.id === checkId))) fail(`contract.outcomes[${index2}] references an unknown check.`, "INVALID_CONFIG");
203
+ return { id: id2, statement, checks: [...new Set(ids)] };
204
+ };
205
+ var validateConfig = (rawValue) => {
206
+ const raw = resolveProfile(asRecord(rawValue, "verification config"));
207
+ if (raw["schemaVersion"] !== 1) fail("verification config schemaVersion must be 1.", "INVALID_CONFIG");
208
+ const project = stringValue(raw["project"], "verification config project");
209
+ const contractRaw = asRecord(raw["contract"], "contract");
210
+ const rawChecks = raw["checks"];
211
+ const checks = Array.isArray(rawChecks) ? rawChecks.map(parseCheck) : fail("checks must be a non-empty array.", "INVALID_CONFIG");
212
+ if (!checks.length || new Set(checks.map((check) => check.id)).size !== checks.length) fail("check ids must be unique.", "INVALID_CONFIG");
213
+ const scopeRaw = asRecord(contractRaw["scope"], "contract.scope");
214
+ const scope = { inScope: stringArray(scopeRaw["inScope"], "contract.scope.inScope"), outOfScope: stringArray(scopeRaw["outOfScope"], "contract.scope.outOfScope") };
215
+ const ambiguities = stringArray(contractRaw["ambiguities"], "contract.ambiguities");
216
+ const rawOutcomes = contractRaw["outcomes"];
217
+ const outcomes = Array.isArray(rawOutcomes) ? rawOutcomes.map((outcome, index2) => parseOutcome(outcome, index2, checks)) : fail("contract.outcomes must be a non-empty array.", "INVALID_CONFIG");
218
+ if (!outcomes.length || new Set(outcomes.map((outcome) => outcome.id)).size !== outcomes.length) fail("outcome ids must be unique.", "INVALID_CONFIG");
219
+ const mapped = new Set(outcomes.flatMap((outcome) => outcome.checks));
220
+ if (checks.some((check) => check.required && !mapped.has(check.id))) fail("every required check must map to an outcome.", "INVALID_CONFIG");
221
+ const rawSurfaces = isRecord2(raw["surfaces"]) ? raw["surfaces"] : void 0;
222
+ const surfaces = Object.fromEntries(SURFACE_NAMES.map((name) => [name, surface(rawSurfaces?.[name] ?? name === "logic", name)]));
223
+ for (const name of SURFACE_NAMES) if (surfaces[name].required && !checks.some((check) => check.required && check.category === name)) fail(`required surface ${name} has no required check.`, "INVALID_CONFIG");
224
+ const trackingRaw = isRecord2(raw["tracking"]) ? raw["tracking"] : { required: false, reason: "tracking is not configured for this run." };
225
+ if (trackingRaw["required"] === true && typeof trackingRaw["target"] !== "string") fail("tracking.target is required when tracking is enabled.", "INVALID_CONFIG");
226
+ if (trackingRaw["required"] !== true && typeof trackingRaw["reason"] !== "string") fail("tracking.reason is required when tracking is disabled.", "INVALID_CONFIG");
227
+ const budgetRaw = raw["budget"] === void 0 ? void 0 : asRecord(raw["budget"], "budget");
228
+ if (budgetRaw && budgetRaw["maxDurationMs"] !== void 0 && (!Number.isInteger(budgetRaw["maxDurationMs"]) || typeof budgetRaw["maxDurationMs"] !== "number" || budgetRaw["maxDurationMs"] < 1)) fail("budget.maxDurationMs must be positive.", "INVALID_CONFIG");
229
+ const cleanupRaw = raw["cleanup"] === void 0 ? void 0 : asRecord(raw["cleanup"], "cleanup");
230
+ const cleanup = cleanupRaw ? { roots: cleanupRaw["roots"] === void 0 ? void 0 : stringArray(cleanupRaw["roots"], "cleanup.roots") } : void 0;
231
+ const benchmarkRaw = raw["benchmark"] === void 0 ? void 0 : asRecord(raw["benchmark"], "benchmark");
232
+ const benchmark = benchmarkRaw ? { suiteId: stringValue(benchmarkRaw["suiteId"], "benchmark.suiteId"), taskId: stringValue(benchmarkRaw["taskId"], "benchmark.taskId"), mode: benchmarkRaw["mode"] === "harness" ? "harness" : fail("benchmark.mode must be harness.", "INVALID_CONFIG") } : void 0;
233
+ const contract = { intent: stringValue(contractRaw["intent"], "contract.intent"), scope, ambiguities, outcomes };
234
+ const tracking = { required: trackingRaw["required"] === true, ...typeof trackingRaw["target"] === "string" ? { target: trackingRaw["target"] } : {}, ...typeof trackingRaw["reason"] === "string" ? { reason: trackingRaw["reason"] } : {} };
235
+ return { schemaVersion: 1, project, ...typeof raw["root"] === "string" ? { root: raw["root"] } : {}, ...typeof raw["stateDir"] === "string" ? { stateDir: raw["stateDir"] } : {}, profile: typeof raw["profile"] === "string" ? raw["profile"] : "strict", contract, surfaces, checks, tracking, ...budgetRaw ? { budget: { maxDurationMs: budgetRaw["maxDurationMs"] } } : {}, ...cleanup ? { cleanup } : {}, ...benchmark ? { benchmark } : {} };
236
+ };
237
+ var loadConfig = (configPath = ".codex/verification.json") => {
238
+ const absolute = resolve(configPath);
239
+ const raw = readJson(absolute);
240
+ const rawRecord = asRecord(raw, "verification config");
241
+ const root = resolve(dirname(absolute), typeof rawRecord["root"] === "string" ? rawRecord["root"] : ".");
242
+ const stateDir = resolve(root, typeof rawRecord["stateDir"] === "string" ? rawRecord["stateDir"] : ".codex/verification");
243
+ if (!pathInside(root, stateDir)) fail("stateDir must be inside the project root.", "INVALID_CONFIG");
244
+ const config = validateConfig(raw);
245
+ return { absolute, root, stateDir, config, configHash: hashJson(config) };
246
+ };
247
+
248
+ // src/state-machine.ts
249
+ var transition = (run, to, reason, actor = "harness") => {
250
+ if (!STATES.includes(to)) fail(`Unknown state ${to}.`, "INVALID_STATE");
251
+ if (run.state !== to && !LEGAL_TRANSITIONS[run.state].some((state) => state === to)) fail(`Illegal transition ${run.state} -> ${to}.`, "INVALID_STATE");
252
+ const event = { from: run.state, to, at: (/* @__PURE__ */ new Date()).toISOString(), actor, ...reason ? { reason } : {} };
253
+ return { ...run, state: to, transitions: [...run.transitions, event] };
254
+ };
255
+ var assertHuman = (actor) => {
256
+ if (actor !== "human") fail("This action requires --by human.", "HUMAN_APPROVAL_REQUIRED");
257
+ };
258
+ var approvedDecision = (decision) => {
259
+ if (!DECISIONS.has(decision)) fail("Decision must be approved or rejected.", "INVALID_INPUT");
260
+ return ["approved", "approve", "yes", "ok"].includes(decision);
261
+ };
262
+ var HARNESS_EVENT_SCHEMA_VERSION = 1;
263
+ var EVENT_LOG_GENESIS = "GENESIS";
264
+ var HARNESS_EVENT_TYPES = ["run.created", "state.transitioned", "context.attached", "verification.completed", "approval.recorded", "authorization.recorded", "session.started", "session.resumed", "agent.turn.started", "policy.evaluated", "tool.approval.requested", "tool.approval.recorded", "tool.requested", "tool.execution.started", "tool.recovery.recorded", "tool.blocked", "tool.completed", "tool.failed", "session.ended"];
265
+ var SESSION_EVENT_TYPES = /* @__PURE__ */ new Set(["session.started", "session.resumed", "agent.turn.started", "policy.evaluated", "tool.approval.requested", "tool.approval.recorded", "tool.requested", "tool.execution.started", "tool.recovery.recorded", "tool.blocked", "tool.completed", "tool.failed", "session.ended"]);
266
+ var eventPath = (stateDir, runId) => join(stateDir, "runs", runId, "events.ndjson");
267
+ var lockPath = (stateDir, runId) => `${eventPath(stateDir, runId)}.lock`;
268
+ var parseLock = (value) => {
269
+ try {
270
+ const record3 = JSON.parse(value);
271
+ if (!Number.isInteger(record3["pid"]) || record3["pid"] <= 0 || typeof record3["at"] !== "string" || !Number.isFinite(Date.parse(record3["at"]))) fail("Event log lock metadata is invalid.", "HARNESS_ERROR");
272
+ return { pid: record3["pid"], at: record3["at"] };
273
+ } catch (error) {
274
+ if (error instanceof SyntaxError) fail("Event log lock metadata is invalid.", "HARNESS_ERROR");
275
+ throw error;
276
+ }
277
+ };
278
+ var readLock = (stateDir, runId) => {
279
+ const path = lockPath(stateDir, runId);
280
+ return existsSync(path) ? parseLock(readFileSync(path, "utf8")) : null;
281
+ };
282
+ var isEventType = (value) => typeof value === "string" && HARNESS_EVENT_TYPES.includes(value);
283
+ var digest = (value) => /^[a-f0-9]{64}$/.test(value);
284
+ var eventBody = (event) => {
285
+ const { eventHash: _eventHash, ...body2 } = event;
286
+ return body2;
287
+ };
288
+ var eventDigest = (event) => sha256(JSON.stringify(eventBody(event)));
289
+ var parseEvent = (value, expectedSequence) => {
290
+ if (typeof value !== "object" || value === null || Array.isArray(value)) fail("Event log contains a non-object record.", "HARNESS_ERROR");
291
+ const record3 = value;
292
+ if (record3["schemaVersion"] !== HARNESS_EVENT_SCHEMA_VERSION || typeof record3["runId"] !== "string" || typeof record3["sequence"] !== "number" || record3["sequence"] !== expectedSequence || typeof record3["at"] !== "string" || typeof record3["sourceRevision"] !== "string" || typeof record3["configHash"] !== "string" || !isEventType(record3["type"]) || typeof record3["payload"] !== "object" || record3["payload"] === null || record3["sessionId"] !== void 0 && (typeof record3["sessionId"] !== "string" || !record3["sessionId"].trim()) || SESSION_EVENT_TYPES.has(record3["type"]) && typeof record3["sessionId"] !== "string") fail("Event log is invalid or out of order.", "HARNESS_ERROR");
293
+ const hasPreviousHash = record3["previousHash"] !== void 0;
294
+ const hasEventHash = record3["eventHash"] !== void 0;
295
+ if (hasPreviousHash !== hasEventHash || hasPreviousHash && (typeof record3["previousHash"] !== "string" || record3["previousHash"] !== EVENT_LOG_GENESIS && !digest(record3["previousHash"]) || typeof record3["eventHash"] !== "string" || !digest(record3["eventHash"]))) fail("Event log integrity metadata is invalid.", "HARNESS_ERROR");
296
+ return record3;
297
+ };
298
+ var validateChain = (events) => {
299
+ const current = events.filter((event) => event.eventHash !== void 0);
300
+ if (!current.length) return { status: "legacy", eventCount: events.length };
301
+ if (current.length !== events.length) fail("Event log mixes legacy and hashed records.", "HARNESS_ERROR");
302
+ let previous = EVENT_LOG_GENESIS;
303
+ for (const event of events) {
304
+ const eventHash = event.eventHash ?? fail("Event log hash chain is invalid.", "HARNESS_ERROR");
305
+ if (event.previousHash !== previous || eventHash !== eventDigest(event)) fail("Event log hash chain is invalid.", "HARNESS_ERROR");
306
+ previous = eventHash;
307
+ }
308
+ return { status: "verified", eventCount: events.length, ...events.length ? { headHash: previous } : {} };
309
+ };
310
+ var FileEventStore = class {
311
+ constructor(stateDir) {
312
+ this.stateDir = stateDir;
313
+ }
314
+ stateDir;
315
+ append(event) {
316
+ if (!event.runId.trim()) fail("Event runId is required.", "INVALID_INPUT");
317
+ if (!event.sourceRevision.trim() || !event.configHash.trim()) fail("Event sourceRevision and configHash are required.", "INVALID_INPUT");
318
+ if (!isEventType(event.type)) fail("Event type is invalid.", "INVALID_INPUT");
319
+ if (SESSION_EVENT_TYPES.has(event.type) && (!event.sessionId || !event.sessionId.trim())) fail("Session events require a sessionId.", "INVALID_INPUT");
320
+ if (event.sessionId !== void 0 && !event.sessionId.trim()) fail("Event sessionId cannot be empty.", "INVALID_INPUT");
321
+ const path = eventPath(this.stateDir, event.runId);
322
+ const lock = lockPath(this.stateDir, event.runId);
323
+ mkdirSync(join(this.stateDir, "runs", event.runId), { recursive: true });
324
+ let lockFd;
325
+ try {
326
+ lockFd = openSync(lock, "wx");
327
+ writeSync(lockFd, JSON.stringify({ pid: process.pid, at: (/* @__PURE__ */ new Date()).toISOString() }));
328
+ } catch (error) {
329
+ if (error.code === "EEXIST") fail("Event log is busy; retry the operation.", "HARNESS_ERROR");
330
+ throw error;
331
+ }
332
+ try {
333
+ const events = this.readUnlocked(event.runId);
334
+ const previous = events.at(-1);
335
+ const body2 = { schemaVersion: HARNESS_EVENT_SCHEMA_VERSION, sequence: events.length + 1, at: (/* @__PURE__ */ new Date()).toISOString(), runId: event.runId, sourceRevision: event.sourceRevision, configHash: event.configHash, ...event.sessionId ? { sessionId: event.sessionId } : {}, ...previous?.eventHash ? { previousHash: previous.eventHash } : events.length ? {} : { previousHash: EVENT_LOG_GENESIS }, type: event.type, payload: event.payload };
336
+ const record3 = events.length && !previous?.eventHash ? body2 : { ...body2, eventHash: eventDigest(body2) };
337
+ appendFileSync(path, `${JSON.stringify(record3)}
338
+ `, "utf8");
339
+ return record3;
340
+ } finally {
341
+ closeSync(lockFd);
342
+ unlinkSync(lock);
343
+ }
344
+ }
345
+ readUnlocked(runId) {
346
+ const path = eventPath(this.stateDir, runId);
347
+ if (!existsSync(path)) return [];
348
+ const events = readFileSync(path, "utf8").split(/\r?\n/).map((line) => line.trim()).filter(Boolean).map((line, index2) => {
349
+ try {
350
+ return parseEvent(JSON.parse(line), index2 + 1);
351
+ } catch (error) {
352
+ if (error instanceof SyntaxError) fail("Event log contains invalid JSON.", "HARNESS_ERROR");
353
+ throw error;
354
+ }
355
+ });
356
+ validateChain(events);
357
+ return events;
358
+ }
359
+ read(runId) {
360
+ if (existsSync(lockPath(this.stateDir, runId))) fail("Event log is busy; retry the operation.", "HARNESS_ERROR");
361
+ return this.readUnlocked(runId);
362
+ }
363
+ verify(runId) {
364
+ return validateChain(this.read(runId));
365
+ }
366
+ };
367
+ var inspectEventLogLock = (stateDir, runId) => {
368
+ const path = lockPath(stateDir, runId);
369
+ const lock = readLock(stateDir, runId);
370
+ return lock ? { status: "locked", path, lock } : { status: "unlocked", path };
371
+ };
372
+ var recoverEventLogLock = ({ stateDir, runId, actor, maxAgeMs = 3e5 }) => {
373
+ if (actor !== "human") fail("Event log lock recovery requires a human actor.", "HUMAN_APPROVAL_REQUIRED");
374
+ if (!Number.isInteger(maxAgeMs) || maxAgeMs < 0) fail("maxAgeMs must be a non-negative integer.", "INVALID_INPUT");
375
+ const path = lockPath(stateDir, runId);
376
+ const lock = readLock(stateDir, runId);
377
+ if (!lock) return { status: "unlocked", path };
378
+ const ageMs = Date.now() - Date.parse(lock.at);
379
+ if (ageMs < maxAgeMs) fail("Event log lock is not old enough to recover.", "HARNESS_ERROR");
380
+ try {
381
+ process.kill(lock.pid, 0);
382
+ } catch (error) {
383
+ if (error.code !== "ESRCH") fail("Event log lock owner cannot be proven dead.", "HARNESS_ERROR");
384
+ unlinkSync(path);
385
+ return { status: "recovered", path, lock };
386
+ }
387
+ return fail("Event log lock owner is still alive.", "HARNESS_ERROR");
388
+ };
389
+
390
+ // src/plugins.ts
391
+ var HARNESS_PLUGIN_API_VERSION = 1;
392
+ var createPluginSlot = (id2) => {
393
+ if (!id2.trim()) fail("Plugin slot id is required.", "INVALID_INPUT");
394
+ return { id: id2 };
395
+ };
396
+ var validId = (value, label) => {
397
+ if (!value.trim()) fail(`${label} is required.`, "INVALID_INPUT");
398
+ return value;
399
+ };
400
+ var createPluginRegistry = () => {
401
+ const plugins = /* @__PURE__ */ new Map();
402
+ const contributions = /* @__PURE__ */ new Map();
403
+ const listeners = /* @__PURE__ */ new Map();
404
+ const cleanups = [];
405
+ let mounted = false;
406
+ let disposed = false;
407
+ const ensureOpen = () => {
408
+ if (disposed) fail("Plugin registry has been disposed.", "HARNESS_ERROR");
409
+ };
410
+ const removeContribution = (slot, id2, pluginId) => {
411
+ const entries = contributions.get(slot.id);
412
+ if (entries?.get(id2)?.pluginId === pluginId) entries.delete(id2);
413
+ };
414
+ const registerContribution = (pluginId, slot, id2, value) => {
415
+ const entries = contributions.get(slot.id) ?? /* @__PURE__ */ new Map();
416
+ if (entries.has(id2)) fail(`Plugin contribution already exists: ${slot.id}/${id2}.`, "INVALID_INPUT");
417
+ entries.set(id2, { pluginId, id: id2, value });
418
+ contributions.set(slot.id, entries);
419
+ const disposer = () => removeContribution(slot, id2, pluginId);
420
+ cleanups.push(disposer);
421
+ return disposer;
422
+ };
423
+ const order = () => {
424
+ const result = [];
425
+ const visiting = /* @__PURE__ */ new Set();
426
+ const visited = /* @__PURE__ */ new Set();
427
+ const visit = (id2) => {
428
+ if (visited.has(id2)) return;
429
+ if (visiting.has(id2)) fail(`Plugin dependency cycle includes ${id2}.`, "INVALID_INPUT");
430
+ const candidate = plugins.get(id2);
431
+ if (!candidate) fail(`Plugin dependency is missing: ${id2}.`, "INVALID_INPUT");
432
+ const plugin = candidate;
433
+ visiting.add(id2);
434
+ for (const dependency of plugin.requires ?? []) visit(dependency);
435
+ visiting.delete(id2);
436
+ visited.add(id2);
437
+ result.push(plugin);
438
+ };
439
+ for (const id2 of plugins.keys()) visit(id2);
440
+ return result;
441
+ };
442
+ const registry = {
443
+ register(plugin) {
444
+ ensureOpen();
445
+ if (mounted) fail("Plugins cannot be registered after mount.", "HARNESS_ERROR");
446
+ validId(plugin.id, "Plugin id");
447
+ validId(plugin.version, "Plugin version");
448
+ if (plugin.apiVersion !== HARNESS_PLUGIN_API_VERSION) fail(`Unsupported plugin API version: ${String(plugin.apiVersion)}.`, "INVALID_INPUT");
449
+ if (plugins.has(plugin.id)) fail(`Plugin already registered: ${plugin.id}.`, "INVALID_INPUT");
450
+ plugins.set(plugin.id, plugin);
451
+ },
452
+ mount() {
453
+ ensureOpen();
454
+ if (mounted) return;
455
+ try {
456
+ for (const plugin of order()) {
457
+ const context = {
458
+ apiVersion: HARNESS_PLUGIN_API_VERSION,
459
+ register: (slot, id2, value) => registerContribution(plugin.id, slot, validId(id2, "Plugin contribution id"), value),
460
+ effect: (disposer) => {
461
+ cleanups.push(disposer);
462
+ },
463
+ on: (type, listener) => {
464
+ const handlers = listeners.get(type) ?? /* @__PURE__ */ new Set();
465
+ handlers.add(listener);
466
+ listeners.set(type, handlers);
467
+ const disposer = () => {
468
+ handlers.delete(listener);
469
+ };
470
+ cleanups.push(disposer);
471
+ return disposer;
472
+ }
473
+ };
474
+ const cleanup = plugin.apply(context);
475
+ if (cleanup) cleanups.push(cleanup);
476
+ }
477
+ mounted = true;
478
+ } catch (error) {
479
+ registry.dispose();
480
+ throw error;
481
+ }
482
+ },
483
+ emit(event) {
484
+ ensureOpen();
485
+ for (const listener of listeners.get(event.type) ?? []) listener(event);
486
+ },
487
+ on(type, listener) {
488
+ ensureOpen();
489
+ const handlers = listeners.get(type) ?? /* @__PURE__ */ new Set();
490
+ handlers.add(listener);
491
+ listeners.set(type, handlers);
492
+ return () => {
493
+ handlers.delete(listener);
494
+ };
495
+ },
496
+ contributions: (slot) => [...contributions.get(slot.id)?.values() ?? []],
497
+ dispose() {
498
+ if (disposed) return;
499
+ let firstError;
500
+ for (const cleanup of cleanups.splice(0).reverse()) {
501
+ try {
502
+ cleanup();
503
+ } catch (error) {
504
+ firstError ??= error;
505
+ }
506
+ }
507
+ contributions.clear();
508
+ listeners.clear();
509
+ disposed = true;
510
+ mounted = false;
511
+ if (firstError) throw firstError;
512
+ }
513
+ };
514
+ return registry;
515
+ };
516
+
517
+ // src/context.ts
518
+ var hashContextSnapshot = ({ providerId, query, references, sourceHash: sourceHash2 }) => hashJson({ providerId, query, references, sourceHash: sourceHash2 });
519
+ var hashContextSnapshots = (snapshots) => hashJson(snapshots.map(({ providerId, query, references, sourceHash: sourceHash2, snapshotHash }) => ({ providerId, query, references, sourceHash: sourceHash2, snapshotHash })));
520
+ var record2 = (value, label) => {
521
+ if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${label} must be an object.`, "INVALID_INPUT");
522
+ return value;
523
+ };
524
+ var requiredString = (value, label) => {
525
+ if (typeof value !== "string" || !value.trim()) fail(`${label} must be a non-empty string.`, "INVALID_INPUT");
526
+ return value;
527
+ };
528
+ var validateContextSnapshot = (value, index2 = 0) => {
529
+ const raw = record2(value, `context snapshot ${index2}`);
530
+ const rawQuery = record2(raw["query"], `context snapshot ${index2}.query`);
531
+ const rawReferences = raw["references"];
532
+ if (!Array.isArray(rawReferences)) fail(`context snapshot ${index2}.references must be an array.`, "INVALID_INPUT");
533
+ const references = rawReferences.map((reference, referenceIndex) => {
534
+ const rawReference = record2(reference, `context snapshot ${index2}.references[${referenceIndex}]`);
535
+ return {
536
+ id: requiredString(rawReference["id"], `context snapshot ${index2}.references[${referenceIndex}].id`),
537
+ uri: requiredString(rawReference["uri"], `context snapshot ${index2}.references[${referenceIndex}].uri`),
538
+ ...typeof rawReference["title"] === "string" ? { title: rawReference["title"] } : {},
539
+ ...typeof rawReference["version"] === "string" ? { version: rawReference["version"] } : {},
540
+ ...typeof rawReference["contentHash"] === "string" ? { contentHash: rawReference["contentHash"] } : {}
541
+ };
542
+ });
543
+ const scope = rawQuery["scope"] === void 0 ? void 0 : Array.isArray(rawQuery["scope"]) && rawQuery["scope"].every((item) => typeof item === "string") ? rawQuery["scope"] : fail(`context snapshot ${index2}.query.scope must be an array of strings.`, "INVALID_INPUT");
544
+ const snapshot = {
545
+ providerId: requiredString(raw["providerId"], `context snapshot ${index2}.providerId`),
546
+ query: { query: requiredString(rawQuery["query"], `context snapshot ${index2}.query.query`), ...scope ? { scope } : {}, ...typeof rawQuery["sourceRevision"] === "string" ? { sourceRevision: rawQuery["sourceRevision"] } : {} },
547
+ references,
548
+ sourceHash: requiredString(raw["sourceHash"], `context snapshot ${index2}.sourceHash`),
549
+ snapshotHash: requiredString(raw["snapshotHash"], `context snapshot ${index2}.snapshotHash`),
550
+ resolvedAt: requiredString(raw["resolvedAt"], `context snapshot ${index2}.resolvedAt`)
551
+ };
552
+ if (snapshot.snapshotHash !== hashContextSnapshot(snapshot)) fail(`context snapshot ${index2}.snapshotHash does not match its contents.`, "INVALID_INPUT");
553
+ return snapshot;
554
+ };
555
+ var readContextSnapshots = (path) => {
556
+ const value = JSON.parse(readFileSync(path, "utf8"));
557
+ return (Array.isArray(value) ? value : [value]).map((snapshot, index2) => validateContextSnapshot(snapshot, index2));
558
+ };
559
+ var validateContextSnapshots = (snapshots) => snapshots.map((snapshot, index2) => validateContextSnapshot(snapshot, index2));
560
+ var CONTEXT_PROVIDER_SLOT = createPluginSlot("context.provider");
561
+
562
+ // src/runs.ts
563
+ var now = () => (/* @__PURE__ */ new Date()).toISOString();
564
+ var newRunId = () => `${Date.now()}-${process.pid}-${Math.random().toString(36).slice(2, 8)}`;
565
+ var saveRun2 = (stateDir, run) => {
566
+ saveRun(stateDir, run);
567
+ const store = new FileEventStore(stateDir);
568
+ const events = store.read(run.runId);
569
+ if (!events.some((event) => event.type === "run.created")) store.append({ runId: run.runId, sourceRevision: run.sourceRevision, configHash: run.configHash, type: "run.created", payload: { project: run.project, baselineRevision: run.baseline.revision, baselineStatusHash: run.baseline.statusHash } });
570
+ const loggedTransitions = new Set(events.filter((event) => event.type === "state.transitioned").map((event) => event.payload.transitionIndex));
571
+ run.transitions.forEach((transition2, transitionIndex) => {
572
+ if (loggedTransitions.has(transitionIndex)) return;
573
+ store.append({ runId: run.runId, sourceRevision: run.sourceRevision, configHash: run.configHash, type: "state.transitioned", payload: { from: transition2.from, to: transition2.to, actor: transition2.actor ?? "harness", ...transition2.reason ? { reason: transition2.reason } : {}, transitionIndex } });
574
+ });
575
+ const loggedSnapshots = new Set(events.filter((event) => event.type === "context.attached").map((event) => event.payload.snapshotHash));
576
+ for (const snapshot of run.contextSnapshots ?? []) {
577
+ if (loggedSnapshots.has(snapshot.snapshotHash)) continue;
578
+ store.append({ runId: run.runId, sourceRevision: run.sourceRevision, configHash: run.configHash, type: "context.attached", payload: { providerId: snapshot.providerId, sourceHash: snapshot.sourceHash, snapshotHash: snapshot.snapshotHash, query: snapshot.query } });
579
+ }
580
+ };
581
+ var createRun = async ({ loaded, baseline, supersedes, dirtyBaselineAuthorized, contextSnapshots = [], planner = "human" }) => {
582
+ const contractHash = hashJson(loaded.config.contract);
583
+ const run = {
584
+ type: "agentskit-harness-run",
585
+ schemaVersion: 1,
586
+ runId: newRunId(),
587
+ project: loaded.config.project,
588
+ state: "PLANNED",
589
+ configHash: loaded.configHash,
590
+ contractHash,
591
+ sourceRevision: baseline.revision,
592
+ sourceStatusHash: baseline.statusHash,
593
+ baseline,
594
+ ...planner === "human" ? { contractApproval: { actor: "human", at: now(), contractHash } } : { contractPreparation: { actor: "ci", at: now(), contractHash } },
595
+ checks: loaded.config.checks.map(({ id: id2, category }) => ({ id: id2, category, status: "pending" })),
596
+ contextSnapshots,
597
+ ...contextSnapshots.length ? { contextHash: hashContextSnapshots(contextSnapshots) } : {},
598
+ ...loaded.config.benchmark ? { benchmark: loaded.config.benchmark } : {},
599
+ outcomes: loaded.config.contract.outcomes.map(({ id: id2, statement, checks }) => ({ id: id2, statement, checks, status: "pending" })),
600
+ transitions: [{ from: null, to: "PLANNED", at: now(), actor: planner }],
601
+ evidenceReferences: [],
602
+ ...supersedes ? { supersedes } : {},
603
+ ...dirtyBaselineAuthorized ? { dirtyBaselineAuthorized: true } : {}
604
+ };
605
+ saveRun2(loaded.stateDir, run);
606
+ setLatest(loaded.stateDir, run);
607
+ return run;
608
+ };
609
+ var parseStructuredEvidence = (stdout) => {
610
+ for (const line of stdout.split(/\r?\n/).map((item) => item.trim()).filter(Boolean).reverse()) {
611
+ try {
612
+ const value = JSON.parse(line);
613
+ if (typeof value === "object" && value !== null && !Array.isArray(value) && typeof value["status"] === "string") return value;
614
+ } catch {
615
+ }
616
+ }
617
+ return null;
618
+ };
619
+ var isRecord3 = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
620
+ var viewportValid = (viewport) => typeof viewport === "string" || isRecord3(viewport) && typeof viewport["width"] === "number" && viewport["width"] > 0 && typeof viewport["height"] === "number" && viewport["height"] > 0;
621
+ var validateEvidence = (root, check, evidence, outcomeIds) => {
622
+ if (!evidence || evidence.status !== "passed") return ["structured evidence did not pass"];
623
+ if (!Array.isArray(evidence.criteria) || !evidence.criteria.every((id2) => typeof id2 === "string") || outcomeIds.some((id2) => !evidence.criteria.includes(id2))) return [`evidence must map criteria: ${outcomeIds.join(", ")}`];
624
+ const failures = [];
625
+ if (check.category === "ui") {
626
+ if (evidence.capability !== "real-browser") failures.push("UI evidence must declare capability real-browser");
627
+ if (!Array.isArray(evidence.artifacts) || evidence.artifacts.length === 0) failures.push("UI evidence requires screenshot artifacts");
628
+ }
629
+ const artifacts = Array.isArray(evidence.artifacts) ? evidence.artifacts : [];
630
+ for (const artifactValue of artifacts) {
631
+ if (!isRecord3(artifactValue) || typeof artifactValue["path"] !== "string" || typeof artifactValue["sha256"] !== "string") {
632
+ failures.push("artifact requires string path and sha256");
633
+ continue;
634
+ }
635
+ const artifact = artifactValue;
636
+ const artifactPath = join(root, artifact.path);
637
+ if (!pathInside(root, artifactPath)) failures.push(`artifact path escapes project root: ${artifact.path}`);
638
+ else if (!existsSync(artifactPath) || sha256(readFileSync(artifactPath)) !== artifact.sha256) failures.push(`artifact hash mismatch: ${artifact.path}`);
639
+ if (check.category === "ui" && (artifact.type !== "screenshot" || !viewportValid(artifact.viewport))) failures.push(`UI artifact requires type=screenshot and viewport: ${artifact.path}`);
640
+ }
641
+ return failures;
642
+ };
643
+ var execFileAsync = promisify(execFile);
644
+ var git = async (root, args) => {
645
+ try {
646
+ return (await execFileAsync("git", ["-C", root, ...args], { encoding: "utf8" })).stdout.trim();
647
+ } catch {
648
+ return "";
649
+ }
650
+ };
651
+ var sourceSnapshot = async (root, stateDir) => {
652
+ const revision = await git(root, ["rev-parse", "HEAD"]);
653
+ const stateRelative = relative(root, stateDir).replaceAll("\\", "/");
654
+ const pathspec = ["--", "."];
655
+ if (stateRelative && stateRelative !== ".." && !stateRelative.startsWith("../")) pathspec.push(`:(exclude)${stateRelative}`);
656
+ const status = await git(root, ["status", "--porcelain=v1", "--untracked-files=all", ...pathspec]);
657
+ const diff = await git(root, ["diff", "--no-ext-diff", "--binary", "HEAD", ...pathspec]);
658
+ const untrackedPaths = (await git(root, ["ls-files", "--others", "--exclude-standard", "-z"])).split("\0").filter(Boolean).filter((path) => !stateRelative || path !== stateRelative && !path.startsWith(`${stateRelative}/`));
659
+ const untracked = untrackedPaths.flatMap((path) => {
660
+ const absolute = resolve(root, path);
661
+ try {
662
+ return lstatSync(absolute).isFile() ? [{ path, hash: sha256(readFileSync(absolute)) }] : [];
663
+ } catch {
664
+ return [];
665
+ }
666
+ });
667
+ const fingerprint = { revision, status, diff, untracked };
668
+ return { revision: revision || `content:${hashJson(fingerprint)}`, status, statusHash: hashJson(fingerprint) };
669
+ };
670
+
671
+ // src/verification.ts
672
+ var now2 = () => (/* @__PURE__ */ new Date()).toISOString();
673
+ var requireRun = (run) => run ?? fail("No verification run exists.", "NO_RUN");
674
+ var verificationProjection = (run) => ({ checks: run.checks, outcomes: run.outcomes, metrics: run.metrics });
675
+ var verificationDigest = (run) => hashJson(verificationProjection(run));
676
+ var runCommand = (check, cwd) => new Promise((resolveResult) => {
677
+ const started = Date.now();
678
+ const child = spawn(check.command, { cwd, shell: true, env: process.env });
679
+ let stdout = "";
680
+ let stderr = "";
681
+ let timedOut = false;
682
+ const timer = setTimeout(() => {
683
+ timedOut = true;
684
+ child.kill("SIGTERM");
685
+ }, check.timeoutMs);
686
+ child.stdout.on("data", (chunk) => {
687
+ stdout += chunk.toString();
688
+ });
689
+ child.stderr.on("data", (chunk) => {
690
+ stderr += chunk.toString();
691
+ });
692
+ child.on("close", (exitCode) => {
693
+ clearTimeout(timer);
694
+ resolveResult({ exitCode: exitCode ?? 1, timedOut, stdout, stderr, durationMs: Date.now() - started });
695
+ });
696
+ });
697
+ var currentBinding = async (loaded) => ({ source: await sourceSnapshot(loaded.root, loaded.stateDir), configHash: loaded.configHash });
698
+ var staleRun = (loaded, run, reason) => {
699
+ const stale = transition(run, "STALE", reason);
700
+ saveRun2(loaded.stateDir, stale);
701
+ setLatest(loaded.stateDir, stale);
702
+ return fail(reason, "STALE");
703
+ };
704
+ var isFresh = async (loaded, run) => {
705
+ const current = await currentBinding(loaded);
706
+ return current.configHash === run.configHash && current.source.revision === run.sourceRevision && current.source.statusHash === run.sourceStatusHash;
707
+ };
708
+ var planRun = async ({ configPath, decision, actor = "human", allowDirty = false, contextSnapshots = [] }) => {
709
+ const automatedPreparation = actor === "ci" && decision === "prepared";
710
+ if (!automatedPreparation) {
711
+ assertHuman(actor);
712
+ if (!approvedDecision(decision)) fail("Contract was not approved.", "CLARIFYING");
713
+ }
714
+ const loaded = loadConfig(configPath);
715
+ if (loaded.config.contract.ambiguities.length) fail(`Unresolved ambiguities remain: ${loaded.config.contract.ambiguities.join(" | ")}`, "CLARIFYING");
716
+ const validatedContextSnapshots = validateContextSnapshots(contextSnapshots);
717
+ const baseline = await sourceSnapshot(loaded.root, loaded.stateDir);
718
+ const configRelative = relative(loaded.root, loaded.absolute);
719
+ const meaningful = baseline.status.split("\n").filter(Boolean).filter((line) => !line.endsWith(` ${configRelative}`) && !line.endsWith(` ${configRelative.replaceAll("/", "\\")}`));
720
+ if (meaningful.length && !allowDirty) fail(`Worktree is dirty before planning:
721
+ ${meaningful.join("\n")}
722
+ Use --allow-dirty only with explicit human authorization.`, "WORKTREE_DIRTY");
723
+ const previous = loadLatestRun(loaded.stateDir);
724
+ if (previous && !["STALE", "SUPERSEDED"].includes(previous.state)) fail(`An active run already exists: ${previous.runId} (${previous.state}).`, "ACTIVE_RUN");
725
+ return createRun({ loaded, baseline, supersedes: previous?.runId, dirtyBaselineAuthorized: allowDirty, contextSnapshots: validatedContextSnapshots, planner: automatedPreparation ? "ci" : "human" });
726
+ };
727
+ var startRun = (loaded) => {
728
+ const run = requireRun(loadLatestRun(loaded.stateDir));
729
+ const next = transition(run, "IMPLEMENTING", "Implementation started.", "agent");
730
+ saveRun2(loaded.stateDir, next);
731
+ setLatest(loaded.stateDir, next);
732
+ return next;
733
+ };
734
+ var cancelRun = async ({ configPath, runId, reason = "Run cancelled by a human.", actor = "human" }) => {
735
+ assertHuman(actor);
736
+ const loaded = loadConfig(configPath);
737
+ const run = requireRun(runId ? readRun(loaded.stateDir, runId) : loadLatestRun(loaded.stateDir));
738
+ const next = transition(run, "CANCELLED", reason, "human");
739
+ saveRun2(loaded.stateDir, next);
740
+ setLatest(loaded.stateDir, next);
741
+ return next;
742
+ };
743
+ var verifyRun = async ({ configPath }) => {
744
+ const loaded = loadConfig(configPath);
745
+ const run = requireRun(loadLatestRun(loaded.stateDir));
746
+ if (!["IMPLEMENTING", "VERIFYING"].includes(run.state)) {
747
+ if (["AWAITING_HUMAN_APPROVAL", "AWAITING_AUTHORIZATION", "COMPLETE"].includes(run.state) && !await isFresh(loaded, run)) staleRun(loaded, run, "Run is stale because source or contract changed.");
748
+ fail(`Cannot verify from ${run.state}.`, "INVALID_STATE");
749
+ }
750
+ if (run.configHash !== loaded.configHash) staleRun(loaded, run, "Run is stale because the verification contract changed.");
751
+ const binding = await currentBinding(loaded);
752
+ let current = { ...transition(run, "VERIFYING", "Verification started.", "agent"), sourceRevision: binding.source.revision, sourceStatusHash: binding.source.statusHash };
753
+ saveRun2(loaded.stateDir, current);
754
+ const checkDir = join(loaded.stateDir, "runs", current.runId, "checks");
755
+ mkdirSync(checkDir, { recursive: true });
756
+ const outcomesByCheck = new Map(loaded.config.checks.map((check) => [check.id, loaded.config.contract.outcomes.filter((outcome) => outcome.checks.includes(check.id)).map((outcome) => outcome.id)]));
757
+ let totalDurationMs = 0;
758
+ for (const check of loaded.config.checks) {
759
+ const result = await runCommand(check, loaded.root);
760
+ totalDurationMs += result.durationMs;
761
+ const stdoutPath = join(checkDir, `${check.id}.stdout`);
762
+ const stderrPath = join(checkDir, `${check.id}.stderr`);
763
+ writeFileSync(stdoutPath, result.stdout, "utf8");
764
+ writeFileSync(stderrPath, result.stderr, "utf8");
765
+ const evidence = parseStructuredEvidence(result.stdout);
766
+ const failures = result.exitCode === 0 && !result.timedOut && evidence ? validateEvidence(loaded.root, check, evidence, outcomesByCheck.get(check.id) ?? []) : [result.timedOut ? "check timed out" : result.exitCode !== 0 ? `exit code ${result.exitCode}` : "missing final structured evidence"];
767
+ const nextCheck = { id: check.id, category: check.category, status: failures.length ? "failed" : "passed", exitCode: result.exitCode, durationMs: result.durationMs, ...evidence ? { evidence } : {}, ...failures.length ? { failures } : {} };
768
+ current = { ...current, evidenceReferences: [...current.evidenceReferences, { checkId: check.id, stdout: relative(loaded.stateDir, stdoutPath), stderr: relative(loaded.stateDir, stderrPath) }], checks: current.checks.map((item) => item.id === check.id ? nextCheck : item) };
769
+ saveRun2(loaded.stateDir, current);
770
+ }
771
+ const statuses = new Map(current.checks.map((check) => [check.id, check.status]));
772
+ const budgetExceeded = loaded.config.budget?.maxDurationMs !== void 0 && totalDurationMs > loaded.config.budget.maxDurationMs;
773
+ const allPassed = loaded.config.checks.every((check) => statuses.get(check.id) === "passed") && !budgetExceeded;
774
+ current = { ...current, outcomes: current.outcomes.map((outcome) => ({ ...outcome, status: outcome.checks.every((id2) => statuses.get(id2) === "passed") ? "passed" : "failed" })), metrics: { totalDurationMs, budgetExceeded } };
775
+ const digest2 = verificationDigest(current);
776
+ current = { ...current, verificationDigest: digest2 };
777
+ saveRun2(loaded.stateDir, current);
778
+ new FileEventStore(loaded.stateDir).append({ runId: current.runId, sourceRevision: current.sourceRevision, configHash: current.configHash, type: "verification.completed", payload: { verificationDigest: digest2, checkCount: current.checks.length, outcomeCount: current.outcomes.length, totalDurationMs, budgetExceeded } });
779
+ const nextState = allPassed ? "AWAITING_HUMAN_APPROVAL" : "BLOCKED";
780
+ current = { ...transition(current, nextState, allPassed ? "All configured checks passed; human approval is required." : budgetExceeded ? "Verification budget was exceeded." : "A configured check failed or lacked structured evidence.", "harness") };
781
+ saveRun2(loaded.stateDir, current);
782
+ setLatest(loaded.stateDir, current);
783
+ return current;
784
+ };
785
+ var assertFresh = async (loaded, run) => {
786
+ if (!await isFresh(loaded, run)) staleRun(loaded, run, "Run is stale because source or worktree changed after verification.");
787
+ };
788
+ var assertVerificationAttestation = (loaded, run) => {
789
+ const expected = verificationDigest(run);
790
+ if (run.verificationDigest !== expected) fail("Verification projection attestation does not match run.json.", "HARNESS_ERROR");
791
+ const event = new FileEventStore(loaded.stateDir).read(run.runId).filter((item) => item.type === "verification.completed").at(-1);
792
+ if (!event || event.payload.verificationDigest !== expected || event.sourceRevision !== run.sourceRevision || event.configHash !== run.configHash) fail("Verification projection attestation is missing from the event log.", "HARNESS_ERROR");
793
+ };
794
+ var plannerForRetry = (loaded, run) => {
795
+ if (run.contractPreparation) return "ci";
796
+ if (!run.supersedes) return "human";
797
+ return plannerForRetry(loaded, readRun(loaded.stateDir, run.supersedes));
798
+ };
799
+ var recordDecision = (loaded, run, type, payload) => {
800
+ new FileEventStore(loaded.stateDir).append({ runId: run.runId, sourceRevision: run.sourceRevision, configHash: run.configHash, type, payload: type === "authorization.recorded" ? { ...payload, target: payload.target ?? fail("tracking.target is required when authorizing.", "INVALID_CONFIG") } : payload });
801
+ };
802
+ var assertDecisionProjection = (run, decision, expectedState) => {
803
+ if (decision.decision !== "approved" || decision.resultingState !== expectedState || decision.verificationDigest !== run.verificationDigest || decision.sourceRevision !== run.sourceRevision || decision.contractHash !== run.contractHash) fail("Terminal decision attestation does not match the run projection.", "HARNESS_ERROR");
804
+ };
805
+ var reconcileRun = async ({ configPath, runId }) => {
806
+ const loaded = loadConfig(configPath);
807
+ const run = requireRun(runId ? readRun(loaded.stateDir, runId) : loadLatestRun(loaded.stateDir));
808
+ await assertFresh(loaded, run);
809
+ const store = new FileEventStore(loaded.stateDir);
810
+ const eventLog = store.verify(run.runId);
811
+ const events = store.read(run.runId);
812
+ if (events.some((event) => event.runId !== run.runId || event.configHash !== run.configHash)) fail("Run event log is not bound to the current run projection.", "HARNESS_ERROR");
813
+ const requiresVerification = ["AWAITING_HUMAN_APPROVAL", "AWAITING_AUTHORIZATION", "COMPLETE"].includes(run.state);
814
+ if (requiresVerification) {
815
+ if (eventLog.status !== "verified") fail("Terminal run requires a verified event log.", "HARNESS_ERROR");
816
+ assertVerificationAttestation(loaded, run);
817
+ }
818
+ if (run.state === "AWAITING_AUTHORIZATION" || run.state === "COMPLETE") {
819
+ const approval = events.filter((event) => event.type === "approval.recorded").at(-1) ?? fail("Terminal run is missing its human approval event.", "HARNESS_ERROR");
820
+ assertDecisionProjection(run, approval.payload, run.state === "COMPLETE" && !loaded.config.tracking.required ? "COMPLETE" : "AWAITING_AUTHORIZATION");
821
+ if (!run.humanApproval || run.humanApproval.actor !== "human" || run.humanApproval.verificationDigest !== run.verificationDigest || run.humanApproval.sourceRevision !== run.sourceRevision || run.humanApproval.contractHash !== run.contractHash) fail("Human approval projection is inconsistent with its audit event.", "HARNESS_ERROR");
822
+ }
823
+ if (run.state === "COMPLETE" && loaded.config.tracking.required) {
824
+ const authorization = events.filter((event) => event.type === "authorization.recorded").at(-1) ?? fail("Complete tracked run is missing its authorization event.", "HARNESS_ERROR");
825
+ assertDecisionProjection(run, authorization.payload, "COMPLETE");
826
+ if (!run.authorization || run.authorization.actor !== "human" || run.authorization.verificationDigest !== run.verificationDigest || run.authorization.target !== authorization.payload.target || run.authorization.sourceRevision !== run.sourceRevision || run.authorization.contractHash !== run.contractHash) fail("Authorization projection is inconsistent with its audit event.", "HARNESS_ERROR");
827
+ }
828
+ return { status: "verified", runId: run.runId, state: run.state, eventCount: eventLog.eventCount, ...eventLog.headHash ? { headHash: eventLog.headHash } : {}, ...run.verificationDigest ? { verificationDigest: run.verificationDigest } : {} };
829
+ };
830
+ var approveRun = async ({ configPath, runId, decision, actor = "human" }) => {
831
+ assertHuman(actor);
832
+ const loaded = loadConfig(configPath);
833
+ const run = requireRun(runId ? readRun(loaded.stateDir, runId) : loadLatestRun(loaded.stateDir));
834
+ if (run.state !== "AWAITING_HUMAN_APPROVAL") fail(`Cannot approve from ${run.state}.`, "INVALID_STATE");
835
+ await assertFresh(loaded, run);
836
+ assertVerificationAttestation(loaded, run);
837
+ if (!approvedDecision(decision)) {
838
+ const blocked = transition(run, "BLOCKED", "Human rejected the verification result.", "human");
839
+ saveRun2(loaded.stateDir, blocked);
840
+ recordDecision(loaded, run, "approval.recorded", { decision: "rejected", resultingState: blocked.state, verificationDigest: run.verificationDigest, actor: "human", sourceRevision: run.sourceRevision, contractHash: run.contractHash });
841
+ setLatest(loaded.stateDir, blocked);
842
+ return blocked;
843
+ }
844
+ const nextState = loaded.config.tracking.required ? "AWAITING_AUTHORIZATION" : "COMPLETE";
845
+ const next = { ...transition(run, nextState, "Human approved the verification result.", "human"), humanApproval: { actor: "human", at: now2(), sourceRevision: run.sourceRevision, contractHash: run.contractHash, verificationDigest: run.verificationDigest } };
846
+ saveRun2(loaded.stateDir, next);
847
+ recordDecision(loaded, run, "approval.recorded", { decision: "approved", resultingState: nextState, verificationDigest: run.verificationDigest, actor: "human", sourceRevision: run.sourceRevision, contractHash: run.contractHash });
848
+ setLatest(loaded.stateDir, next);
849
+ return next;
850
+ };
851
+ var authorizeRun = async ({ configPath, runId, decision, actor = "human" }) => {
852
+ assertHuman(actor);
853
+ const loaded = loadConfig(configPath);
854
+ const run = requireRun(runId ? readRun(loaded.stateDir, runId) : loadLatestRun(loaded.stateDir));
855
+ if (run.state !== "AWAITING_AUTHORIZATION") fail(`Cannot authorize from ${run.state}.`, "INVALID_STATE");
856
+ await assertFresh(loaded, run);
857
+ assertVerificationAttestation(loaded, run);
858
+ if (!approvedDecision(decision)) {
859
+ const blocked = transition(run, "BLOCKED", "Human rejected external tracking authorization.", "human");
860
+ saveRun2(loaded.stateDir, blocked);
861
+ recordDecision(loaded, run, "authorization.recorded", { decision: "rejected", resultingState: blocked.state, verificationDigest: run.verificationDigest, actor: "human", target: loaded.config.tracking.target ?? "", sourceRevision: run.sourceRevision, contractHash: run.contractHash });
862
+ setLatest(loaded.stateDir, blocked);
863
+ return blocked;
864
+ }
865
+ if (!loaded.config.tracking.target) fail("tracking.target is required when authorizing.", "INVALID_CONFIG");
866
+ const next = { ...transition(run, "COMPLETE", "External tracking was authorized.", "human"), authorization: { actor: "human", at: now2(), target: loaded.config.tracking.target, sourceRevision: run.sourceRevision, contractHash: run.contractHash, verificationDigest: run.verificationDigest } };
867
+ saveRun2(loaded.stateDir, next);
868
+ recordDecision(loaded, run, "authorization.recorded", { decision: "approved", resultingState: "COMPLETE", verificationDigest: run.verificationDigest, actor: "human", target: loaded.config.tracking.target, sourceRevision: run.sourceRevision, contractHash: run.contractHash });
869
+ setLatest(loaded.stateDir, next);
870
+ return next;
871
+ };
872
+ var retryRun = async ({ configPath }) => {
873
+ const loaded = loadConfig(configPath);
874
+ const previous = loadLatestRun(loaded.stateDir);
875
+ const previousRun = requireRun(previous);
876
+ if (!["BLOCKED", "STALE", "CANCELLED"].includes(previousRun.state)) fail(`Cannot retry from ${previousRun.state}.`, "INVALID_STATE");
877
+ const baseline = await sourceSnapshot(loaded.root, loaded.stateDir);
878
+ const superseded = transition(previousRun, "SUPERSEDED", "Retry superseded the previous run.", "harness");
879
+ saveRun2(loaded.stateDir, superseded);
880
+ const run = await createRun({ loaded, baseline, supersedes: previousRun.runId, dirtyBaselineAuthorized: previousRun.dirtyBaselineAuthorized, planner: plannerForRetry(loaded, previousRun) });
881
+ const next = transition(run, "IMPLEMENTING", "Retry started after a previous attempt.", "agent");
882
+ saveRun2(loaded.stateDir, next);
883
+ setLatest(loaded.stateDir, next);
884
+ return next;
885
+ };
886
+ var cleanTaskArtifacts = (configPath) => cleanConfiguredArtifacts(loadConfig(configPath));
887
+ var index = (root, indexPath) => JSON.parse(readFileSync(resolve(root, indexPath), "utf8"));
888
+ var text = (entry) => [entry.id, entry.type, entry.title, entry.path, entry.description, entry.body, ...Array.isArray(entry.tags) ? entry.tags : []].filter((value) => typeof value === "string").join(" ").toLowerCase();
889
+ var sourceHash = (document) => typeof document.contentHash === "string" && document.contentHash.length > 0 ? document.contentHash : hashJson(document);
890
+ var matches = (entry, query) => {
891
+ const needle = query.query.trim().toLowerCase();
892
+ const scopes = query.scope?.map((scope) => scope.toLowerCase()) ?? [];
893
+ const value = text(entry);
894
+ return Boolean(needle && value.includes(needle) && (scopes.length === 0 || scopes.some((scope) => value.includes(scope))));
895
+ };
896
+ var createDocBridgeContextProvider = ({ root, indexPath = ".doc-bridge/index.json" }) => ({
897
+ id: "doc-bridge",
898
+ version: "1.0.0",
899
+ resolve: async (query) => {
900
+ const document = index(root, indexPath);
901
+ const contentHash = sourceHash(document);
902
+ const entries = Array.isArray(document.knowledge) ? document.knowledge.filter((value) => typeof value === "object" && value !== null && !Array.isArray(value)).filter((entry) => matches(entry, query)).sort((left, right) => String(left.id ?? "").localeCompare(String(right.id ?? ""))).slice(0, 8) : [];
903
+ const references = entries.flatMap((entry) => typeof entry.id === "string" && typeof entry.path === "string" ? [{ id: entry.id, uri: `doc-bridge://${entry.path}`, ...typeof entry.title === "string" ? { title: entry.title } : {}, contentHash }] : []);
904
+ return { providerId: "doc-bridge", query, references, sourceHash: contentHash, snapshotHash: hashContextSnapshot({ providerId: "doc-bridge", query, references, sourceHash: contentHash }), resolvedAt: (/* @__PURE__ */ new Date()).toISOString() };
905
+ }
906
+ });
907
+ var BENCHMARK_SCHEMA_VERSION = 1;
908
+ var percentage = (part, total) => total ? Number((part / total).toFixed(4)) : null;
909
+ var DEFAULT_POLICY = { minComparableTasks: 3, maxDurationRegressionRate: 0.2, minCompletedRunsPerTask: 3, minBaselineSamplesPerTask: 3, requireZeroEscapedIncomplete: true };
910
+ var improvementRate = (baseline, current) => baseline === void 0 || current === void 0 || baseline === 0 ? null : Number(((baseline - current) / baseline).toFixed(4));
911
+ var increaseRate = (baseline, current) => baseline === void 0 || current === void 0 || baseline === 0 ? null : Number(((current - baseline) / baseline).toFixed(4));
912
+ var increaseDelta = (baseline, current) => baseline === void 0 || current === null ? null : Number((current - baseline).toFixed(4));
913
+ var increaseDirection = (rate2, delta) => delta === null ? improvementDirection(rate2) : delta > 0 ? "improved" : delta < 0 ? "regressed" : "unchanged";
914
+ var improvementDirection = (rate2) => rate2 === null ? "unavailable" : rate2 > 0 ? "improved" : rate2 < 0 ? "regressed" : "unchanged";
915
+ var count = (items, predicate) => items.filter(predicate).length;
916
+ var median = (values) => {
917
+ if (!values.length) return null;
918
+ const sorted = [...values].sort((left, right) => left - right);
919
+ const middle = Math.floor(sorted.length / 2);
920
+ return sorted.length % 2 ? sorted[middle] ?? null : ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2;
921
+ };
922
+ var reviewMinutes = (run) => {
923
+ if (!run.humanApproval) return void 0;
924
+ const reviewStart = run.transitions.find((transition2) => transition2.to === "AWAITING_HUMAN_APPROVAL")?.at;
925
+ if (!reviewStart) return void 0;
926
+ const elapsed = Date.parse(run.humanApproval.at) - Date.parse(reviewStart);
927
+ return Number.isFinite(elapsed) && elapsed >= 0 ? Number((elapsed / 6e4).toFixed(2)) : void 0;
928
+ };
929
+ var confidence = (comparable, completedRuns, policy) => !comparable ? "insufficient" : completedRuns >= policy.minCompletedRunsPerTask ? "reliable" : "directional";
930
+ var artifactAcceptanceRate = (run) => {
931
+ const rates = run.checks.flatMap((check) => {
932
+ const evidence = check.evidence;
933
+ if (!evidence) return [];
934
+ const direct = typeof evidence["artifactAcceptanceRate"] === "number" ? [evidence["artifactAcceptanceRate"]] : [];
935
+ const reports = Array.isArray(evidence["reports"]) ? evidence["reports"].flatMap((report) => typeof report === "object" && report !== null && typeof report["artifactAcceptanceRate"] === "number" ? [report["artifactAcceptanceRate"]] : []) : [];
936
+ return [...direct, ...reports].filter((rate2) => Number.isFinite(rate2) && rate2 >= 0 && rate2 <= 1);
937
+ });
938
+ return rates.length ? Number((rates.reduce((total, rate2) => total + rate2, 0) / rates.length).toFixed(4)) : void 0;
939
+ };
940
+ var projectRun = (run) => {
941
+ const checks = { total: run.checks.length, passed: count(run.checks, (check) => check.status === "passed"), failed: count(run.checks, (check) => check.status === "failed") };
942
+ const outcomes = { total: run.outcomes.length, passed: count(run.outcomes, (outcome) => outcome.status === "passed"), failed: count(run.outcomes, (outcome) => outcome.status === "failed") };
943
+ const evidence = { total: run.checks.length, attached: count(run.checks, (check) => check.evidence !== void 0) };
944
+ const acceptanceRate = artifactAcceptanceRate(run);
945
+ const humanReviewMinutes = reviewMinutes(run);
946
+ const escapedIncomplete = run.state === "COMPLETE" && checks.failed === 0 && outcomes.failed === 0 && evidence.attached === evidence.total ? 0 : void 0;
947
+ return { runId: run.runId, state: run.state, sourceRevision: run.sourceRevision, configHash: run.configHash, contractHash: run.contractHash, ...run.supersedes ? { supersedes: run.supersedes } : {}, ...run.metrics ? { durationMs: run.metrics.totalDurationMs } : {}, checks, outcomes, evidence, ...acceptanceRate === void 0 ? {} : { artifactAcceptanceRate: acceptanceRate }, ...escapedIncomplete === void 0 ? {} : { escapedIncomplete }, humanApproved: run.humanApproval !== void 0, ...humanReviewMinutes === void 0 ? {} : { humanReviewMinutes }, authorized: run.authorization !== void 0, ...run.benchmark ? { benchmark: run.benchmark } : {} };
948
+ };
949
+ var summarize = (runs) => {
950
+ const stateCounts = Object.fromEntries(RUN_STATES.map((state) => [state, count(runs, (run) => run.state === state)]));
951
+ const checksTotal = runs.reduce((total, run) => total + run.checks.total, 0);
952
+ const checksPassed = runs.reduce((total, run) => total + run.checks.passed, 0);
953
+ const outcomesTotal = runs.reduce((total, run) => total + run.outcomes.total, 0);
954
+ const outcomesPassed = runs.reduce((total, run) => total + run.outcomes.passed, 0);
955
+ const evidenceTotal = runs.reduce((total, run) => total + run.evidence.total, 0);
956
+ const evidenceAttached = runs.reduce((total, run) => total + run.evidence.attached, 0);
957
+ const firstAttempts = runs.filter((run) => !run.supersedes);
958
+ const superseded = new Set(runs.flatMap((run) => run.supersedes ? [run.supersedes] : []));
959
+ const effectiveRuns = runs.filter((run) => !superseded.has(run.runId));
960
+ const effectiveChecksTotal = effectiveRuns.reduce((total, run) => total + run.checks.total, 0);
961
+ const effectiveChecksPassed = effectiveRuns.reduce((total, run) => total + run.checks.passed, 0);
962
+ const effectiveOutcomesTotal = effectiveRuns.reduce((total, run) => total + run.outcomes.total, 0);
963
+ const effectiveOutcomesPassed = effectiveRuns.reduce((total, run) => total + run.outcomes.passed, 0);
964
+ const effectiveEvidenceTotal = effectiveRuns.reduce((total, run) => total + run.evidence.total, 0);
965
+ const effectiveEvidenceAttached = effectiveRuns.reduce((total, run) => total + run.evidence.attached, 0);
966
+ const durations = runs.flatMap((run) => run.durationMs === void 0 ? [] : [run.durationMs]);
967
+ return {
968
+ totalRuns: runs.length,
969
+ stateCounts,
970
+ completeRuns: stateCounts.COMPLETE,
971
+ retriedRuns: count(runs, (run) => run.supersedes !== void 0),
972
+ staleRuns: stateCounts.STALE,
973
+ firstAttemptRuns: firstAttempts.length,
974
+ humanApprovedRuns: count(runs, (run) => run.humanApproved),
975
+ authorizedRuns: count(runs, (run) => run.authorized),
976
+ effectiveRunCount: effectiveRuns.length,
977
+ effectiveCompleteRuns: count(effectiveRuns, (run) => run.state === "COMPLETE"),
978
+ effectiveCompletionRate: percentage(count(effectiveRuns, (run) => run.state === "COMPLETE"), effectiveRuns.length),
979
+ effectiveCheckPassRate: percentage(effectiveChecksPassed, effectiveChecksTotal),
980
+ effectiveOutcomePassRate: percentage(effectiveOutcomesPassed, effectiveOutcomesTotal),
981
+ effectiveEvidenceCoverageRate: percentage(effectiveEvidenceAttached, effectiveEvidenceTotal),
982
+ checkPassRate: percentage(checksPassed, checksTotal),
983
+ outcomePassRate: percentage(outcomesPassed, outcomesTotal),
984
+ evidenceCoverageRate: percentage(evidenceAttached, evidenceTotal),
985
+ firstAttemptApprovalRate: percentage(count(firstAttempts, (run) => run.humanApproved), firstAttempts.length),
986
+ retryRate: percentage(count(runs, (run) => run.supersedes !== void 0), runs.length),
987
+ staleRate: percentage(stateCounts.STALE, runs.length),
988
+ averageDurationMs: durations.length ? Math.round(durations.reduce((total, duration3) => total + duration3, 0) / durations.length) : null,
989
+ medianDurationMs: median(durations)
990
+ };
991
+ };
992
+ var readRuns = (stateDir) => {
993
+ const runsDir = join(stateDir, "runs");
994
+ if (!existsSync(runsDir)) return [];
995
+ return readdirSync(runsDir, { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => {
996
+ try {
997
+ const candidate = readJson(join(runsDir, entry.name, "run.json"));
998
+ return candidate.type === "agentskit-harness-run" ? readRun(stateDir, entry.name) : void 0;
999
+ } catch (error) {
1000
+ return fail(`Benchmark could not read run ${entry.name}: ${error instanceof Error ? error.message : String(error)}`, "HARNESS_ERROR");
1001
+ }
1002
+ }).filter((run) => run !== void 0);
1003
+ };
1004
+ var nonEmptyString = (value, label) => {
1005
+ if (typeof value !== "string") return fail(`${label} must be a non-empty string.`, "INVALID_CONFIG");
1006
+ const result = value.trim();
1007
+ if (!result) return fail(`${label} must be a non-empty string.`, "INVALID_CONFIG");
1008
+ return result;
1009
+ };
1010
+ var sha2562 = (value, label) => {
1011
+ if (value === void 0) return void 0;
1012
+ const result = nonEmptyString(value, label);
1013
+ if (!/^[a-f0-9]{64}$/.test(result)) return fail(`${label} must be a lowercase SHA-256 digest.`, "INVALID_CONFIG");
1014
+ return result;
1015
+ };
1016
+ var stringList = (value, label) => {
1017
+ if (!Array.isArray(value)) return fail(`${label} must be a non-empty string array.`, "INVALID_CONFIG");
1018
+ const items = value;
1019
+ if (!items.length || !items.every((item) => typeof item === "string" && Boolean(item.trim()))) return fail(`${label} must be a non-empty string array.`, "INVALID_CONFIG");
1020
+ return items.map((item) => String(item).trim());
1021
+ };
1022
+ var nonNegativeNumber = (value, label) => {
1023
+ if (value === void 0) return void 0;
1024
+ if (typeof value !== "number") return fail(`${label} must be a non-negative number.`, "INVALID_CONFIG");
1025
+ if (!Number.isFinite(value) || value < 0) return fail(`${label} must be a non-negative number.`, "INVALID_CONFIG");
1026
+ const result = value;
1027
+ return result;
1028
+ };
1029
+ var nonNegativeInteger = (value, label) => {
1030
+ const result = nonNegativeNumber(value, label);
1031
+ if (result !== void 0 && !Number.isInteger(result)) return fail(`${label} must be an integer.`, "INVALID_CONFIG");
1032
+ return result;
1033
+ };
1034
+ var rate = (value, label) => {
1035
+ const result = nonNegativeNumber(value, label);
1036
+ if (result !== void 0 && result > 1) return fail(`${label} must be between 0 and 1.`, "INVALID_CONFIG");
1037
+ return result;
1038
+ };
1039
+ var timestamp = (value, label) => {
1040
+ const result = nonEmptyString(value, label);
1041
+ if (!Number.isFinite(Date.parse(result))) return fail(`${label} must be a valid timestamp.`, "INVALID_CONFIG");
1042
+ return result;
1043
+ };
1044
+ var relativePath = (value, label) => {
1045
+ const result = nonEmptyString(value, label);
1046
+ if (result.startsWith("/") || result.split("/").includes("..")) return fail(`${label} must be a repository-relative path.`, "INVALID_CONFIG");
1047
+ return result;
1048
+ };
1049
+ var taskFile = (value, label) => {
1050
+ if (value === void 0) return void 0;
1051
+ if (typeof value !== "object" || value === null || Array.isArray(value)) return fail(`${label} must be an object.`, "INVALID_CONFIG");
1052
+ const raw = value;
1053
+ return { path: relativePath(raw["path"], `${label}.path`), sha256: sha2562(raw["sha256"], `${label}.sha256`) ?? fail(`${label}.sha256 is required.`, "INVALID_CONFIG") };
1054
+ };
1055
+ var taskSource = (value, label) => {
1056
+ if (value === void 0) return void 0;
1057
+ if (typeof value !== "object" || value === null || Array.isArray(value)) return fail(`${label} must be an object.`, "INVALID_CONFIG");
1058
+ const raw = value;
1059
+ return { repository: nonEmptyString(raw["repository"], `${label}.repository`), path: relativePath(raw["path"], `${label}.path`), revision: nonEmptyString(raw["revision"], `${label}.revision`) };
1060
+ };
1061
+ var suiteSource = (value, label) => {
1062
+ if (value === void 0) return void 0;
1063
+ if (typeof value !== "object" || value === null || Array.isArray(value)) return fail(`${label} must be an object.`, "INVALID_CONFIG");
1064
+ const raw = value;
1065
+ return { repository: nonEmptyString(raw["repository"], `${label}.repository`), revision: nonEmptyString(raw["revision"], `${label}.revision`), taskDefinition: relativePath(raw["taskDefinition"], `${label}.taskDefinition`) };
1066
+ };
1067
+ var taskScope = (value, label) => {
1068
+ if (value === void 0) return void 0;
1069
+ if (typeof value !== "object" || value === null || Array.isArray(value)) return fail(`${label} must be an object.`, "INVALID_CONFIG");
1070
+ const raw = value;
1071
+ return { read: stringList(raw["read"], `${label}.read`), write: stringList(raw["write"], `${label}.write`) };
1072
+ };
1073
+ var taskSurfaces = (value, label) => {
1074
+ if (value === void 0) return void 0;
1075
+ if (!Array.isArray(value) || !value.length) return fail(`${label} must be a non-empty array.`, "INVALID_CONFIG");
1076
+ const surfaces = value.map((item, index2) => nonEmptyString(item, `${label}[${index2}]`));
1077
+ if (surfaces.some((surface2) => !SURFACE_NAMES.includes(surface2))) fail(`${label} contains an unknown surface.`, "INVALID_CONFIG");
1078
+ if (new Set(surfaces).size !== surfaces.length) fail(`${label} must contain unique surfaces.`, "INVALID_CONFIG");
1079
+ return surfaces;
1080
+ };
1081
+ var benchmarkPolicy = (value) => {
1082
+ if (value === void 0) return void 0;
1083
+ if (typeof value !== "object" || value === null || Array.isArray(value)) return fail("benchmark.policy must be an object.", "INVALID_CONFIG");
1084
+ const raw = value;
1085
+ const minComparableTasks = nonNegativeInteger(raw["minComparableTasks"], "benchmark.policy.minComparableTasks");
1086
+ const maxDurationRegressionRate = nonNegativeNumber(raw["maxDurationRegressionRate"], "benchmark.policy.maxDurationRegressionRate");
1087
+ const minCompletedRunsPerTask = nonNegativeInteger(raw["minCompletedRunsPerTask"], "benchmark.policy.minCompletedRunsPerTask");
1088
+ const minBaselineSamplesPerTask = nonNegativeInteger(raw["minBaselineSamplesPerTask"] ?? 1, "benchmark.policy.minBaselineSamplesPerTask");
1089
+ if (minComparableTasks === void 0 || minComparableTasks < 1) return fail("benchmark.policy.minComparableTasks must be at least 1.", "INVALID_CONFIG");
1090
+ if (maxDurationRegressionRate === void 0 || maxDurationRegressionRate > 1) return fail("benchmark.policy.maxDurationRegressionRate must be between 0 and 1.", "INVALID_CONFIG");
1091
+ if (minCompletedRunsPerTask === void 0 || minCompletedRunsPerTask < 1) return fail("benchmark.policy.minCompletedRunsPerTask must be at least 1.", "INVALID_CONFIG");
1092
+ if (minBaselineSamplesPerTask === void 0 || minBaselineSamplesPerTask < 1) return fail("benchmark.policy.minBaselineSamplesPerTask must be at least 1.", "INVALID_CONFIG");
1093
+ if (typeof raw["requireZeroEscapedIncomplete"] !== "boolean") return fail("benchmark.policy.requireZeroEscapedIncomplete must be boolean.", "INVALID_CONFIG");
1094
+ return { minComparableTasks, maxDurationRegressionRate, minCompletedRunsPerTask, minBaselineSamplesPerTask, requireZeroEscapedIncomplete: raw["requireZeroEscapedIncomplete"] };
1095
+ };
1096
+ var validateBenchmarkManifest = (value) => {
1097
+ if (typeof value !== "object" || value === null || Array.isArray(value)) fail("benchmark manifest must be an object.", "INVALID_CONFIG");
1098
+ const raw = value;
1099
+ if (raw["type"] !== "agentskit-harness-benchmark-manifest" || raw["schemaVersion"] !== BENCHMARK_SCHEMA_VERSION) fail("benchmark manifest type or schemaVersion is invalid.", "INVALID_CONFIG");
1100
+ const rawTasks = Array.isArray(raw["tasks"]) && raw["tasks"].length ? raw["tasks"] : fail("benchmark manifest tasks must be non-empty.", "INVALID_CONFIG");
1101
+ const tasks = rawTasks.map((item, index2) => {
1102
+ if (typeof item !== "object" || item === null || Array.isArray(item)) fail(`benchmark.tasks[${index2}] must be an object.`, "INVALID_CONFIG");
1103
+ const task = item;
1104
+ const kind = task["kind"] === void 0 ? void 0 : nonEmptyString(task["kind"], `benchmark.tasks[${index2}].kind`);
1105
+ const prompt = taskFile(task["prompt"], `benchmark.tasks[${index2}].prompt`);
1106
+ const source = taskSource(task["source"], `benchmark.tasks[${index2}].source`);
1107
+ const scope = taskScope(task["scope"], `benchmark.tasks[${index2}].scope`);
1108
+ const surfaces = taskSurfaces(task["surfaces"], `benchmark.tasks[${index2}].surfaces`);
1109
+ return { id: nonEmptyString(task["id"], `benchmark.tasks[${index2}].id`), title: nonEmptyString(task["title"], `benchmark.tasks[${index2}].title`), acceptanceCriteria: stringList(task["acceptanceCriteria"], `benchmark.tasks[${index2}].acceptanceCriteria`), ...surfaces === void 0 ? {} : { surfaces }, ...kind === void 0 ? {} : { kind }, ...prompt === void 0 ? {} : { prompt }, ...source === void 0 ? {} : { source }, ...scope === void 0 ? {} : { scope } };
1110
+ });
1111
+ if (new Set(tasks.map((task) => task.id)).size !== tasks.length) fail("benchmark task ids must be unique.", "INVALID_CONFIG");
1112
+ const taskIds = new Set(tasks.map((task) => task.id));
1113
+ const rawObservations = raw["observations"] === void 0 ? [] : Array.isArray(raw["observations"]) ? raw["observations"] : fail("benchmark.observations must be an array.", "INVALID_CONFIG");
1114
+ const observations = rawObservations.map((item, index2) => {
1115
+ if (typeof item !== "object" || item === null || Array.isArray(item)) fail(`benchmark.observations[${index2}] must be an object.`, "INVALID_CONFIG");
1116
+ const observation = item;
1117
+ const status = observation["status"];
1118
+ if (!["passed", "failed", "blocked", "not-run"].includes(String(status))) fail(`benchmark.observations[${index2}].status is invalid.`, "INVALID_CONFIG");
1119
+ const taskId = nonEmptyString(observation["taskId"], `benchmark.observations[${index2}].taskId`);
1120
+ if (!taskIds.has(taskId)) fail(`benchmark observation references unknown task: ${taskId}.`, "INVALID_CONFIG");
1121
+ const task = tasks.find((candidate) => candidate.id === taskId) ?? fail(`benchmark observation references unknown task: ${taskId}.`, "INVALID_CONFIG");
1122
+ const attempts = nonNegativeInteger(observation["attempts"], `benchmark.observations[${index2}].attempts`);
1123
+ const durationMs = nonNegativeNumber(observation["durationMs"], `benchmark.observations[${index2}].durationMs`);
1124
+ const durationSamplesMs = observation["durationSamplesMs"] === void 0 ? void 0 : Array.isArray(observation["durationSamplesMs"]) ? observation["durationSamplesMs"].map((sample, sampleIndex) => nonNegativeNumber(sample, `benchmark.observations[${index2}].durationSamplesMs[${sampleIndex}]`) ?? fail(`benchmark.observations[${index2}].durationSamplesMs must contain numbers.`, "INVALID_CONFIG")) : fail(`benchmark.observations[${index2}].durationSamplesMs must be an array.`, "INVALID_CONFIG");
1125
+ if (durationSamplesMs && !durationSamplesMs.length) fail(`benchmark.observations[${index2}].durationSamplesMs must not be empty.`, "INVALID_CONFIG");
1126
+ const artifactAcceptanceRate2 = rate(observation["artifactAcceptanceRate"], `benchmark.observations[${index2}].artifactAcceptanceRate`);
1127
+ const protocolCompletionRate = rate(observation["protocolCompletionRate"], `benchmark.observations[${index2}].protocolCompletionRate`);
1128
+ const reviewMinutes2 = nonNegativeNumber(observation["reviewMinutes"], `benchmark.observations[${index2}].reviewMinutes`);
1129
+ const escapedIncomplete = nonNegativeInteger(observation["escapedIncomplete"], `benchmark.observations[${index2}].escapedIncomplete`);
1130
+ const evidenceDigest = sha2562(observation["evidenceDigest"], `benchmark.observations[${index2}].evidenceDigest`);
1131
+ const rawEvidence = observation["evidence"] === void 0 ? void 0 : Array.isArray(observation["evidence"]) ? observation["evidence"] : fail(`benchmark.observations[${index2}].evidence must be an array.`, "INVALID_CONFIG");
1132
+ const evidence = rawEvidence?.map((item2, evidenceIndex) => {
1133
+ if (typeof item2 !== "object" || item2 === null || Array.isArray(item2)) fail(`benchmark.observations[${index2}].evidence[${evidenceIndex}] must be an object.`, "INVALID_CONFIG");
1134
+ const entry = item2;
1135
+ const criterion = nonEmptyString(entry["criterion"], `benchmark.observations[${index2}].evidence[${evidenceIndex}].criterion`);
1136
+ if (!task.acceptanceCriteria.includes(criterion)) fail(`benchmark evidence references unknown criterion: ${criterion}.`, "INVALID_CONFIG");
1137
+ const evidenceStatus = entry["status"];
1138
+ if (!["passed", "failed", "blocked", "not-run"].includes(String(evidenceStatus))) fail(`benchmark.observations[${index2}].evidence[${evidenceIndex}].status is invalid.`, "INVALID_CONFIG");
1139
+ return { criterion, status: evidenceStatus, source: nonEmptyString(entry["source"], `benchmark.observations[${index2}].evidence[${evidenceIndex}].source`) };
1140
+ });
1141
+ if (evidence && new Set(evidence.map((entry) => entry.criterion)).size !== evidence.length) fail(`benchmark.observations[${index2}].evidence criteria must be unique.`, "INVALID_CONFIG");
1142
+ return { taskId, mode: "baseline", status, source: nonEmptyString(observation["source"], `benchmark.observations[${index2}].source`), recordedAt: timestamp(observation["recordedAt"], `benchmark.observations[${index2}].recordedAt`), ...attempts === void 0 ? {} : { attempts }, ...durationMs === void 0 ? {} : { durationMs }, ...durationSamplesMs === void 0 ? {} : { durationSamplesMs }, ...artifactAcceptanceRate2 === void 0 ? {} : { artifactAcceptanceRate: artifactAcceptanceRate2 }, ...protocolCompletionRate === void 0 ? {} : { protocolCompletionRate }, ...reviewMinutes2 === void 0 ? {} : { reviewMinutes: reviewMinutes2 }, ...escapedIncomplete === void 0 ? {} : { escapedIncomplete }, ...evidence === void 0 ? {} : { evidence }, ...evidenceDigest === void 0 ? {} : { evidenceDigest } };
1143
+ });
1144
+ if (new Set(observations.map((observation) => observation.taskId)).size !== observations.length) fail("benchmark allows at most one baseline observation per task.", "INVALID_CONFIG");
1145
+ const provenance = suiteSource(raw["provenance"], "benchmark.provenance");
1146
+ const policy = benchmarkPolicy(raw["policy"]);
1147
+ return { type: "agentskit-harness-benchmark-manifest", schemaVersion: BENCHMARK_SCHEMA_VERSION, suiteId: nonEmptyString(raw["suiteId"], "benchmark.suiteId"), name: nonEmptyString(raw["name"], "benchmark.name"), ...provenance === void 0 ? {} : { provenance }, tasks, observations, ...policy === void 0 ? {} : { policy } };
1148
+ };
1149
+ var loadBenchmarkManifest = (path) => {
1150
+ try {
1151
+ return validateBenchmarkManifest(JSON.parse(readFileSync(path, "utf8")));
1152
+ } catch (error) {
1153
+ if (error instanceof SyntaxError) return fail(`Invalid benchmark manifest JSON: ${error.message}`, "INVALID_CONFIG");
1154
+ throw error;
1155
+ }
1156
+ };
1157
+ var recordBenchmarkObservation = (path, input) => {
1158
+ const originalContent = readFileSync(path, "utf8");
1159
+ const manifest = loadBenchmarkManifest(path);
1160
+ const taskId = nonEmptyString(input.taskId, "benchmark observation.taskId");
1161
+ if (!manifest.tasks.some((task) => task.id === taskId)) fail(`benchmark observation references unknown task: ${taskId}.`, "INVALID_INPUT");
1162
+ if (manifest.observations.some((observation2) => observation2.taskId === taskId)) fail(`benchmark already has an observation for task: ${taskId}.`, "INVALID_INPUT");
1163
+ const observation = validateBenchmarkManifest({
1164
+ ...manifest,
1165
+ observations: [...manifest.observations, {
1166
+ taskId,
1167
+ mode: "baseline",
1168
+ status: input.status,
1169
+ source: input.source,
1170
+ recordedAt: input.recordedAt ?? (/* @__PURE__ */ new Date()).toISOString(),
1171
+ ...input.attempts === void 0 ? {} : { attempts: input.attempts },
1172
+ ...input.durationMs === void 0 ? {} : { durationMs: input.durationMs },
1173
+ ...input.durationSamplesMs === void 0 ? {} : { durationSamplesMs: input.durationSamplesMs },
1174
+ ...input.artifactAcceptanceRate === void 0 ? {} : { artifactAcceptanceRate: input.artifactAcceptanceRate },
1175
+ ...input.protocolCompletionRate === void 0 ? {} : { protocolCompletionRate: input.protocolCompletionRate },
1176
+ ...input.reviewMinutes === void 0 ? {} : { reviewMinutes: input.reviewMinutes },
1177
+ ...input.escapedIncomplete === void 0 ? {} : { escapedIncomplete: input.escapedIncomplete },
1178
+ ...input.evidence === void 0 ? {} : { evidence: input.evidence },
1179
+ ...input.evidenceDigest === void 0 ? {} : { evidenceDigest: input.evidenceDigest }
1180
+ }]
1181
+ });
1182
+ const temporaryRoot = mkdtempSync(join(tmpdir(), "agentskit-harness-baseline-"));
1183
+ const temporaryPath = join(temporaryRoot, "manifest.json");
1184
+ try {
1185
+ writeFileSync(temporaryPath, `${JSON.stringify(observation, null, 2)}
1186
+ `, "utf8");
1187
+ if (readFileSync(path, "utf8") !== originalContent) fail("benchmark manifest changed while recording an observation.", "STALE");
1188
+ renameSync(temporaryPath, path);
1189
+ } finally {
1190
+ rmSync(temporaryRoot, { recursive: true, force: true });
1191
+ }
1192
+ return observation;
1193
+ };
1194
+ var comparisons = (runs, manifest, policy) => manifest.tasks.map((task) => {
1195
+ const taskRuns = runs.filter((run) => run.benchmark?.suiteId === manifest.suiteId && run.benchmark.taskId === task.id);
1196
+ const latest = taskRuns.at(-1);
1197
+ const baseline = manifest.observations.find((observation) => observation.taskId === task.id);
1198
+ const coveredCriteria = new Set((baseline?.evidence ?? []).map((entry) => entry.criterion));
1199
+ const baselineEvidenceCoverageRate = baseline ? percentage(coveredCriteria.size, task.acceptanceCriteria.length) : null;
1200
+ const baselineEvidenceComplete = baselineEvidenceCoverageRate === 1;
1201
+ const baselineEvidencePassed = baselineEvidenceComplete && (baseline?.evidence?.every((entry) => entry.status === "passed") ?? false);
1202
+ const baselineDeliveryComplete = baseline?.status === "passed" && baselineEvidencePassed;
1203
+ const baselineDurationSamples = baseline?.durationSamplesMs ?? (baseline?.durationMs === void 0 ? [] : [baseline.durationMs]);
1204
+ const baselineMedianDurationMs = median(baselineDurationSamples);
1205
+ const baselineSamplesSufficient = baselineDurationSamples.length >= policy.minBaselineSamplesPerTask;
1206
+ const comparable = baseline !== void 0 && baseline.status !== "not-run" && baselineEvidenceComplete && baselineDeliveryComplete && baselineSamplesSufficient && latest?.state === "COMPLETE";
1207
+ const comparability = comparable ? "comparable" : baseline === void 0 ? "missing-baseline" : baseline.status === "not-run" ? "baseline-not-run" : !baselineEvidenceComplete ? "baseline-evidence-missing" : latest === void 0 ? "harness-not-run" : latest.state !== "COMPLETE" ? "harness-not-complete" : !baselineDeliveryComplete ? "baseline-incomplete" : "baseline-samples-insufficient";
1208
+ const completedTaskRuns = taskRuns.filter((run) => run.state === "COMPLETE");
1209
+ const durationSamplesMs = completedTaskRuns.flatMap((run) => run.durationMs === void 0 ? [] : [run.durationMs]);
1210
+ const medianDurationMs = median(durationSamplesMs);
1211
+ const retryCount = count(taskRuns, (run) => run.supersedes !== void 0);
1212
+ const attempts = retryCount + (taskRuns.length ? 1 : 0);
1213
+ const durationRate = comparable ? improvementRate(baselineMedianDurationMs ?? void 0, medianDurationMs ?? void 0) : null;
1214
+ const attemptsRate = comparable ? improvementRate(baseline?.attempts, attempts) : null;
1215
+ const reviewRate = comparable ? improvementRate(baseline?.reviewMinutes, latest?.humanReviewMinutes) : null;
1216
+ const acceptanceSamples = taskRuns.flatMap((run) => run.artifactAcceptanceRate === void 0 ? [] : [run.artifactAcceptanceRate]);
1217
+ const harnessArtifactAcceptanceRate = acceptanceSamples.length ? Number((acceptanceSamples.reduce((total, rate2) => total + rate2, 0) / acceptanceSamples.length).toFixed(4)) : null;
1218
+ const artifactAcceptanceImprovementRate = increaseRate(baseline?.artifactAcceptanceRate, harnessArtifactAcceptanceRate ?? void 0);
1219
+ const artifactAcceptanceDelta = increaseDelta(baseline?.artifactAcceptanceRate, harnessArtifactAcceptanceRate);
1220
+ const completedRuns = completedTaskRuns.length;
1221
+ const harnessProtocolCompletionRate = taskRuns.length ? percentage(completedRuns, taskRuns.length) : null;
1222
+ const protocolCompletionImprovementRate = increaseRate(baseline?.protocolCompletionRate, harnessProtocolCompletionRate ?? void 0);
1223
+ const protocolCompletionDelta = increaseDelta(baseline?.protocolCompletionRate, harnessProtocolCompletionRate);
1224
+ const escapedIncompleteRate = improvementRate(baseline?.escapedIncomplete, latest?.escapedIncomplete);
1225
+ return { taskId: task.id, title: task.title, comparability, comparable, baselineDeliveryComplete, baselineEvidenceCoverageRate, baselineSampleCount: baselineDurationSamples.length, baselineArtifactAcceptanceRate: baseline?.artifactAcceptanceRate ?? null, baselineProtocolCompletionRate: baseline?.protocolCompletionRate ?? null, ...baselineMedianDurationMs === null ? {} : { baselineMedianDurationMs }, improvement: { durationRate, duration: improvementDirection(durationRate), attemptsRate, attempts: improvementDirection(attemptsRate), reviewRate, review: improvementDirection(reviewRate), artifactAcceptanceRate: artifactAcceptanceImprovementRate, artifactAcceptance: increaseDirection(artifactAcceptanceImprovementRate, artifactAcceptanceDelta), artifactAcceptanceDelta, protocolCompletionRate: protocolCompletionImprovementRate, protocolCompletion: increaseDirection(protocolCompletionImprovementRate, protocolCompletionDelta), protocolCompletionDelta, escapedIncompleteRate, escapedIncomplete: improvementDirection(escapedIncompleteRate) }, ...baseline ? { baseline } : {}, harness: { attempts, retryCount, completedRuns, durationSamplesMs, ...medianDurationMs === null ? {} : { medianDurationMs }, ...harnessArtifactAcceptanceRate === null ? {} : { artifactAcceptanceRate: harnessArtifactAcceptanceRate }, artifactAcceptanceSampleCount: acceptanceSamples.length, protocolCompletionRate: harnessProtocolCompletionRate, protocolCompletionSampleCount: taskRuns.length, latestState: latest?.state ?? "NOT_RUN", ...latest ? { latestRunId: latest.runId } : {}, ...latest?.durationMs === void 0 ? {} : { latestDurationMs: latest.durationMs }, checkPassRate: latest ? percentage(latest.checks.passed, latest.checks.total) : null, outcomePassRate: latest ? percentage(latest.outcomes.passed, latest.outcomes.total) : null, evidenceCoverageRate: latest ? percentage(latest.evidence.attached, latest.evidence.total) : null, ...latest?.escapedIncomplete === void 0 ? {} : { escapedIncomplete: latest.escapedIncomplete }, ...latest?.humanReviewMinutes === void 0 ? {} : { humanReviewMinutes: latest.humanReviewMinutes }, humanApproved: latest?.humanApproved ?? false }, confidence: confidence(comparable, completedRuns, policy), ...comparable && baselineMedianDurationMs !== null && medianDurationMs !== null ? { durationDeltaMs: medianDurationMs - baselineMedianDurationMs } : {}, ...comparable && baseline?.attempts !== void 0 ? { attemptDelta: attempts - baseline.attempts } : {}, ...comparable && baseline?.reviewMinutes !== void 0 && latest?.humanReviewMinutes !== void 0 ? { reviewDeltaMinutes: latest.humanReviewMinutes - baseline.reviewMinutes } : {}, ...baseline?.escapedIncomplete !== void 0 && latest?.escapedIncomplete !== void 0 ? { escapedIncompleteDelta: latest.escapedIncomplete - baseline.escapedIncomplete } : {} };
1226
+ });
1227
+ var benchmarkRuns = (stateDir, manifest) => {
1228
+ const runs = readRuns(stateDir).map(projectRun).sort((left, right) => left.runId.localeCompare(right.runId));
1229
+ const policy = manifest?.policy ?? DEFAULT_POLICY;
1230
+ const reportComparisons = manifest ? comparisons(runs, manifest, policy) : [];
1231
+ const comparable = reportComparisons.filter((comparison) => comparison.comparable);
1232
+ const durationRegressionTaskIds = comparable.filter((comparison) => (comparison.improvement.durationRate ?? 0) < -policy.maxDurationRegressionRate).map((comparison) => comparison.taskId);
1233
+ const escapedIncompleteTaskIds = comparable.filter((comparison) => comparison.harness.escapedIncomplete !== 0).map((comparison) => comparison.taskId);
1234
+ const reasons = [];
1235
+ if (comparable.length < policy.minComparableTasks) reasons.push(`requires at least ${policy.minComparableTasks} comparable tasks`);
1236
+ const incompleteBaselineTaskIds = reportComparisons.filter((comparison) => comparison.comparability === "baseline-incomplete").map((comparison) => comparison.taskId);
1237
+ if (incompleteBaselineTaskIds.length) reasons.push(`baseline delivery incomplete: ${incompleteBaselineTaskIds.join(", ")}`);
1238
+ const baselineSampleGaps = reportComparisons.filter((comparison) => comparison.baselineSampleCount < policy.minBaselineSamplesPerTask).map((comparison) => `${comparison.taskId} (${comparison.baselineSampleCount}/${policy.minBaselineSamplesPerTask})`);
1239
+ if (baselineSampleGaps.length) reasons.push(`requires ${policy.minBaselineSamplesPerTask} baseline samples per task: ${baselineSampleGaps.join(", ")}`);
1240
+ if (durationRegressionTaskIds.length) reasons.push(`duration regression exceeds ${policy.maxDurationRegressionRate * 100}%: ${durationRegressionTaskIds.join(", ")}`);
1241
+ if (policy.requireZeroEscapedIncomplete && escapedIncompleteTaskIds.length) reasons.push(`escaped incomplete delivery: ${escapedIncompleteTaskIds.join(", ")}`);
1242
+ const confidenceLevel = comparable.length < policy.minComparableTasks ? "insufficient" : comparable.every((comparison) => comparison.confidence === "reliable") ? "reliable" : "directional";
1243
+ const qualityGate = { status: comparable.length < policy.minComparableTasks ? "insufficient-data" : reasons.length ? "failed" : "passed", confidence: confidenceLevel, comparableTaskCount: comparable.length, policy, durationRegressionTaskIds, escapedIncompleteTaskIds, reasons };
1244
+ return { type: "agentskit-harness-benchmark", schemaVersion: BENCHMARK_SCHEMA_VERSION, stateDir, generatedAt: (/* @__PURE__ */ new Date()).toISOString(), runs, summary: summarize(runs), comparisons: reportComparisons, qualityGate, ...manifest ? { manifest: { suiteId: manifest.suiteId, taskCount: manifest.tasks.length, baselineCount: manifest.observations.length, comparableTaskCount: comparable.length } } : {} };
1245
+ };
1246
+
1247
+ // src/coding-benchmark.ts
1248
+ var text2 = (value, label) => {
1249
+ if (typeof value !== "string" || !value.trim()) fail(`${label} must be a non-empty string.`, "INVALID_INPUT");
1250
+ return value.trim();
1251
+ };
1252
+ var numberValue = (value, label, integer = false) => {
1253
+ if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || integer && !Number.isInteger(value)) fail(`${label} must be a non-negative ${integer ? "integer" : "number"}.`, "INVALID_INPUT");
1254
+ return value;
1255
+ };
1256
+ var validateExternalCodingBenchmarkReport = (value) => {
1257
+ if (typeof value !== "object" || value === null || Array.isArray(value)) fail("coding benchmark report must be an object.", "INVALID_INPUT");
1258
+ const raw = value;
1259
+ if (!["edit", "fix-bug", "add-feature", "refactor", "add-test", "review-pr", "free-form"].includes(text2(raw["kind"], "report.kind"))) fail("report.kind is not a supported coding task kind.", "INVALID_INPUT");
1260
+ if (typeof raw["dryRun"] !== "boolean" || typeof raw["isolateWorktrees"] !== "boolean") fail("report.dryRun and report.isolateWorktrees must be booleans.", "INVALID_INPUT");
1261
+ if (!Array.isArray(raw["rows"]) || raw["rows"].length === 0) fail("report.rows must contain at least one provider result.", "INVALID_INPUT");
1262
+ const rows = raw["rows"].map((item, index2) => {
1263
+ if (typeof item !== "object" || item === null || Array.isArray(item)) fail(`report.rows[${index2}] must be an object.`, "INVALID_INPUT");
1264
+ const row = item;
1265
+ const status = text2(row["status"], `report.rows[${index2}].status`);
1266
+ if (!["ok", "partial", "fail", "timeout"].includes(status)) fail(`report.rows[${index2}].status is invalid.`, "INVALID_INPUT");
1267
+ const completenessScore = numberValue(row["completenessScore"], `report.rows[${index2}].completenessScore`);
1268
+ if (completenessScore > 100) fail(`report.rows[${index2}].completenessScore must be between 0 and 100.`, "INVALID_INPUT");
1269
+ const optional = (key) => row[key] === void 0 ? void 0 : numberValue(row[key], `report.rows[${index2}].${key}`);
1270
+ return {
1271
+ providerId: text2(row["providerId"], `report.rows[${index2}].providerId`),
1272
+ status,
1273
+ completenessScore,
1274
+ fileEditCount: numberValue(row["fileEditCount"], `report.rows[${index2}].fileEditCount`, true),
1275
+ summary: text2(row["summary"], `report.rows[${index2}].summary`),
1276
+ ...optional("durationMs") === void 0 ? {} : { durationMs: optional("durationMs") },
1277
+ ...optional("inputTokens") === void 0 ? {} : { inputTokens: optional("inputTokens") },
1278
+ ...optional("outputTokens") === void 0 ? {} : { outputTokens: optional("outputTokens") },
1279
+ ...optional("costUsd") === void 0 ? {} : { costUsd: optional("costUsd") },
1280
+ ...row["successPassed"] === void 0 ? {} : typeof row["successPassed"] !== "boolean" ? fail(`report.rows[${index2}].successPassed must be a boolean.`, "INVALID_INPUT") : { successPassed: row["successPassed"] }
1281
+ };
1282
+ });
1283
+ if (new Set(rows.map((row) => row.providerId)).size !== rows.length) fail("coding benchmark provider ids must be unique.", "INVALID_INPUT");
1284
+ return { kind: text2(raw["kind"], "report.kind"), prompt: text2(raw["prompt"], "report.prompt"), dryRun: raw["dryRun"], isolateWorktrees: raw["isolateWorktrees"], repoRoot: text2(raw["repoRoot"], "report.repoRoot"), rows };
1285
+ };
1286
+ var required = (value, label) => {
1287
+ if (typeof value !== "string" || !value.trim()) fail(`${label} is required.`, "INVALID_INPUT");
1288
+ return value.trim();
1289
+ };
1290
+ var duration = (value) => {
1291
+ if (!Number.isFinite(value) || value < 0) fail("Tool durationMs must be a non-negative number.", "INVALID_INPUT");
1292
+ return value;
1293
+ };
1294
+ var createSessionRecorder = ({ stateDir, run, adapter, policy, runtime, sessionId = randomUUID(), resume = false }) => {
1295
+ if (run.state !== "IMPLEMENTING") fail(`Agent sessions can only start during IMPLEMENTING, not ${run.state}.`, "INVALID_STATE");
1296
+ const id2 = required(sessionId, "sessionId");
1297
+ const adapterId = required(adapter.id, "adapter.id");
1298
+ const adapterVersion = required(adapter.version, "adapter.version");
1299
+ if (!policy || typeof policy.evaluate !== "function") fail("policy.evaluate is required.", "INVALID_INPUT");
1300
+ if (!runtime || typeof runtime.execute !== "function") fail("runtime.execute is required.", "INVALID_INPUT");
1301
+ if (!Array.isArray(adapter.capabilities) || adapter.capabilities.some((capability) => typeof capability !== "string" || !capability.trim())) fail("adapter.capabilities must contain non-empty strings.", "INVALID_INPUT");
1302
+ const store = new FileEventStore(stateDir);
1303
+ const append = (type, payload) => store.append({ runId: run.runId, sourceRevision: run.sourceRevision, configHash: run.configHash, sessionId: id2, type, payload });
1304
+ const turns = /* @__PURE__ */ new Set();
1305
+ const actions = /* @__PURE__ */ new Set();
1306
+ const pending = /* @__PURE__ */ new Map();
1307
+ const approvals = /* @__PURE__ */ new Map();
1308
+ const released = /* @__PURE__ */ new Map();
1309
+ const attempts = /* @__PURE__ */ new Map();
1310
+ const executing = /* @__PURE__ */ new Set();
1311
+ let ended = false;
1312
+ if (resume) {
1313
+ const prior = store.read(run.runId).filter((event) => event.sessionId === id2);
1314
+ if (!prior.some((event) => event.type === "session.started")) fail(`Session does not exist: ${id2}.`, "INVALID_STATE");
1315
+ if (prior.some((event) => event.type === "session.ended")) fail(`Session has already ended: ${id2}.`, "INVALID_STATE");
1316
+ for (const event of prior) {
1317
+ if (event.type === "agent.turn.started") turns.add(event.payload.turnId);
1318
+ if (event.type === "tool.approval.requested") {
1319
+ actions.add(event.payload.actionId);
1320
+ approvals.set(event.payload.actionId, { turnId: event.payload.turnId, toolId: event.payload.toolId, argumentsHash: event.payload.argumentsHash, policyId: event.payload.policyId, reason: event.payload.reason });
1321
+ }
1322
+ if (event.type === "tool.approval.recorded") {
1323
+ approvals.delete(event.payload.actionId);
1324
+ if (event.payload.decision === "approved") released.set(event.payload.actionId, { turnId: event.payload.turnId, toolId: event.payload.toolId, argumentsHash: event.payload.argumentsHash, executionStarted: false });
1325
+ }
1326
+ if (event.type === "tool.requested") {
1327
+ actions.add(event.payload.actionId);
1328
+ pending.set(event.payload.actionId, { turnId: event.payload.turnId, toolId: event.payload.toolId, argumentsHash: event.payload.argumentsHash, executionStarted: false });
1329
+ released.delete(event.payload.actionId);
1330
+ }
1331
+ if (event.type === "tool.execution.started") {
1332
+ const action = pending.get(event.payload.actionId);
1333
+ if (action) action.executionStarted = true;
1334
+ attempts.set(event.payload.actionId, event.payload.attempt);
1335
+ }
1336
+ if (event.type === "tool.recovery.recorded" && event.payload.decision === "retry") {
1337
+ const action = pending.get(event.payload.actionId);
1338
+ if (action) action.executionStarted = false;
1339
+ }
1340
+ if (event.type === "tool.completed" || event.type === "tool.failed" || event.type === "tool.blocked") pending.delete(event.payload.actionId);
1341
+ }
1342
+ for (const [actionId, action] of released) if (!pending.has(actionId)) pending.set(actionId, action);
1343
+ }
1344
+ const open = () => {
1345
+ if (ended) fail("Session has already ended.", "INVALID_STATE");
1346
+ };
1347
+ const complete = (input) => {
1348
+ open();
1349
+ const actionId = required(input.actionId, "actionId");
1350
+ if (!pending.has(actionId)) fail(`Tool action is not pending: ${actionId}.`, "INVALID_STATE");
1351
+ const event = append("tool.completed", { actionId, resultHash: required(input.resultHash, "resultHash"), durationMs: duration(input.durationMs), ...input.runtimeEvidence ? { runtimeEvidence: input.runtimeEvidence } : {} });
1352
+ pending.delete(actionId);
1353
+ return event;
1354
+ };
1355
+ const failAction = (input) => {
1356
+ open();
1357
+ const actionId = required(input.actionId, "actionId");
1358
+ if (!pending.has(actionId)) fail(`Tool action is not pending: ${actionId}.`, "INVALID_STATE");
1359
+ if (typeof input.retryable !== "boolean") fail("retryable must be boolean.", "INVALID_INPUT");
1360
+ const event = append("tool.failed", { actionId, errorCode: required(input.errorCode, "errorCode"), retryable: input.retryable, durationMs: duration(input.durationMs), ...input.runtimeEvidence ? { runtimeEvidence: input.runtimeEvidence } : {} });
1361
+ pending.delete(actionId);
1362
+ return event;
1363
+ };
1364
+ const recorder = {
1365
+ sessionId: id2,
1366
+ startTurn: (inputHash, turnId = randomUUID()) => {
1367
+ open();
1368
+ const turn = required(turnId, "turnId");
1369
+ if (turns.has(turn)) fail(`Turn already exists: ${turn}.`, "INVALID_STATE");
1370
+ const event = append("agent.turn.started", { turnId: turn, inputHash: required(inputHash, "inputHash") });
1371
+ turns.add(turn);
1372
+ return event;
1373
+ },
1374
+ requestTool: (input) => {
1375
+ open();
1376
+ const turnId = required(input.turnId, "turnId");
1377
+ if (!turns.has(turnId)) fail(`Turn does not exist: ${turnId}.`, "INVALID_STATE");
1378
+ const actionId = required(input.actionId ?? randomUUID(), "actionId");
1379
+ if (actions.has(actionId)) fail(`Tool action already exists: ${actionId}.`, "INVALID_STATE");
1380
+ const toolId = required(input.toolId, "toolId");
1381
+ const argumentsHash = required(input.argumentsHash, "argumentsHash");
1382
+ const decision = policy.evaluate({ actionId, turnId, toolId, argumentsHash });
1383
+ if (!decision || decision.decision !== "allow" && decision.decision !== "block" && decision.decision !== "approve") fail("Policy decision is invalid.", "HARNESS_ERROR");
1384
+ const policyId = required(decision.policyId, "policyId");
1385
+ const reason = required(decision.reason, "policy reason");
1386
+ append("policy.evaluated", { actionId, turnId, toolId, decision: decision.decision, policyId, reason });
1387
+ actions.add(actionId);
1388
+ if (decision.decision === "block") {
1389
+ append("tool.blocked", { turnId, actionId, toolId, policyId, reason });
1390
+ fail(`Tool action blocked by policy: ${policyId}.`, "POLICY_BLOCKED");
1391
+ }
1392
+ if (decision.decision === "approve") {
1393
+ const event2 = append("tool.approval.requested", { turnId, actionId, toolId, argumentsHash, policyId, reason });
1394
+ approvals.set(actionId, { turnId, toolId, argumentsHash, policyId, reason });
1395
+ return event2;
1396
+ }
1397
+ const event = append("tool.requested", { turnId, actionId, toolId, argumentsHash });
1398
+ pending.set(actionId, { turnId, toolId, argumentsHash, executionStarted: false });
1399
+ return event;
1400
+ },
1401
+ approveTool: (input) => {
1402
+ open();
1403
+ const actionId = required(input.actionId, "actionId");
1404
+ const approval = approvals.get(actionId) ?? fail(`Tool action is not awaiting human approval: ${actionId}.`, "INVALID_STATE");
1405
+ if (input.actor !== void 0 && input.actor !== "human") fail("Tool approval requires a human actor.", "HUMAN_APPROVAL_REQUIRED");
1406
+ const decision = input.decision;
1407
+ if (decision !== "approved" && decision !== "rejected") fail("Tool approval decision is invalid.", "INVALID_INPUT");
1408
+ append("tool.approval.recorded", { turnId: approval.turnId, actionId, toolId: approval.toolId, argumentsHash: approval.argumentsHash, decision, actor: "human", policyId: approval.policyId, reason: approval.reason });
1409
+ approvals.delete(actionId);
1410
+ if (decision === "rejected") return append("tool.blocked", { turnId: approval.turnId, actionId, toolId: approval.toolId, policyId: approval.policyId, reason: "Human rejected the tool action." });
1411
+ const event = append("tool.requested", { turnId: approval.turnId, actionId, toolId: approval.toolId, argumentsHash: approval.argumentsHash });
1412
+ pending.set(actionId, { turnId: approval.turnId, toolId: approval.toolId, argumentsHash: approval.argumentsHash, executionStarted: false });
1413
+ return event;
1414
+ },
1415
+ recoverTool: (input) => {
1416
+ open();
1417
+ const actionId = required(input.actionId, "actionId");
1418
+ const action = pending.get(actionId) ?? fail(`Tool action is not pending: ${actionId}.`, "INVALID_STATE");
1419
+ if (!action.executionStarted) fail(`Tool action does not require recovery: ${actionId}.`, "INVALID_STATE");
1420
+ if (input.actor !== void 0 && input.actor !== "human") fail("Tool recovery requires a human actor.", "HUMAN_APPROVAL_REQUIRED");
1421
+ if (input.decision !== "retry" && input.decision !== "abandon") fail("Tool recovery decision is invalid.", "INVALID_INPUT");
1422
+ const reason = input.decision === "retry" ? "Human authorized a retry after interruption." : "Human abandoned the interrupted tool action.";
1423
+ const recovery = append("tool.recovery.recorded", { turnId: action.turnId, actionId, toolId: action.toolId, decision: input.decision, actor: "human", reason });
1424
+ if (input.decision === "retry") {
1425
+ action.executionStarted = false;
1426
+ return recovery;
1427
+ }
1428
+ pending.delete(actionId);
1429
+ return append("tool.blocked", { turnId: action.turnId, actionId, toolId: action.toolId, policyId: "recovery", reason });
1430
+ },
1431
+ completeTool: complete,
1432
+ failTool: failAction,
1433
+ executeTool: async (input) => {
1434
+ open();
1435
+ const actionId = required(input.actionId, "actionId");
1436
+ const action = pending.get(actionId) ?? fail(`Tool action is not pending: ${actionId}.`, "INVALID_STATE");
1437
+ if (action.executionStarted) fail(`Tool action requires human recovery decision: ${actionId}.`, "HUMAN_APPROVAL_REQUIRED");
1438
+ if (executing.has(actionId)) fail(`Tool action is already executing: ${actionId}.`, "INVALID_STATE");
1439
+ executing.add(actionId);
1440
+ action.executionStarted = true;
1441
+ const attempt = (attempts.get(actionId) ?? 0) + 1;
1442
+ attempts.set(actionId, attempt);
1443
+ append("tool.execution.started", { actionId, turnId: action.turnId, toolId: action.toolId, attempt });
1444
+ try {
1445
+ const result = await runtime.execute({ actionId, turnId: action.turnId, toolId: action.toolId, argumentsHash: action.argumentsHash, arguments: input.arguments });
1446
+ if (result.status === "completed") {
1447
+ complete({ actionId, resultHash: result.resultHash, durationMs: result.durationMs, runtimeEvidence: result.runtimeEvidence });
1448
+ return result;
1449
+ }
1450
+ if (result.status === "failed") {
1451
+ failAction({ actionId, errorCode: result.errorCode, retryable: result.retryable, durationMs: result.durationMs, runtimeEvidence: result.runtimeEvidence });
1452
+ return result;
1453
+ }
1454
+ return fail("Runtime returned an invalid execution result.", "HARNESS_ERROR");
1455
+ } catch {
1456
+ const result = { status: "failed", errorCode: "RUNTIME_ERROR", retryable: true, durationMs: 0 };
1457
+ if (pending.has(actionId)) failAction({ actionId, ...result });
1458
+ return result;
1459
+ } finally {
1460
+ executing.delete(actionId);
1461
+ }
1462
+ },
1463
+ end: (status) => {
1464
+ open();
1465
+ if (!["completed", "failed", "cancelled"].includes(status)) fail("Session status is invalid.", "INVALID_INPUT");
1466
+ if (pending.size) fail("Session cannot end while tool actions are pending.", "INVALID_STATE");
1467
+ if (approvals.size) fail("Session cannot end while tool approvals are pending.", "HUMAN_APPROVAL_REQUIRED");
1468
+ const event = append("session.ended", { status });
1469
+ ended = true;
1470
+ return event;
1471
+ }
1472
+ };
1473
+ if (resume) append("session.resumed", { recovery: "event-log" });
1474
+ else append("session.started", { adapterId, adapterVersion, capabilities: adapter.capabilities.map((capability) => capability.trim()) });
1475
+ return recorder;
1476
+ };
1477
+
1478
+ // src/policy.ts
1479
+ var required2 = (value, label) => {
1480
+ if (typeof value !== "string" || !value.trim()) fail(`${label} is required.`, "INVALID_INPUT");
1481
+ return value.trim();
1482
+ };
1483
+ var createPolicyGate = ({ rules }) => {
1484
+ if (!Array.isArray(rules)) fail("Policy rules must be an array.", "INVALID_INPUT");
1485
+ const normalized = rules.map((rule, index2) => {
1486
+ if (typeof rule !== "object" || rule === null || Array.isArray(rule)) fail(`rules[${index2}] must be an object.`, "INVALID_INPUT");
1487
+ const id2 = required2(rule.id, `rules[${index2}].id`);
1488
+ if (rule.effect !== "allow" && rule.effect !== "block" && rule.effect !== "approve") fail(`rules[${index2}].effect is invalid.`, "INVALID_INPUT");
1489
+ if (!Array.isArray(rule.toolIds) || !rule.toolIds.length || rule.toolIds.some((toolId) => typeof toolId !== "string" || !toolId.trim())) fail(`rules[${index2}].toolIds must contain non-empty strings.`, "INVALID_INPUT");
1490
+ return { id: id2, effect: rule.effect, toolIds: rule.toolIds.map((toolId) => required2(toolId, `rules[${index2}].toolIds`)), reason: required2(rule.reason, `rules[${index2}].reason`) };
1491
+ });
1492
+ if (new Set(normalized.map((rule) => rule.id)).size !== normalized.length) fail("Policy rules must have unique ids.", "INVALID_INPUT");
1493
+ return {
1494
+ evaluate: (request) => {
1495
+ if (typeof request !== "object" || request === null || Array.isArray(request)) fail("Policy request must be an object.", "INVALID_INPUT");
1496
+ required2(request.actionId, "request.actionId");
1497
+ required2(request.turnId, "request.turnId");
1498
+ const toolId = required2(request.toolId, "request.toolId");
1499
+ required2(request.argumentsHash, "request.argumentsHash");
1500
+ const rule = normalized.find((candidate) => candidate.toolIds.includes(toolId));
1501
+ return rule ? { decision: rule.effect, policyId: rule.id, reason: rule.reason } : { decision: "block", policyId: "default-deny", reason: `No policy rule allows tool: ${toolId}.` };
1502
+ }
1503
+ };
1504
+ };
1505
+ var required3 = (value, label) => {
1506
+ if (typeof value !== "string" || !value.trim()) fail(`${label} is required.`, "INVALID_INPUT");
1507
+ return value.trim();
1508
+ };
1509
+ var duration2 = (value) => {
1510
+ if (!Number.isFinite(value) || value < 0) fail("Tool durationMs must be a non-negative number.", "INVALID_INPUT");
1511
+ return value;
1512
+ };
1513
+ var positiveNumber = (value, label) => {
1514
+ const normalized = String(value).trim();
1515
+ if (!normalized || !/^(?:\d+(?:\.\d+)?|\.\d+)$/.test(normalized) || Number(normalized) <= 0) fail(`${label} must be positive.`, "INVALID_INPUT");
1516
+ return normalized;
1517
+ };
1518
+ var absolutePath = (value, label) => {
1519
+ const normalized = required3(value, label);
1520
+ if (!normalized.startsWith("/") || normalized.includes(",")) fail(`${label} must be an absolute path without commas.`, "INVALID_INPUT");
1521
+ return normalized;
1522
+ };
1523
+ var dockerEnvironment = (env, label) => {
1524
+ if (env === void 0) return [];
1525
+ if (typeof env !== "object" || env === null || Array.isArray(env)) fail(`${label} must be an object.`, "INVALID_INPUT");
1526
+ return Object.entries(env).map(([key, value]) => {
1527
+ if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key) || typeof value !== "string") fail(`${label} must contain valid string environment entries.`, "INVALID_INPUT");
1528
+ return `${key}=${value}`;
1529
+ });
1530
+ };
1531
+ var inspectImage = promisify(execFile);
1532
+ var createToolRuntime = ({ tools, timeoutMs = 3e4 }) => {
1533
+ if (!Array.isArray(tools)) fail("Runtime tools must be an array.", "INVALID_INPUT");
1534
+ if (!Number.isInteger(timeoutMs) || timeoutMs < 1) fail("Runtime timeoutMs must be a positive integer.", "INVALID_INPUT");
1535
+ const normalized = tools.map((tool, index2) => {
1536
+ if (typeof tool !== "object" || tool === null || Array.isArray(tool)) fail(`tools[${index2}] must be an object.`, "INVALID_INPUT");
1537
+ const toolId = required3(tool.toolId, `tools[${index2}].toolId`);
1538
+ if (typeof tool.execute !== "function") fail(`tools[${index2}].execute is required.`, "INVALID_INPUT");
1539
+ return { toolId, execute: tool.execute };
1540
+ });
1541
+ if (new Set(normalized.map((tool) => tool.toolId)).size !== normalized.length) fail("Runtime tools must have unique ids.", "INVALID_INPUT");
1542
+ return {
1543
+ execute: async (request) => {
1544
+ const started = Date.now();
1545
+ const actionId = required3(request.actionId, "request.actionId");
1546
+ const turnId = required3(request.turnId, "request.turnId");
1547
+ const toolId = required3(request.toolId, "request.toolId");
1548
+ const argumentsHash = required3(request.argumentsHash, "request.argumentsHash");
1549
+ const tool = normalized.find((candidate) => candidate.toolId === toolId);
1550
+ if (!tool) return { status: "failed", errorCode: "TOOL_NOT_FOUND", retryable: false, durationMs: duration2(Date.now() - started) };
1551
+ const controller = new AbortController();
1552
+ let timedOut = false;
1553
+ let timer;
1554
+ try {
1555
+ const timeout = new Promise((_, reject) => {
1556
+ timer = setTimeout(() => {
1557
+ timedOut = true;
1558
+ controller.abort();
1559
+ reject(new Error("Tool execution timed out."));
1560
+ }, timeoutMs);
1561
+ });
1562
+ const result = await Promise.race([Promise.resolve(tool.execute({ actionId, turnId, toolId, argumentsHash, arguments: request.arguments, signal: controller.signal })), timeout]);
1563
+ return { status: "completed", resultHash: hashJson(result === void 0 ? null : result), durationMs: duration2(Date.now() - started) };
1564
+ } catch {
1565
+ return { status: "failed", errorCode: timedOut ? "TIMEOUT" : "RUNTIME_ERROR", retryable: true, durationMs: duration2(Date.now() - started) };
1566
+ } finally {
1567
+ if (timer) clearTimeout(timer);
1568
+ }
1569
+ }
1570
+ };
1571
+ };
1572
+ var createProcessToolRuntime = ({ tools, timeoutMs = 3e4, maxOutputBytes = 1048576 }) => {
1573
+ if (!Array.isArray(tools)) fail("Process runtime tools must be an array.", "INVALID_INPUT");
1574
+ if (!Number.isInteger(timeoutMs) || timeoutMs < 1) fail("Process runtime timeoutMs must be a positive integer.", "INVALID_INPUT");
1575
+ if (!Number.isInteger(maxOutputBytes) || maxOutputBytes < 1) fail("Process runtime maxOutputBytes must be a positive integer.", "INVALID_INPUT");
1576
+ const normalized = tools.map((tool, index2) => {
1577
+ if (typeof tool !== "object" || tool === null || Array.isArray(tool)) fail(`tools[${index2}] must be an object.`, "INVALID_INPUT");
1578
+ const toolId = required3(tool.toolId, `tools[${index2}].toolId`);
1579
+ const command = required3(tool.command, `tools[${index2}].command`);
1580
+ if (tool.args !== void 0 && (!Array.isArray(tool.args) || tool.args.some((arg) => typeof arg !== "string"))) fail(`tools[${index2}].args must contain strings.`, "INVALID_INPUT");
1581
+ if (tool.env !== void 0 && (typeof tool.env !== "object" || tool.env === null || Array.isArray(tool.env) || Object.values(tool.env).some((value) => typeof value !== "string"))) fail(`tools[${index2}].env must contain string values.`, "INVALID_INPUT");
1582
+ return { toolId, command, args: tool.args ? [...tool.args] : [], ...tool.cwd ? { cwd: tool.cwd } : {}, env: tool.env ? { ...tool.env } : { PATH: process.env["PATH"] ?? "" } };
1583
+ });
1584
+ if (new Set(normalized.map((tool) => tool.toolId)).size !== normalized.length) fail("Process runtime tools must have unique ids.", "INVALID_INPUT");
1585
+ return {
1586
+ execute: async (request) => {
1587
+ const started = Date.now();
1588
+ const actionId = required3(request.actionId, "request.actionId");
1589
+ const turnId = required3(request.turnId, "request.turnId");
1590
+ const toolId = required3(request.toolId, "request.toolId");
1591
+ const argumentsHash = required3(request.argumentsHash, "request.argumentsHash");
1592
+ const tool = normalized.find((candidate) => candidate.toolId === toolId);
1593
+ if (!tool) return { status: "failed", errorCode: "TOOL_NOT_FOUND", retryable: false, durationMs: Date.now() - started };
1594
+ let input;
1595
+ try {
1596
+ input = JSON.stringify({ actionId, turnId, toolId, argumentsHash, arguments: request.arguments });
1597
+ } catch {
1598
+ return { status: "failed", errorCode: "SERIALIZATION_ERROR", retryable: false, durationMs: Date.now() - started };
1599
+ }
1600
+ return new Promise((resolve6) => {
1601
+ const child = spawn(tool.command, tool.args, { cwd: tool.cwd, env: tool.env, shell: false, stdio: ["pipe", "pipe", "pipe"] });
1602
+ let stdout = "";
1603
+ let timedOut = false;
1604
+ let outputLimit = false;
1605
+ let spawnError = false;
1606
+ let settled = false;
1607
+ const timer = setTimeout(() => {
1608
+ timedOut = true;
1609
+ child.kill("SIGKILL");
1610
+ }, timeoutMs);
1611
+ const finish = (result) => {
1612
+ if (settled) return;
1613
+ settled = true;
1614
+ clearTimeout(timer);
1615
+ resolve6(result);
1616
+ };
1617
+ child.stdout.on("data", (chunk) => {
1618
+ stdout += chunk.toString();
1619
+ if (Buffer.byteLength(stdout) > maxOutputBytes) {
1620
+ outputLimit = true;
1621
+ child.kill("SIGKILL");
1622
+ }
1623
+ });
1624
+ child.stderr.on("data", (chunk) => {
1625
+ if (chunk.length > maxOutputBytes) {
1626
+ outputLimit = true;
1627
+ child.kill("SIGKILL");
1628
+ }
1629
+ });
1630
+ child.on("error", () => {
1631
+ spawnError = true;
1632
+ });
1633
+ child.on("close", (code) => {
1634
+ const durationMs = Date.now() - started;
1635
+ if (timedOut) return finish({ status: "failed", errorCode: "TIMEOUT", retryable: true, durationMs });
1636
+ if (outputLimit) return finish({ status: "failed", errorCode: "OUTPUT_LIMIT", retryable: false, durationMs });
1637
+ if (spawnError || code === null) return finish({ status: "failed", errorCode: "PROCESS_ERROR", retryable: true, durationMs });
1638
+ if (code !== 0) return finish({ status: "failed", errorCode: "PROCESS_EXIT", retryable: false, durationMs });
1639
+ finish({ status: "completed", resultHash: hashJson(stdout.trim() || null), durationMs });
1640
+ });
1641
+ child.stdin.on("error", () => {
1642
+ spawnError = true;
1643
+ });
1644
+ child.stdin.end(input);
1645
+ });
1646
+ }
1647
+ };
1648
+ };
1649
+ var createDockerToolRuntime = ({
1650
+ tools,
1651
+ timeoutMs = 3e4,
1652
+ maxOutputBytes = 1048576,
1653
+ dockerCommand = "docker",
1654
+ memoryLimit = "512m",
1655
+ cpus = 1,
1656
+ pidsLimit = 128,
1657
+ user = "65532:65532",
1658
+ pull = "never"
1659
+ }) => {
1660
+ if (!Array.isArray(tools)) fail("Docker runtime tools must be an array.", "INVALID_INPUT");
1661
+ const command = required3(dockerCommand, "dockerCommand");
1662
+ const memory = required3(memoryLimit, "memoryLimit");
1663
+ const cpu = positiveNumber(cpus, "cpus");
1664
+ if (!Number.isInteger(pidsLimit) || pidsLimit < 1) fail("pidsLimit must be a positive integer.", "INVALID_INPUT");
1665
+ const normalizedUser = required3(user, "user");
1666
+ if (normalizedUser.includes(" ")) fail("user must not contain spaces.", "INVALID_INPUT");
1667
+ if (pull !== "never" && pull !== "missing" && pull !== "always") fail("pull must be never, missing, or always.", "INVALID_INPUT");
1668
+ const normalized = tools.map((tool, index2) => {
1669
+ if (typeof tool !== "object" || tool === null || Array.isArray(tool)) fail(`tools[${index2}] must be an object.`, "INVALID_INPUT");
1670
+ const toolId = required3(tool.toolId, `tools[${index2}].toolId`);
1671
+ const image = required3(tool.image, `tools[${index2}].image`);
1672
+ if (!Array.isArray(tool.command) || tool.command.length === 0 || tool.command.some((part) => typeof part !== "string" || !part.trim())) fail(`tools[${index2}].command must be a non-empty string array.`, "INVALID_INPUT");
1673
+ if (tool.args !== void 0 && (!Array.isArray(tool.args) || tool.args.some((arg) => typeof arg !== "string"))) fail(`tools[${index2}].args must contain strings.`, "INVALID_INPUT");
1674
+ const env = dockerEnvironment(tool.env, `tools[${index2}].env`);
1675
+ if (tool.mounts !== void 0 && !Array.isArray(tool.mounts)) fail(`tools[${index2}].mounts must be an array.`, "INVALID_INPUT");
1676
+ const mounts = (tool.mounts ?? []).map((mount, mountIndex) => {
1677
+ if (typeof mount !== "object" || mount === null || Array.isArray(mount)) fail(`tools[${index2}].mounts[${mountIndex}] must be an object.`, "INVALID_INPUT");
1678
+ const source = absolutePath(mount.source, `tools[${index2}].mounts[${mountIndex}].source`);
1679
+ const target = absolutePath(mount.target, `tools[${index2}].mounts[${mountIndex}].target`);
1680
+ if (mount.readOnly !== void 0 && typeof mount.readOnly !== "boolean") fail(`tools[${index2}].mounts[${mountIndex}].readOnly must be boolean.`, "INVALID_INPUT");
1681
+ return `type=bind,src=${source},dst=${target}${mount.readOnly === false ? "" : ",readonly"}`;
1682
+ });
1683
+ const profileHash = hashJson({ provider: "docker", toolId, image, command: [...tool.command], args: tool.args ?? [], cwd: tool.cwd ?? null, env, mounts, network: "none", readOnlyRootFilesystem: true, noNewPrivileges: true, capabilities: "drop-all", user: normalizedUser, memoryLimit: memory, cpus: cpu, pidsLimit, pull });
1684
+ return {
1685
+ toolId,
1686
+ command,
1687
+ image,
1688
+ profileHash,
1689
+ evidence: { provider: "docker", profileHash, image, network: "none", readOnlyRootFilesystem: true, noNewPrivileges: true, capabilities: "drop-all", user: normalizedUser, memoryLimit: memory, cpus: cpu, pidsLimit },
1690
+ args: [
1691
+ "run",
1692
+ "--rm",
1693
+ "--init",
1694
+ "--pull",
1695
+ pull,
1696
+ "--network",
1697
+ "none",
1698
+ "--read-only",
1699
+ "--cap-drop",
1700
+ "ALL",
1701
+ "--security-opt",
1702
+ "no-new-privileges",
1703
+ "--pids-limit",
1704
+ String(pidsLimit),
1705
+ "--memory",
1706
+ memory,
1707
+ "--cpus",
1708
+ cpu,
1709
+ "--user",
1710
+ normalizedUser,
1711
+ ...tool.cwd ? ["--workdir", absolutePath(tool.cwd, `tools[${index2}].cwd`)] : [],
1712
+ ...env.flatMap((entry) => ["--env", entry]),
1713
+ ...mounts.flatMap((mount) => ["--mount", mount]),
1714
+ image,
1715
+ ...tool.command,
1716
+ ...tool.args ?? []
1717
+ ]
1718
+ };
1719
+ });
1720
+ if (new Set(normalized.map((tool) => tool.toolId)).size !== normalized.length) fail("Docker runtime tools must have unique ids.", "INVALID_INPUT");
1721
+ const processRuntime = createProcessToolRuntime({ tools: normalized, timeoutMs, maxOutputBytes });
1722
+ return {
1723
+ execute: async (request) => {
1724
+ const tool = normalized.find((candidate) => candidate.toolId === request.toolId);
1725
+ if (!tool) return processRuntime.execute(request);
1726
+ const started = Date.now();
1727
+ let imageDigest;
1728
+ try {
1729
+ const inspected = await inspectImage(command, ["image", "inspect", tool.image, "--format", "{{.Id}}"], { shell: false, encoding: "utf8", maxBuffer: 64 * 1024, env: { PATH: process.env["PATH"] ?? "" } });
1730
+ imageDigest = inspected.stdout.trim();
1731
+ if (!/^sha256:[a-f0-9]{64}$/.test(imageDigest)) throw new Error("Docker image inspection did not return a digest.");
1732
+ } catch {
1733
+ return { status: "failed", errorCode: "IMAGE_UNAVAILABLE", retryable: true, durationMs: duration2(Date.now() - started), runtimeEvidence: tool.evidence };
1734
+ }
1735
+ const runtimeEvidence = { ...tool.evidence, imageDigest, profileHash: hashJson({ ...tool.evidence, imageDigest }) };
1736
+ const result = await processRuntime.execute(request);
1737
+ return { ...result, runtimeEvidence };
1738
+ }
1739
+ };
1740
+ };
1741
+ var EVIDENCE_BUNDLE_SCHEMA_VERSION = 1;
1742
+ var body = (bundle) => {
1743
+ const { payloadHash: _payloadHash, signature: _signature, ...unsigned } = bundle;
1744
+ return unsigned;
1745
+ };
1746
+ var parseBundle = (path) => {
1747
+ try {
1748
+ return JSON.parse(fileContents(path));
1749
+ } catch (error) {
1750
+ return fail(`Invalid evidence bundle JSON: ${error instanceof Error ? error.message : String(error)}`, "INVALID_INPUT");
1751
+ }
1752
+ };
1753
+ var validDigest = (value) => typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
1754
+ var validKeyId = (value) => typeof value === "string" && /^[A-Za-z0-9._:-]{1,128}$/.test(value);
1755
+ var requireRun2 = (run) => run ?? fail("No verification run exists.", "NO_RUN");
1756
+ var bundleFile = (stateDir, path) => {
1757
+ const absolute = resolve(stateDir, path);
1758
+ if (!pathInside(stateDir, absolute)) fail(`Evidence path escapes state directory: ${path}`, "HARNESS_ERROR");
1759
+ const content = readFileSync(absolute);
1760
+ return { path: relative(stateDir, absolute).split(sep).join("/"), sha256: sha256(content), contentBase64: content.toString("base64") };
1761
+ };
1762
+ var exportEvidenceBundle = async ({ configPath, runId, outputPath, privateKeyPath, keyId }) => {
1763
+ if (!validKeyId(keyId)) fail("keyId must contain only letters, numbers, dot, underscore, colon, or hyphen.", "INVALID_INPUT");
1764
+ const loaded = loadConfig(configPath);
1765
+ const run = requireRun2(runId ? readJson(join(loaded.stateDir, "runs", runId, "run.json")) : loadLatestRun(loaded.stateDir));
1766
+ const reconciliation = await reconcileRun({ configPath, runId: run.runId });
1767
+ const digest2 = run.verificationDigest ?? fail("Only a reconciled COMPLETE run can be exported.", "INVALID_STATE");
1768
+ if (reconciliation.state !== "COMPLETE") fail("Only a reconciled COMPLETE run can be exported.", "INVALID_STATE");
1769
+ const eventLog = new FileEventStore(loaded.stateDir);
1770
+ eventLog.read(run.runId);
1771
+ const eventVerification = eventLog.verify(run.runId);
1772
+ const paths = /* @__PURE__ */ new Set(["runs/" + run.runId + "/run.json", "runs/" + run.runId + "/events.ndjson"]);
1773
+ for (const reference of run.evidenceReferences) {
1774
+ paths.add(reference.stdout);
1775
+ paths.add(reference.stderr);
1776
+ }
1777
+ const files = [...paths].map((path) => {
1778
+ if (!existsSync(resolve(loaded.stateDir, path))) fail(`Evidence file is missing: ${path}`, "HARNESS_ERROR");
1779
+ return bundleFile(loaded.stateDir, path);
1780
+ });
1781
+ const privateKey = createPrivateKey(readFileSync(privateKeyPath));
1782
+ const unsigned = { type: "agentskit-harness-evidence-bundle", schemaVersion: EVIDENCE_BUNDLE_SCHEMA_VERSION, runId: run.runId, signerKeyId: keyId, sourceRevision: run.sourceRevision, configHash: run.configHash, contractHash: run.contractHash, verificationDigest: digest2, eventLog: eventVerification, files };
1783
+ const payloadHash = sha256(JSON.stringify(unsigned));
1784
+ const publicKeyPem = createPublicKey(privateKey).export({ type: "spki", format: "pem" }).toString();
1785
+ const bundle = { ...unsigned, payloadHash, signature: { algorithm: "ed25519", keyId, publicKeyPem, signatureBase64: sign(null, Buffer.from(payloadHash), privateKey).toString("base64") } };
1786
+ writeFileSync(outputPath, `${JSON.stringify(bundle, null, 2)}
1787
+ `, "utf8");
1788
+ return bundle;
1789
+ };
1790
+ var verifyEvidenceBundle = (path, { trustedKeys = [] } = {}) => {
1791
+ const bundle = parseBundle(path);
1792
+ if (bundle.type !== "agentskit-harness-evidence-bundle" || bundle.schemaVersion !== EVIDENCE_BUNDLE_SCHEMA_VERSION || !bundle.runId || !validKeyId(bundle.signerKeyId) || !validDigest(bundle.payloadHash) || bundle.signature?.algorithm !== "ed25519" || bundle.signature.keyId !== bundle.signerKeyId || typeof bundle.signature.publicKeyPem !== "string" || typeof bundle.signature.signatureBase64 !== "string" || !Array.isArray(bundle.files)) fail("Evidence bundle metadata is invalid.", "HARNESS_ERROR");
1793
+ if (trustedKeys.length) {
1794
+ const trusted = trustedKeys.find((key) => key.keyId === bundle.signerKeyId);
1795
+ if (!trusted) return fail(`Evidence bundle key is not trusted: ${bundle.signerKeyId}`, "HARNESS_ERROR");
1796
+ if (trusted.status === "revoked") fail(`Evidence bundle key is revoked: ${bundle.signerKeyId}`, "HARNESS_ERROR");
1797
+ if (trusted.publicKeyPem !== bundle.signature.publicKeyPem) fail(`Evidence bundle key does not match trust store: ${bundle.signerKeyId}`, "HARNESS_ERROR");
1798
+ }
1799
+ const paths = /* @__PURE__ */ new Set();
1800
+ for (const file of bundle.files) {
1801
+ if (!file || typeof file.path !== "string" || paths.has(file.path) || !validDigest(file.sha256) || typeof file.contentBase64 !== "string") fail("Evidence bundle file metadata is invalid.", "HARNESS_ERROR");
1802
+ paths.add(file.path);
1803
+ const content = Buffer.from(file.contentBase64, "base64");
1804
+ if (sha256(content) !== file.sha256) fail(`Evidence bundle file hash mismatch: ${file.path}`, "HARNESS_ERROR");
1805
+ }
1806
+ if (!paths.has(`runs/${bundle.runId}/run.json`) || !paths.has(`runs/${bundle.runId}/events.ndjson`)) fail("Evidence bundle is missing the run projection or event log.", "HARNESS_ERROR");
1807
+ if (sha256(JSON.stringify(body(bundle))) !== bundle.payloadHash) fail("Evidence bundle payload hash mismatch.", "HARNESS_ERROR");
1808
+ let valid = false;
1809
+ try {
1810
+ valid = verify(null, Buffer.from(bundle.payloadHash), createPublicKey(bundle.signature.publicKeyPem), Buffer.from(bundle.signature.signatureBase64, "base64"));
1811
+ } catch {
1812
+ valid = false;
1813
+ }
1814
+ if (!valid) fail("Evidence bundle signature is invalid.", "HARNESS_ERROR");
1815
+ return { status: "verified", runId: bundle.runId, payloadHash: bundle.payloadHash, fileCount: bundle.files.length, signed: true };
1816
+ };
1817
+ var readEvidenceTrustStore = (path) => {
1818
+ const value = readJson(path);
1819
+ if (value.schemaVersion !== 1 || !Array.isArray(value.keys)) fail("Evidence trust store must contain schemaVersion 1 and a keys array.", "INVALID_INPUT");
1820
+ return value.keys.map((key, index2) => {
1821
+ if (typeof key !== "object" || key === null || Array.isArray(key) || !validKeyId(key["keyId"]) || typeof key["publicKeyPem"] !== "string" || key["status"] !== "active" && key["status"] !== "revoked") fail(`Invalid evidence trust store key at index ${index2}.`, "INVALID_INPUT");
1822
+ return key;
1823
+ });
1824
+ };
1825
+
1826
+ export { BENCHMARK_SCHEMA_VERSION, CONTEXT_PROVIDER_SLOT, EVENT_LOG_GENESIS, EVIDENCE_BUNDLE_SCHEMA_VERSION, FileEventStore, HARNESS_EVENT_SCHEMA_VERSION, HARNESS_EVENT_TYPES, HARNESS_PLUGIN_API_VERSION, HarnessError, LEGAL_TRANSITIONS, STATES, approveRun, approvedDecision, assertHuman, authorizeRun, benchmarkRuns, cancelRun, cleanTaskArtifacts, createDocBridgeContextProvider, createDockerToolRuntime, createPluginRegistry, createPluginSlot, createPolicyGate, createProcessToolRuntime, createSessionRecorder, createToolRuntime, exportEvidenceBundle, hashContextSnapshot, hashContextSnapshots, inspectEventLogLock, loadBenchmarkManifest, loadConfig, loadLatestRun, planRun, readContextSnapshots, readEvidenceTrustStore, reconcileRun, recordBenchmarkObservation, recoverEventLogLock, retryRun, startRun, transition, validateBenchmarkManifest, validateConfig, validateContextSnapshot, validateContextSnapshots, validateExternalCodingBenchmarkReport, verifyEvidenceBundle, verifyRun };
1827
+ //# sourceMappingURL=index.js.map
1828
+ //# sourceMappingURL=index.js.map