@agentskit/harness 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -32
- package/CONTRIBUTING.md +60 -12
- package/MANIFESTO.md +23 -0
- package/README.md +199 -142
- package/dist/cli.js +796 -221
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +739 -121
- package/dist/index.js +1179 -303
- package/dist/index.js.map +1 -1
- package/docs/ADR-0025-portable-orchestration-controls.md +27 -0
- package/docs/ORGANIZATION.md +37 -0
- package/package.json +41 -34
- package/docs/ADR-0025-ci-dogfood.md +0 -22
- package/docs/ADR-0026-ci-evidence-artifact.md +0 -22
- package/docs/ADR-0027-portable-evidence.md +0 -19
- package/docs/ADR-0028-effective-metrics.md +0 -20
- package/docs/ADR-0029-honest-ci-preparation.md +0 -20
- package/docs/ADR-0030-agentskit-os-benchmark-bridge.md +0 -20
- package/docs/ADR-0031-real-provider-baseline.md +0 -18
- package/docs/ADR-0032-harness-equivalent-benchmark.md +0 -25
- package/docs/ADR-0033-portable-agent-gate.md +0 -25
- package/docs/ADR-0034-measurement-quality-gates.md +0 -25
- package/docs/ADR-0035-reproducible-benchmark-samples.md +0 -20
- package/docs/ADR-0036-comparable-baseline-samples.md +0 -20
- package/docs/ADR-0037-replicated-baseline-collection.md +0 -27
- package/docs/ADR-0038-end-to-end-benchmark-boundary.md +0 -28
- package/docs/ADR-0039-artifact-and-protocol-metrics.md +0 -39
- package/docs/ADR-0040-benchmark-corpus-surfaces.md +0 -32
package/dist/index.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { resolve, dirname, join, relative, sep } from 'path';
|
|
2
|
-
import { randomUUID, createPrivateKey, createPublicKey, sign, verify
|
|
3
|
-
import { existsSync, mkdirSync, openSync, writeSync, appendFileSync, closeSync, unlinkSync, readFileSync, writeFileSync, mkdtempSync, renameSync, rmSync,
|
|
1
|
+
import { resolve, dirname, join, relative, basename, extname, sep } from 'path';
|
|
2
|
+
import { createHash, randomUUID, createPrivateKey, createPublicKey, sign, verify } from 'crypto';
|
|
3
|
+
import { existsSync, mkdirSync, openSync, writeSync, appendFileSync, closeSync, unlinkSync, readFileSync, writeFileSync, mkdtempSync, renameSync, rmSync, readdirSync } from 'fs';
|
|
4
4
|
import { execFile, spawn } from 'child_process';
|
|
5
5
|
import { promisify } from 'util';
|
|
6
|
-
import { tmpdir } from 'os';
|
|
6
|
+
import { cpus, loadavg, freemem, totalmem, tmpdir } from 'os';
|
|
7
7
|
|
|
8
8
|
// src/constants.ts
|
|
9
9
|
var STATES = [
|
|
@@ -23,7 +23,7 @@ var LEGAL_TRANSITIONS = {
|
|
|
23
23
|
CLARIFYING: ["PLANNED", "BLOCKED", "CANCELLED"],
|
|
24
24
|
PLANNED: ["IMPLEMENTING", "CLARIFYING", "STALE", "CANCELLED"],
|
|
25
25
|
IMPLEMENTING: ["VERIFYING", "CLARIFYING", "STALE", "CANCELLED"],
|
|
26
|
-
VERIFYING: ["AWAITING_HUMAN_APPROVAL", "BLOCKED", "STALE", "CANCELLED"],
|
|
26
|
+
VERIFYING: ["AWAITING_HUMAN_APPROVAL", "COMPLETE", "BLOCKED", "STALE", "CANCELLED"],
|
|
27
27
|
AWAITING_HUMAN_APPROVAL: ["AWAITING_AUTHORIZATION", "COMPLETE", "BLOCKED", "IMPLEMENTING", "STALE", "CANCELLED"],
|
|
28
28
|
AWAITING_AUTHORIZATION: ["COMPLETE", "BLOCKED", "IMPLEMENTING", "STALE", "CANCELLED"],
|
|
29
29
|
COMPLETE: ["STALE", "SUPERSEDED"],
|
|
@@ -61,9 +61,11 @@ var id = (value, label) => {
|
|
|
61
61
|
var parents = (value, label) => value === void 0 ? [] : Array.isArray(value) ? value.map((item, index2) => id(item, `${label}[${index2}]`)) : [id(value, label)];
|
|
62
62
|
var merge = (base, overlay) => {
|
|
63
63
|
const result = { ...base };
|
|
64
|
-
for (const key of ["surfaces", "budget", "cleanup"]) {
|
|
64
|
+
for (const key of ["surfaces", "budget", "cleanup", "runtime", "verification"]) {
|
|
65
65
|
if (overlay[key] !== void 0) result[key] = { ...isRecord(result[key]) ? result[key] : {}, ...record(overlay[key], `profile.${key}`) };
|
|
66
66
|
}
|
|
67
|
+
if (overlay["autonomy"] !== void 0) result["autonomy"] = overlay["autonomy"];
|
|
68
|
+
if (Array.isArray(overlay["checks"])) result["checks"] = overlay["checks"];
|
|
67
69
|
if (overlay["checkOverrides"] !== void 0) {
|
|
68
70
|
if (!Array.isArray(overlay["checkOverrides"])) fail("profile.checkOverrides must be an array.", "INVALID_CONFIG");
|
|
69
71
|
const checks = Array.isArray(result["checks"]) ? [...result["checks"]] : [];
|
|
@@ -181,18 +183,20 @@ var surface = (value, name) => {
|
|
|
181
183
|
var parseCheck = (value, index2) => {
|
|
182
184
|
const record3 = asRecord(value, `checks[${index2}]`);
|
|
183
185
|
const id2 = stringValue(record3["id"], `checks[${index2}].id`);
|
|
184
|
-
const
|
|
185
|
-
if (!CHECK_CATEGORIES.includes(
|
|
186
|
+
const category2 = stringValue(record3["category"], `checks[${index2}].category`);
|
|
187
|
+
if (!CHECK_CATEGORIES.includes(category2)) fail(`checks[${index2}].category is invalid.`, "INVALID_CONFIG");
|
|
186
188
|
const command = stringValue(record3["command"], `checks[${index2}].command`);
|
|
187
|
-
if (REAL_CATEGORIES.has(
|
|
189
|
+
if (REAL_CATEGORIES.has(category2) && record3["execution"] !== "real") fail(`checks[${index2}] must declare execution: real.`, "INVALID_CONFIG");
|
|
188
190
|
if (record3["evidence"] !== "structured") fail(`checks[${index2}] must declare evidence: structured.`, "INVALID_CONFIG");
|
|
189
191
|
if (record3["capabilities"] !== void 0 && (!Array.isArray(record3["capabilities"]) || !record3["capabilities"].every((item) => typeof item === "string"))) fail(`checks[${index2}].capabilities must be strings.`, "INVALID_CONFIG");
|
|
190
192
|
const capabilities = Array.isArray(record3["capabilities"]) ? record3["capabilities"].filter((item) => typeof item === "string") : void 0;
|
|
191
|
-
if (
|
|
192
|
-
if (
|
|
193
|
+
if (category2 === "ui" && !capabilities?.includes("real-browser")) fail(`checks[${index2}] must declare real-browser.`, "INVALID_CONFIG");
|
|
194
|
+
if (category2 === "ui" && !capabilities?.includes("screenshot")) fail(`checks[${index2}] must declare screenshot.`, "INVALID_CONFIG");
|
|
193
195
|
if (record3["required"] !== void 0 && typeof record3["required"] !== "boolean") fail(`checks[${index2}].required must be boolean.`, "INVALID_CONFIG");
|
|
196
|
+
if (record3["required"] === false && typeof record3["reason"] !== "string") fail(`checks[${index2}].reason is required when required is false.`, "INVALID_CONFIG");
|
|
194
197
|
if (record3["timeoutMs"] !== void 0 && (!Number.isInteger(record3["timeoutMs"]) || typeof record3["timeoutMs"] !== "number" || record3["timeoutMs"] < 1)) fail(`checks[${index2}].timeoutMs must be positive.`, "INVALID_CONFIG");
|
|
195
|
-
|
|
198
|
+
const dependsOn = record3["dependsOn"] === void 0 ? void 0 : stringArray(record3["dependsOn"], `checks[${index2}].dependsOn`);
|
|
199
|
+
return { id: id2, category: category2, command, required: record3["required"] !== false, ...typeof record3["reason"] === "string" ? { reason: record3["reason"] } : {}, timeoutMs: typeof record3["timeoutMs"] === "number" ? record3["timeoutMs"] : 12e4, ...record3["execution"] === "real" ? { execution: "real" } : {}, ...capabilities ? { capabilities } : {}, ...dependsOn ? { dependsOn: [...new Set(dependsOn)] } : {}, evidence: "structured" };
|
|
196
200
|
};
|
|
197
201
|
var parseOutcome = (value, index2, checks) => {
|
|
198
202
|
const record3 = asRecord(value, `contract.outcomes[${index2}]`);
|
|
@@ -206,10 +210,28 @@ var validateConfig = (rawValue) => {
|
|
|
206
210
|
const raw = resolveProfile(asRecord(rawValue, "verification config"));
|
|
207
211
|
if (raw["schemaVersion"] !== 1) fail("verification config schemaVersion must be 1.", "INVALID_CONFIG");
|
|
208
212
|
const project = stringValue(raw["project"], "verification config project");
|
|
213
|
+
const runtimeRaw = asRecord(raw["runtime"] ?? { kind: "process" }, "runtime");
|
|
214
|
+
if (runtimeRaw["kind"] !== "process" && runtimeRaw["kind"] !== "docker") fail("runtime.kind must be process or docker.", "INVALID_CONFIG");
|
|
215
|
+
const runtime = { kind: runtimeRaw["kind"] };
|
|
216
|
+
if (raw["autonomy"] !== void 0 && raw["autonomy"] !== "controlled" && raw["autonomy"] !== "yolo") fail("autonomy must be controlled or yolo.", "INVALID_CONFIG");
|
|
217
|
+
const autonomy = raw["autonomy"] ?? "controlled";
|
|
209
218
|
const contractRaw = asRecord(raw["contract"], "contract");
|
|
210
219
|
const rawChecks = raw["checks"];
|
|
211
220
|
const checks = Array.isArray(rawChecks) ? rawChecks.map(parseCheck) : fail("checks must be a non-empty array.", "INVALID_CONFIG");
|
|
212
221
|
if (!checks.length || new Set(checks.map((check) => check.id)).size !== checks.length) fail("check ids must be unique.", "INVALID_CONFIG");
|
|
222
|
+
const checkIds = new Set(checks.map((check) => check.id));
|
|
223
|
+
for (const check of checks) for (const dependency of check.dependsOn ?? []) if (!checkIds.has(dependency) || dependency === check.id) fail(`check ${check.id} has an invalid dependency: ${dependency}.`, "INVALID_CONFIG");
|
|
224
|
+
const visiting = /* @__PURE__ */ new Set();
|
|
225
|
+
const visited = /* @__PURE__ */ new Set();
|
|
226
|
+
const visit = (id2) => {
|
|
227
|
+
if (visiting.has(id2)) fail(`check dependency cycle includes ${id2}.`, "INVALID_CONFIG");
|
|
228
|
+
if (visited.has(id2)) return;
|
|
229
|
+
visiting.add(id2);
|
|
230
|
+
for (const dependency of checks.find((check) => check.id === id2)?.dependsOn ?? []) visit(dependency);
|
|
231
|
+
visiting.delete(id2);
|
|
232
|
+
visited.add(id2);
|
|
233
|
+
};
|
|
234
|
+
for (const check of checks) visit(check.id);
|
|
213
235
|
const scopeRaw = asRecord(contractRaw["scope"], "contract.scope");
|
|
214
236
|
const scope = { inScope: stringArray(scopeRaw["inScope"], "contract.scope.inScope"), outOfScope: stringArray(scopeRaw["outOfScope"], "contract.scope.outOfScope") };
|
|
215
237
|
const ambiguities = stringArray(contractRaw["ambiguities"], "contract.ambiguities");
|
|
@@ -226,13 +248,15 @@ var validateConfig = (rawValue) => {
|
|
|
226
248
|
if (trackingRaw["required"] !== true && typeof trackingRaw["reason"] !== "string") fail("tracking.reason is required when tracking is disabled.", "INVALID_CONFIG");
|
|
227
249
|
const budgetRaw = raw["budget"] === void 0 ? void 0 : asRecord(raw["budget"], "budget");
|
|
228
250
|
if (budgetRaw && budgetRaw["maxDurationMs"] !== void 0 && (!Number.isInteger(budgetRaw["maxDurationMs"]) || typeof budgetRaw["maxDurationMs"] !== "number" || budgetRaw["maxDurationMs"] < 1)) fail("budget.maxDurationMs must be positive.", "INVALID_CONFIG");
|
|
251
|
+
const verificationRaw = raw["verification"] === void 0 ? void 0 : asRecord(raw["verification"], "verification");
|
|
252
|
+
if (verificationRaw && verificationRaw["maxConcurrency"] !== void 0 && (!Number.isInteger(verificationRaw["maxConcurrency"]) || typeof verificationRaw["maxConcurrency"] !== "number" || verificationRaw["maxConcurrency"] < 1)) fail("verification.maxConcurrency must be a positive integer.", "INVALID_CONFIG");
|
|
229
253
|
const cleanupRaw = raw["cleanup"] === void 0 ? void 0 : asRecord(raw["cleanup"], "cleanup");
|
|
230
254
|
const cleanup = cleanupRaw ? { roots: cleanupRaw["roots"] === void 0 ? void 0 : stringArray(cleanupRaw["roots"], "cleanup.roots") } : void 0;
|
|
231
255
|
const benchmarkRaw = raw["benchmark"] === void 0 ? void 0 : asRecord(raw["benchmark"], "benchmark");
|
|
232
256
|
const benchmark = benchmarkRaw ? { suiteId: stringValue(benchmarkRaw["suiteId"], "benchmark.suiteId"), taskId: stringValue(benchmarkRaw["taskId"], "benchmark.taskId"), mode: benchmarkRaw["mode"] === "harness" ? "harness" : fail("benchmark.mode must be harness.", "INVALID_CONFIG") } : void 0;
|
|
233
257
|
const contract = { intent: stringValue(contractRaw["intent"], "contract.intent"), scope, ambiguities, outcomes };
|
|
234
258
|
const tracking = { required: trackingRaw["required"] === true, ...typeof trackingRaw["target"] === "string" ? { target: trackingRaw["target"] } : {}, ...typeof trackingRaw["reason"] === "string" ? { reason: trackingRaw["reason"] } : {} };
|
|
235
|
-
return { schemaVersion: 1, project, ...typeof raw["root"] === "string" ? { root: raw["root"] } : {}, ...typeof raw["stateDir"] === "string" ? { stateDir: raw["stateDir"] } : {}, profile: typeof raw["profile"] === "string" ? raw["profile"] : "strict", contract, surfaces, checks, tracking, ...budgetRaw ? { budget: { maxDurationMs: budgetRaw["maxDurationMs"] } } : {}, ...cleanup ? { cleanup } : {}, ...benchmark ? { benchmark } : {} };
|
|
259
|
+
return { schemaVersion: 1, project, ...typeof raw["root"] === "string" ? { root: raw["root"] } : {}, ...typeof raw["stateDir"] === "string" ? { stateDir: raw["stateDir"] } : {}, profile: typeof raw["profile"] === "string" ? raw["profile"] : "strict", runtime, autonomy, contract, surfaces, checks, tracking, ...verificationRaw ? { verification: { maxConcurrency: verificationRaw["maxConcurrency"] } } : {}, ...budgetRaw ? { budget: { maxDurationMs: budgetRaw["maxDurationMs"] } } : {}, ...cleanup ? { cleanup } : {}, ...benchmark ? { benchmark } : {} };
|
|
236
260
|
};
|
|
237
261
|
var loadConfig = (configPath = ".codex/verification.json") => {
|
|
238
262
|
const absolute = resolve(configPath);
|
|
@@ -240,7 +264,7 @@ var loadConfig = (configPath = ".codex/verification.json") => {
|
|
|
240
264
|
const rawRecord = asRecord(raw, "verification config");
|
|
241
265
|
const root = resolve(dirname(absolute), typeof rawRecord["root"] === "string" ? rawRecord["root"] : ".");
|
|
242
266
|
const stateDir = resolve(root, typeof rawRecord["stateDir"] === "string" ? rawRecord["stateDir"] : ".codex/verification");
|
|
243
|
-
if (
|
|
267
|
+
if (stateDir === root) fail("stateDir must be separate from the project root.", "INVALID_CONFIG");
|
|
244
268
|
const config = validateConfig(raw);
|
|
245
269
|
return { absolute, root, stateDir, config, configHash: hashJson(config) };
|
|
246
270
|
};
|
|
@@ -289,7 +313,9 @@ var eventDigest = (event) => sha256(JSON.stringify(eventBody(event)));
|
|
|
289
313
|
var parseEvent = (value, expectedSequence) => {
|
|
290
314
|
if (typeof value !== "object" || value === null || Array.isArray(value)) fail("Event log contains a non-object record.", "HARNESS_ERROR");
|
|
291
315
|
const record3 = value;
|
|
292
|
-
|
|
316
|
+
const context = record3["correlation"];
|
|
317
|
+
const validContext = context === void 0 || typeof context === "object" && context !== null && !Array.isArray(context) && Object.entries(context).every(([key, value2]) => ["operationId", "runId", "sessionId", "turnId", "actionId", "traceId"].includes(key) && typeof value2 === "string" && /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(value2)) && typeof context["operationId"] === "string";
|
|
318
|
+
if (record3["schemaVersion"] !== HARNESS_EVENT_SCHEMA_VERSION || typeof record3["runId"] !== "string" || typeof record3["sequence"] !== "number" || record3["sequence"] !== expectedSequence || typeof record3["at"] !== "string" || typeof record3["sourceRevision"] !== "string" || typeof record3["configHash"] !== "string" || !isEventType(record3["type"]) || typeof record3["payload"] !== "object" || record3["payload"] === null || record3["sessionId"] !== void 0 && (typeof record3["sessionId"] !== "string" || !record3["sessionId"].trim()) || SESSION_EVENT_TYPES.has(record3["type"]) && typeof record3["sessionId"] !== "string" || !validContext) fail("Event log is invalid or out of order.", "HARNESS_ERROR");
|
|
293
319
|
const hasPreviousHash = record3["previousHash"] !== void 0;
|
|
294
320
|
const hasEventHash = record3["eventHash"] !== void 0;
|
|
295
321
|
if (hasPreviousHash !== hasEventHash || hasPreviousHash && (typeof record3["previousHash"] !== "string" || record3["previousHash"] !== EVENT_LOG_GENESIS && !digest(record3["previousHash"]) || typeof record3["eventHash"] !== "string" || !digest(record3["eventHash"]))) fail("Event log integrity metadata is invalid.", "HARNESS_ERROR");
|
|
@@ -318,6 +344,7 @@ var FileEventStore = class {
|
|
|
318
344
|
if (!isEventType(event.type)) fail("Event type is invalid.", "INVALID_INPUT");
|
|
319
345
|
if (SESSION_EVENT_TYPES.has(event.type) && (!event.sessionId || !event.sessionId.trim())) fail("Session events require a sessionId.", "INVALID_INPUT");
|
|
320
346
|
if (event.sessionId !== void 0 && !event.sessionId.trim()) fail("Event sessionId cannot be empty.", "INVALID_INPUT");
|
|
347
|
+
if (event.correlation !== void 0 && (!event.correlation.operationId || !event.correlation.operationId.trim())) fail("Event correlation operationId is required.", "INVALID_INPUT");
|
|
321
348
|
const path = eventPath(this.stateDir, event.runId);
|
|
322
349
|
const lock = lockPath(this.stateDir, event.runId);
|
|
323
350
|
mkdirSync(join(this.stateDir, "runs", event.runId), { recursive: true });
|
|
@@ -332,7 +359,7 @@ var FileEventStore = class {
|
|
|
332
359
|
try {
|
|
333
360
|
const events = this.readUnlocked(event.runId);
|
|
334
361
|
const previous = events.at(-1);
|
|
335
|
-
const body2 = { schemaVersion: HARNESS_EVENT_SCHEMA_VERSION, sequence: events.length + 1, at: (/* @__PURE__ */ new Date()).toISOString(), runId: event.runId, sourceRevision: event.sourceRevision, configHash: event.configHash, ...event.sessionId ? { sessionId: event.sessionId } : {}, ...previous?.eventHash ? { previousHash: previous.eventHash } : events.length ? {} : { previousHash: EVENT_LOG_GENESIS }, type: event.type, payload: event.payload };
|
|
362
|
+
const body2 = { schemaVersion: HARNESS_EVENT_SCHEMA_VERSION, sequence: events.length + 1, at: (/* @__PURE__ */ new Date()).toISOString(), runId: event.runId, sourceRevision: event.sourceRevision, configHash: event.configHash, ...event.correlation ? { correlation: event.correlation } : {}, ...event.sessionId ? { sessionId: event.sessionId } : {}, ...previous?.eventHash ? { previousHash: previous.eventHash } : events.length ? {} : { previousHash: EVENT_LOG_GENESIS }, type: event.type, payload: event.payload };
|
|
336
363
|
const record3 = events.length && !previous?.eventHash ? body2 : { ...body2, eventHash: eventDigest(body2) };
|
|
337
364
|
appendFileSync(path, `${JSON.stringify(record3)}
|
|
338
365
|
`, "utf8");
|
|
@@ -578,8 +605,7 @@ var saveRun2 = (stateDir, run) => {
|
|
|
578
605
|
store.append({ runId: run.runId, sourceRevision: run.sourceRevision, configHash: run.configHash, type: "context.attached", payload: { providerId: snapshot.providerId, sourceHash: snapshot.sourceHash, snapshotHash: snapshot.snapshotHash, query: snapshot.query } });
|
|
579
606
|
}
|
|
580
607
|
};
|
|
581
|
-
var createRun = async ({ loaded, baseline, supersedes, dirtyBaselineAuthorized, contextSnapshots = []
|
|
582
|
-
const contractHash = hashJson(loaded.config.contract);
|
|
608
|
+
var createRun = async ({ loaded, baseline, supersedes, dirtyBaselineAuthorized, contextSnapshots = [] }) => {
|
|
583
609
|
const run = {
|
|
584
610
|
type: "agentskit-harness-run",
|
|
585
611
|
schemaVersion: 1,
|
|
@@ -587,17 +613,18 @@ var createRun = async ({ loaded, baseline, supersedes, dirtyBaselineAuthorized,
|
|
|
587
613
|
project: loaded.config.project,
|
|
588
614
|
state: "PLANNED",
|
|
589
615
|
configHash: loaded.configHash,
|
|
590
|
-
contractHash,
|
|
616
|
+
contractHash: hashJson(loaded.config.contract),
|
|
591
617
|
sourceRevision: baseline.revision,
|
|
592
618
|
sourceStatusHash: baseline.statusHash,
|
|
593
619
|
baseline,
|
|
594
|
-
|
|
595
|
-
|
|
620
|
+
autonomy: loaded.config.autonomy,
|
|
621
|
+
contractApproval: { actor: "human", at: now(), contractHash: hashJson(loaded.config.contract) },
|
|
622
|
+
checks: loaded.config.checks.map(({ id: id2, category: category2 }) => ({ id: id2, category: category2, status: "pending" })),
|
|
596
623
|
contextSnapshots,
|
|
597
624
|
...contextSnapshots.length ? { contextHash: hashContextSnapshots(contextSnapshots) } : {},
|
|
598
625
|
...loaded.config.benchmark ? { benchmark: loaded.config.benchmark } : {},
|
|
599
626
|
outcomes: loaded.config.contract.outcomes.map(({ id: id2, statement, checks }) => ({ id: id2, statement, checks, status: "pending" })),
|
|
600
|
-
transitions: [{ from: null, to: "PLANNED", at: now(), actor:
|
|
627
|
+
transitions: [{ from: null, to: "PLANNED", at: now(), actor: "human" }],
|
|
601
628
|
evidenceReferences: [],
|
|
602
629
|
...supersedes ? { supersedes } : {},
|
|
603
630
|
...dirtyBaselineAuthorized ? { dirtyBaselineAuthorized: true } : {}
|
|
@@ -650,22 +677,97 @@ var git = async (root, args) => {
|
|
|
650
677
|
};
|
|
651
678
|
var sourceSnapshot = async (root, stateDir) => {
|
|
652
679
|
const revision = await git(root, ["rev-parse", "HEAD"]);
|
|
680
|
+
if (!revision) fail("Current-source evidence requires a Git repository with a committed HEAD.", "GIT_REQUIRED");
|
|
653
681
|
const stateRelative = relative(root, stateDir).replaceAll("\\", "/");
|
|
654
682
|
const pathspec = ["--", "."];
|
|
655
683
|
if (stateRelative && stateRelative !== ".." && !stateRelative.startsWith("../")) pathspec.push(`:(exclude)${stateRelative}`);
|
|
656
684
|
const status = await git(root, ["status", "--porcelain=v1", "--untracked-files=all", ...pathspec]);
|
|
657
685
|
const diff = await git(root, ["diff", "--no-ext-diff", "--binary", "HEAD", ...pathspec]);
|
|
658
686
|
const untrackedPaths = (await git(root, ["ls-files", "--others", "--exclude-standard", "-z"])).split("\0").filter(Boolean).filter((path) => !stateRelative || path !== stateRelative && !path.startsWith(`${stateRelative}/`));
|
|
659
|
-
const untracked = untrackedPaths.
|
|
660
|
-
const absolute = resolve(root, path);
|
|
661
|
-
try {
|
|
662
|
-
return lstatSync(absolute).isFile() ? [{ path, hash: sha256(readFileSync(absolute)) }] : [];
|
|
663
|
-
} catch {
|
|
664
|
-
return [];
|
|
665
|
-
}
|
|
666
|
-
});
|
|
687
|
+
const untracked = untrackedPaths.map((path) => ({ path, hash: sha256(readFileSync(resolve(root, path))) }));
|
|
667
688
|
const fingerprint = { revision, status, diff, untracked };
|
|
668
|
-
return { revision
|
|
689
|
+
return { revision, status, statusHash: hashJson(fingerprint) };
|
|
690
|
+
};
|
|
691
|
+
var thresholds = (value = {}) => {
|
|
692
|
+
const result = { warningPercent: value.warningPercent ?? 75, criticalPercent: value.criticalPercent ?? 90 };
|
|
693
|
+
if (![result.warningPercent, result.criticalPercent].every((item) => Number.isFinite(item) && item >= 0 && item <= 100) || result.warningPercent > result.criticalPercent) fail("Machine thresholds must be between 0 and 100 and warning must not exceed critical.", "INVALID_INPUT");
|
|
694
|
+
return result;
|
|
695
|
+
};
|
|
696
|
+
var linuxSwap = () => {
|
|
697
|
+
if (process.platform !== "linux" || !existsSync("/proc/meminfo")) return void 0;
|
|
698
|
+
const values = Object.fromEntries(readFileSync("/proc/meminfo", "utf8").split(/\r?\n/).flatMap((line) => {
|
|
699
|
+
const match = line.match(/^(SwapTotal|SwapFree):\s+(\d+)\s+kB$/);
|
|
700
|
+
return match ? [[match[1], Number(match[2])]] : [];
|
|
701
|
+
}));
|
|
702
|
+
if (!values["SwapTotal"]) return void 0;
|
|
703
|
+
return Number(((1 - (values["SwapFree"] ?? 0) / values["SwapTotal"]) * 100).toFixed(2));
|
|
704
|
+
};
|
|
705
|
+
var sampleMachine = () => {
|
|
706
|
+
const cpus$1 = Math.max(1, cpus().length);
|
|
707
|
+
const load1 = Math.max(0, loadavg()[0] ?? 0);
|
|
708
|
+
const memory = Math.max(0, Math.min(100, (1 - freemem() / Math.max(1, totalmem())) * 100));
|
|
709
|
+
const swapUsedPercent = linuxSwap();
|
|
710
|
+
return {
|
|
711
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
712
|
+
cpus: cpus$1,
|
|
713
|
+
load1: Number(load1.toFixed(4)),
|
|
714
|
+
load1PerCpuPercent: Number(Math.min(100, load1 / cpus$1 * 100).toFixed(2)),
|
|
715
|
+
memoryUsedPercent: Number(memory.toFixed(2)),
|
|
716
|
+
rssBytes: process.memoryUsage().rss,
|
|
717
|
+
...swapUsedPercent === void 0 ? {} : { swapUsedPercent }
|
|
718
|
+
};
|
|
719
|
+
};
|
|
720
|
+
var summarizeMachine = (samples, sampleIntervalMs = 5e3, limits = {}) => {
|
|
721
|
+
const limit = thresholds(limits);
|
|
722
|
+
const load = samples.map((sample) => sample.load1PerCpuPercent);
|
|
723
|
+
const memory = samples.map((sample) => sample.memoryUsedPercent);
|
|
724
|
+
const rss = samples.map((sample) => sample.rssBytes);
|
|
725
|
+
return {
|
|
726
|
+
sampleIntervalMs,
|
|
727
|
+
samples,
|
|
728
|
+
peakLoad1PerCpuPercent: Number(Math.max(...load, 0).toFixed(2)),
|
|
729
|
+
peakMemoryUsedPercent: Number(Math.max(...memory, 0).toFixed(2)),
|
|
730
|
+
peakRssBytes: Math.max(...rss, 0),
|
|
731
|
+
pressureEvents: samples.filter((sample) => sample.load1PerCpuPercent >= limit.criticalPercent || sample.memoryUsedPercent >= limit.criticalPercent || (sample.swapUsedPercent ?? 0) >= limit.criticalPercent || sample.memoryPressure === "critical").length,
|
|
732
|
+
throttleEvents: 0,
|
|
733
|
+
minimumEffectiveConcurrency: 0
|
|
734
|
+
};
|
|
735
|
+
};
|
|
736
|
+
var adaptiveConcurrency = (configured, sample, limits = {}) => {
|
|
737
|
+
if (!Number.isInteger(configured) || configured < 1) throw new Error("configured concurrency must be a positive integer.");
|
|
738
|
+
const limit = thresholds(limits);
|
|
739
|
+
const critical = sample.load1PerCpuPercent >= limit.criticalPercent || sample.memoryUsedPercent >= limit.criticalPercent || (sample.swapUsedPercent ?? 0) >= limit.criticalPercent || sample.memoryPressure === "critical";
|
|
740
|
+
const warning = sample.load1PerCpuPercent >= limit.warningPercent || sample.memoryUsedPercent >= limit.warningPercent || (sample.swapUsedPercent ?? 0) >= limit.warningPercent || sample.memoryPressure === "warning";
|
|
741
|
+
if (critical) return 1;
|
|
742
|
+
if (warning) return Math.min(configured, 2);
|
|
743
|
+
return configured;
|
|
744
|
+
};
|
|
745
|
+
var createMachineMonitor = (sampleIntervalMs = 5e3, options = {}) => {
|
|
746
|
+
const sampler = options.sample ?? sampleMachine;
|
|
747
|
+
const limits = thresholds(options.thresholds);
|
|
748
|
+
const samples = [sampler()];
|
|
749
|
+
let throttleEvents = 0;
|
|
750
|
+
const effectiveConcurrency = [];
|
|
751
|
+
const record3 = () => {
|
|
752
|
+
const sample = sampler();
|
|
753
|
+
samples.push(sample);
|
|
754
|
+
return sample;
|
|
755
|
+
};
|
|
756
|
+
const timer = setInterval(record3, sampleIntervalMs);
|
|
757
|
+
timer.unref();
|
|
758
|
+
return {
|
|
759
|
+
sample: record3,
|
|
760
|
+
observeConcurrency: (value) => effectiveConcurrency.push(value),
|
|
761
|
+
markThrottle: () => {
|
|
762
|
+
throttleEvents += 1;
|
|
763
|
+
},
|
|
764
|
+
stop: () => {
|
|
765
|
+
clearInterval(timer);
|
|
766
|
+
record3();
|
|
767
|
+
const summary = summarizeMachine(samples, sampleIntervalMs, limits);
|
|
768
|
+
return { ...summary, throttleEvents, minimumEffectiveConcurrency: effectiveConcurrency.length ? Math.min(...effectiveConcurrency) : 0 };
|
|
769
|
+
}
|
|
770
|
+
};
|
|
669
771
|
};
|
|
670
772
|
|
|
671
773
|
// src/verification.ts
|
|
@@ -694,6 +796,12 @@ var runCommand = (check, cwd) => new Promise((resolveResult) => {
|
|
|
694
796
|
resolveResult({ exitCode: exitCode ?? 1, timedOut, stdout, stderr, durationMs: Date.now() - started });
|
|
695
797
|
});
|
|
696
798
|
});
|
|
799
|
+
var executeCheck = async (check, cwd, checkDir, outcomes) => {
|
|
800
|
+
const result = await runCommand(check, cwd);
|
|
801
|
+
const evidence = parseStructuredEvidence(result.stdout);
|
|
802
|
+
const failures = result.exitCode === 0 && !result.timedOut && evidence ? validateEvidence(cwd, check, evidence, outcomes) : [result.timedOut ? "check timed out" : result.exitCode !== 0 ? `exit code ${result.exitCode}` : "missing final structured evidence"];
|
|
803
|
+
return { check: { id: check.id, category: check.category, status: failures.length ? "failed" : "passed", exitCode: result.exitCode, durationMs: result.durationMs, ...evidence ? { evidence } : {}, ...failures.length ? { failures } : {} }, stdout: result.stdout, stderr: result.stderr, durationMs: result.durationMs };
|
|
804
|
+
};
|
|
697
805
|
var currentBinding = async (loaded) => ({ source: await sourceSnapshot(loaded.root, loaded.stateDir), configHash: loaded.configHash });
|
|
698
806
|
var staleRun = (loaded, run, reason) => {
|
|
699
807
|
const stale = transition(run, "STALE", reason);
|
|
@@ -706,11 +814,8 @@ var isFresh = async (loaded, run) => {
|
|
|
706
814
|
return current.configHash === run.configHash && current.source.revision === run.sourceRevision && current.source.statusHash === run.sourceStatusHash;
|
|
707
815
|
};
|
|
708
816
|
var planRun = async ({ configPath, decision, actor = "human", allowDirty = false, contextSnapshots = [] }) => {
|
|
709
|
-
|
|
710
|
-
if (!
|
|
711
|
-
assertHuman(actor);
|
|
712
|
-
if (!approvedDecision(decision)) fail("Contract was not approved.", "CLARIFYING");
|
|
713
|
-
}
|
|
817
|
+
assertHuman(actor);
|
|
818
|
+
if (!approvedDecision(decision)) fail("Contract was not approved.", "CLARIFYING");
|
|
714
819
|
const loaded = loadConfig(configPath);
|
|
715
820
|
if (loaded.config.contract.ambiguities.length) fail(`Unresolved ambiguities remain: ${loaded.config.contract.ambiguities.join(" | ")}`, "CLARIFYING");
|
|
716
821
|
const validatedContextSnapshots = validateContextSnapshots(contextSnapshots);
|
|
@@ -722,7 +827,7 @@ ${meaningful.join("\n")}
|
|
|
722
827
|
Use --allow-dirty only with explicit human authorization.`, "WORKTREE_DIRTY");
|
|
723
828
|
const previous = loadLatestRun(loaded.stateDir);
|
|
724
829
|
if (previous && !["STALE", "SUPERSEDED"].includes(previous.state)) fail(`An active run already exists: ${previous.runId} (${previous.state}).`, "ACTIVE_RUN");
|
|
725
|
-
return createRun({ loaded, baseline, supersedes: previous?.runId, dirtyBaselineAuthorized: allowDirty, contextSnapshots: validatedContextSnapshots
|
|
830
|
+
return createRun({ loaded, baseline, supersedes: previous?.runId, dirtyBaselineAuthorized: allowDirty, contextSnapshots: validatedContextSnapshots });
|
|
726
831
|
};
|
|
727
832
|
var startRun = (loaded) => {
|
|
728
833
|
const run = requireRun(loadLatestRun(loaded.stateDir));
|
|
@@ -742,42 +847,81 @@ var cancelRun = async ({ configPath, runId, reason = "Run cancelled by a human."
|
|
|
742
847
|
};
|
|
743
848
|
var verifyRun = async ({ configPath }) => {
|
|
744
849
|
const loaded = loadConfig(configPath);
|
|
850
|
+
const machineMonitor = createMachineMonitor();
|
|
745
851
|
const run = requireRun(loadLatestRun(loaded.stateDir));
|
|
746
852
|
if (!["IMPLEMENTING", "VERIFYING"].includes(run.state)) {
|
|
747
853
|
if (["AWAITING_HUMAN_APPROVAL", "AWAITING_AUTHORIZATION", "COMPLETE"].includes(run.state) && !await isFresh(loaded, run)) staleRun(loaded, run, "Run is stale because source or contract changed.");
|
|
748
854
|
fail(`Cannot verify from ${run.state}.`, "INVALID_STATE");
|
|
749
855
|
}
|
|
750
856
|
if (run.configHash !== loaded.configHash) staleRun(loaded, run, "Run is stale because the verification contract changed.");
|
|
751
|
-
const
|
|
752
|
-
let current = { ...transition(run, "VERIFYING", "Verification started.", "agent"), sourceRevision:
|
|
857
|
+
const binding2 = await currentBinding(loaded);
|
|
858
|
+
let current = { ...transition(run, "VERIFYING", "Verification started.", "agent"), sourceRevision: binding2.source.revision, sourceStatusHash: binding2.source.statusHash };
|
|
753
859
|
saveRun2(loaded.stateDir, current);
|
|
754
860
|
const checkDir = join(loaded.stateDir, "runs", current.runId, "checks");
|
|
755
861
|
mkdirSync(checkDir, { recursive: true });
|
|
756
862
|
const outcomesByCheck = new Map(loaded.config.checks.map((check) => [check.id, loaded.config.contract.outcomes.filter((outcome) => outcome.checks.includes(check.id)).map((outcome) => outcome.id)]));
|
|
757
863
|
let totalDurationMs = 0;
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
864
|
+
let activeChecks = 0;
|
|
865
|
+
let observedPeakConcurrency = 0;
|
|
866
|
+
const verificationStarted = Date.now();
|
|
867
|
+
const maxConcurrency = loaded.config.verification?.maxConcurrency ?? 1;
|
|
868
|
+
const requiredChecks = loaded.config.checks.filter((check) => check.required);
|
|
869
|
+
for (const check of loaded.config.checks.filter((item) => !item.required)) {
|
|
870
|
+
const nextCheck = { id: check.id, category: check.category, status: "not-applicable", failures: [check.reason ?? "Not required by the selected profile."] };
|
|
871
|
+
current = { ...current, checks: current.checks.map((item) => item.id === check.id ? nextCheck : item) };
|
|
872
|
+
}
|
|
873
|
+
const remaining = new Set(requiredChecks.map((check) => check.id));
|
|
874
|
+
const completed = new Set(loaded.config.checks.filter((check) => !check.required).map((check) => check.id));
|
|
875
|
+
while (remaining.size) {
|
|
876
|
+
const ready = requiredChecks.filter((check) => remaining.has(check.id) && (check.dependsOn ?? []).every((dependency) => completed.has(dependency)));
|
|
877
|
+
if (!ready.length) fail("Check dependency graph contains an unknown dependency or cycle.", "INVALID_CONFIG");
|
|
878
|
+
const buildChecks = ready.filter((check) => check.category === "build");
|
|
879
|
+
const queue = buildChecks.length ? buildChecks : ready;
|
|
880
|
+
let offset = 0;
|
|
881
|
+
while (offset < queue.length) {
|
|
882
|
+
const effectiveConcurrency = buildChecks.length ? 1 : adaptiveConcurrency(maxConcurrency, machineMonitor.sample());
|
|
883
|
+
machineMonitor.observeConcurrency(effectiveConcurrency);
|
|
884
|
+
if (effectiveConcurrency < maxConcurrency) machineMonitor.markThrottle();
|
|
885
|
+
const batch = queue.slice(offset, offset + effectiveConcurrency);
|
|
886
|
+
const executed = await Promise.all(batch.map(async (check) => {
|
|
887
|
+
activeChecks += 1;
|
|
888
|
+
observedPeakConcurrency = Math.max(observedPeakConcurrency, activeChecks);
|
|
889
|
+
try {
|
|
890
|
+
return await executeCheck(check, loaded.root, checkDir, outcomesByCheck.get(check.id) ?? []);
|
|
891
|
+
} finally {
|
|
892
|
+
activeChecks -= 1;
|
|
893
|
+
}
|
|
894
|
+
}));
|
|
895
|
+
for (const item of executed) {
|
|
896
|
+
totalDurationMs += item.durationMs;
|
|
897
|
+
const stdoutPath = join(checkDir, `${item.check.id}.stdout`);
|
|
898
|
+
const stderrPath = join(checkDir, `${item.check.id}.stderr`);
|
|
899
|
+
writeFileSync(stdoutPath, item.stdout, "utf8");
|
|
900
|
+
writeFileSync(stderrPath, item.stderr, "utf8");
|
|
901
|
+
current = { ...current, evidenceReferences: [...current.evidenceReferences, { checkId: item.check.id, stdout: relative(loaded.stateDir, stdoutPath), stderr: relative(loaded.stateDir, stderrPath) }], checks: current.checks.map((check) => check.id === item.check.id ? item.check : check) };
|
|
902
|
+
}
|
|
903
|
+
saveRun2(loaded.stateDir, current);
|
|
904
|
+
for (const check of batch) {
|
|
905
|
+
remaining.delete(check.id);
|
|
906
|
+
completed.add(check.id);
|
|
907
|
+
}
|
|
908
|
+
offset += batch.length;
|
|
909
|
+
}
|
|
770
910
|
}
|
|
771
911
|
const statuses = new Map(current.checks.map((check) => [check.id, check.status]));
|
|
772
912
|
const budgetExceeded = loaded.config.budget?.maxDurationMs !== void 0 && totalDurationMs > loaded.config.budget.maxDurationMs;
|
|
773
|
-
const allPassed = loaded.config.checks.every((check) => statuses.get(check.id) === "passed") && !budgetExceeded;
|
|
774
|
-
current = { ...current, outcomes: current.outcomes.map((outcome) =>
|
|
775
|
-
|
|
776
|
-
|
|
913
|
+
const allPassed = loaded.config.checks.every((check) => !check.required || statuses.get(check.id) === "passed") && !budgetExceeded;
|
|
914
|
+
current = { ...current, outcomes: current.outcomes.map((outcome) => {
|
|
915
|
+
const required15 = outcome.checks.filter((id2) => loaded.config.checks.find((check) => check.id === id2)?.required);
|
|
916
|
+
return { ...outcome, status: required15.length === 0 ? "not-applicable" : required15.every((id2) => statuses.get(id2) === "passed") ? "passed" : "failed" };
|
|
917
|
+
}), metrics: { totalDurationMs, wallDurationMs: Date.now() - verificationStarted, peakConcurrency: observedPeakConcurrency, budgetExceeded, machine: machineMonitor.stop() } };
|
|
918
|
+
const digest3 = verificationDigest(current);
|
|
919
|
+
current = { ...current, verificationDigest: digest3 };
|
|
777
920
|
saveRun2(loaded.stateDir, current);
|
|
778
|
-
new FileEventStore(loaded.stateDir).append({ runId: current.runId, sourceRevision: current.sourceRevision, configHash: current.configHash, type: "verification.completed", payload: { verificationDigest:
|
|
779
|
-
const
|
|
780
|
-
|
|
921
|
+
new FileEventStore(loaded.stateDir).append({ runId: current.runId, sourceRevision: current.sourceRevision, configHash: current.configHash, type: "verification.completed", payload: { verificationDigest: digest3, checkCount: current.checks.length, outcomeCount: current.outcomes.length, totalDurationMs, budgetExceeded } });
|
|
922
|
+
const automatic = allPassed && current.autonomy === "yolo" && !loaded.config.tracking.required && loaded.config.contract.ambiguities.length === 0;
|
|
923
|
+
const nextState = allPassed ? automatic ? "COMPLETE" : "AWAITING_HUMAN_APPROVAL" : "BLOCKED";
|
|
924
|
+
current = { ...transition(current, nextState, automatic ? "All applicable checks passed; YOLO policy permits automatic completion." : allPassed ? "All configured checks passed; human approval is required." : budgetExceeded ? "Verification budget was exceeded." : "A configured check failed or lacked structured evidence.", "harness") };
|
|
781
925
|
saveRun2(loaded.stateDir, current);
|
|
782
926
|
setLatest(loaded.stateDir, current);
|
|
783
927
|
return current;
|
|
@@ -791,11 +935,6 @@ var assertVerificationAttestation = (loaded, run) => {
|
|
|
791
935
|
const event = new FileEventStore(loaded.stateDir).read(run.runId).filter((item) => item.type === "verification.completed").at(-1);
|
|
792
936
|
if (!event || event.payload.verificationDigest !== expected || event.sourceRevision !== run.sourceRevision || event.configHash !== run.configHash) fail("Verification projection attestation is missing from the event log.", "HARNESS_ERROR");
|
|
793
937
|
};
|
|
794
|
-
var plannerForRetry = (loaded, run) => {
|
|
795
|
-
if (run.contractPreparation) return "ci";
|
|
796
|
-
if (!run.supersedes) return "human";
|
|
797
|
-
return plannerForRetry(loaded, readRun(loaded.stateDir, run.supersedes));
|
|
798
|
-
};
|
|
799
938
|
var recordDecision = (loaded, run, type, payload) => {
|
|
800
939
|
new FileEventStore(loaded.stateDir).append({ runId: run.runId, sourceRevision: run.sourceRevision, configHash: run.configHash, type, payload: type === "authorization.recorded" ? { ...payload, target: payload.target ?? fail("tracking.target is required when authorizing.", "INVALID_CONFIG") } : payload });
|
|
801
940
|
};
|
|
@@ -815,7 +954,7 @@ var reconcileRun = async ({ configPath, runId }) => {
|
|
|
815
954
|
if (eventLog.status !== "verified") fail("Terminal run requires a verified event log.", "HARNESS_ERROR");
|
|
816
955
|
assertVerificationAttestation(loaded, run);
|
|
817
956
|
}
|
|
818
|
-
if (run.state === "AWAITING_AUTHORIZATION" || run.state === "COMPLETE") {
|
|
957
|
+
if ((run.state === "AWAITING_AUTHORIZATION" || run.state === "COMPLETE") && run.autonomy !== "yolo") {
|
|
819
958
|
const approval = events.filter((event) => event.type === "approval.recorded").at(-1) ?? fail("Terminal run is missing its human approval event.", "HARNESS_ERROR");
|
|
820
959
|
assertDecisionProjection(run, approval.payload, run.state === "COMPLETE" && !loaded.config.tracking.required ? "COMPLETE" : "AWAITING_AUTHORIZATION");
|
|
821
960
|
if (!run.humanApproval || run.humanApproval.actor !== "human" || run.humanApproval.verificationDigest !== run.verificationDigest || run.humanApproval.sourceRevision !== run.sourceRevision || run.humanApproval.contractHash !== run.contractHash) fail("Human approval projection is inconsistent with its audit event.", "HARNESS_ERROR");
|
|
@@ -877,7 +1016,7 @@ var retryRun = async ({ configPath }) => {
|
|
|
877
1016
|
const baseline = await sourceSnapshot(loaded.root, loaded.stateDir);
|
|
878
1017
|
const superseded = transition(previousRun, "SUPERSEDED", "Retry superseded the previous run.", "harness");
|
|
879
1018
|
saveRun2(loaded.stateDir, superseded);
|
|
880
|
-
const run = await createRun({ loaded, baseline, supersedes: previousRun.runId, dirtyBaselineAuthorized: previousRun.dirtyBaselineAuthorized
|
|
1019
|
+
const run = await createRun({ loaded, baseline, supersedes: previousRun.runId, dirtyBaselineAuthorized: previousRun.dirtyBaselineAuthorized });
|
|
881
1020
|
const next = transition(run, "IMPLEMENTING", "Retry started after a previous attempt.", "agent");
|
|
882
1021
|
saveRun2(loaded.stateDir, next);
|
|
883
1022
|
setLatest(loaded.stateDir, next);
|
|
@@ -904,13 +1043,543 @@ var createDocBridgeContextProvider = ({ root, indexPath = ".doc-bridge/index.jso
|
|
|
904
1043
|
return { providerId: "doc-bridge", query, references, sourceHash: contentHash, snapshotHash: hashContextSnapshot({ providerId: "doc-bridge", query, references, sourceHash: contentHash }), resolvedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
905
1044
|
}
|
|
906
1045
|
});
|
|
1046
|
+
|
|
1047
|
+
// src/discovery.ts
|
|
1048
|
+
var required = (value, label) => {
|
|
1049
|
+
if (typeof value !== "string" || !value.trim()) fail(`${label} is required.`, "INVALID_INPUT");
|
|
1050
|
+
return value.trim();
|
|
1051
|
+
};
|
|
1052
|
+
var unique = (values, label) => {
|
|
1053
|
+
if (new Set(values).size !== values.length) fail(`${label} must be unique.`, "INVALID_INPUT");
|
|
1054
|
+
};
|
|
1055
|
+
var validate = (input) => {
|
|
1056
|
+
required(input.issueId, "issueId");
|
|
1057
|
+
required(input.sourceRevision, "sourceRevision");
|
|
1058
|
+
required(input.contractHash, "contractHash");
|
|
1059
|
+
if (!Array.isArray(input.ambiguities)) fail("ambiguities must be an array.", "INVALID_INPUT");
|
|
1060
|
+
unique(input.ambiguities.map((item) => required(item.id, "ambiguity.id")), "ambiguity ids");
|
|
1061
|
+
const assumptions = /* @__PURE__ */ new Map();
|
|
1062
|
+
for (const assumption of input.approvedAssumptions ?? []) {
|
|
1063
|
+
const id2 = required(assumption.id, "assumption.id");
|
|
1064
|
+
if (assumptions.has(id2)) fail("assumption ids must be unique.", "INVALID_INPUT");
|
|
1065
|
+
assumptions.set(id2, { id: id2, policyId: required(assumption.policyId, "assumption.policyId"), resolution: required(assumption.resolution, "assumption.resolution") });
|
|
1066
|
+
}
|
|
1067
|
+
for (const ambiguity of input.ambiguities) {
|
|
1068
|
+
required(ambiguity.question, "ambiguity.question");
|
|
1069
|
+
if (typeof ambiguity.material !== "boolean") fail("ambiguity.material must be boolean.", "INVALID_INPUT");
|
|
1070
|
+
if (!Array.isArray(ambiguity.options) || ambiguity.options.length < 2 || ambiguity.options.length > 4) fail("ambiguity.options must contain 2 to 4 options.", "INVALID_INPUT");
|
|
1071
|
+
unique(ambiguity.options.map((option) => required(option.id, "option.id")), "option ids");
|
|
1072
|
+
for (const option of ambiguity.options) {
|
|
1073
|
+
required(option.summary, "option.summary");
|
|
1074
|
+
required(option.impact, "option.impact");
|
|
1075
|
+
}
|
|
1076
|
+
if (!ambiguity.options.some((option) => option.id === ambiguity.recommendedOptionId)) fail("recommendedOptionId must identify an option.", "INVALID_INPUT");
|
|
1077
|
+
if (!ambiguity.material && (!ambiguity.assumptionId || !assumptions.has(ambiguity.assumptionId))) fail("non-material ambiguity requires an approved assumption.", "INVALID_INPUT");
|
|
1078
|
+
}
|
|
1079
|
+
return { assumptions };
|
|
1080
|
+
};
|
|
1081
|
+
var digest2 = (result) => hashJson(result);
|
|
1082
|
+
var assessDiscovery = (input) => {
|
|
1083
|
+
const { assumptions } = validate(input);
|
|
1084
|
+
const human = input.ambiguities.filter((ambiguity) => ambiguity.material);
|
|
1085
|
+
const decisionLog = input.ambiguities.map((ambiguity) => {
|
|
1086
|
+
if (ambiguity.material) return { ambiguityId: ambiguity.id, kind: "human-decision-required", detail: `Recommendation: ${ambiguity.recommendedOptionId}.` };
|
|
1087
|
+
const assumption = assumptions.get(ambiguity.assumptionId);
|
|
1088
|
+
return { ambiguityId: ambiguity.id, kind: "approved-assumption", detail: assumption.resolution, policyId: assumption.policyId };
|
|
1089
|
+
});
|
|
1090
|
+
const base = {
|
|
1091
|
+
version: 1,
|
|
1092
|
+
issueId: input.issueId,
|
|
1093
|
+
sourceRevision: input.sourceRevision,
|
|
1094
|
+
contractHash: input.contractHash,
|
|
1095
|
+
...input.contextHash ? { contextHash: input.contextHash } : {},
|
|
1096
|
+
status: human.length ? "awaiting-decision" : "ready",
|
|
1097
|
+
...human.length ? { packet: {
|
|
1098
|
+
issueId: input.issueId,
|
|
1099
|
+
contractHash: input.contractHash,
|
|
1100
|
+
sourceRevision: input.sourceRevision,
|
|
1101
|
+
...input.contextHash ? { contextHash: input.contextHash } : {},
|
|
1102
|
+
decisions: human.map((ambiguity) => ({ id: ambiguity.id, question: ambiguity.question, options: ambiguity.options, recommendedOptionId: ambiguity.recommendedOptionId }))
|
|
1103
|
+
} } : {},
|
|
1104
|
+
decisionLog
|
|
1105
|
+
};
|
|
1106
|
+
return { ...base, digest: digest2(base) };
|
|
1107
|
+
};
|
|
1108
|
+
var isDiscoveryCurrent = (result, current) => {
|
|
1109
|
+
const reasons = [];
|
|
1110
|
+
if (result.sourceRevision !== current.sourceRevision) reasons.push("source");
|
|
1111
|
+
if (result.contractHash !== current.contractHash) reasons.push("contract");
|
|
1112
|
+
if ((result.contextHash ?? "") !== (current.contextHash ?? "")) reasons.push("context");
|
|
1113
|
+
return { current: reasons.length === 0, reasons };
|
|
1114
|
+
};
|
|
1115
|
+
|
|
1116
|
+
// src/wip.ts
|
|
1117
|
+
var WIP_STATES = ["ready", "implementing", "blocked", "awaiting-decision", "awaiting-acceptance", "done", "cancelled"];
|
|
1118
|
+
var terminal = /* @__PURE__ */ new Set(["done", "cancelled"]);
|
|
1119
|
+
var required2 = (value, label) => {
|
|
1120
|
+
if (typeof value !== "string" || !value.trim()) fail(`${label} is required.`, "INVALID_INPUT");
|
|
1121
|
+
return value.trim();
|
|
1122
|
+
};
|
|
1123
|
+
var assessWip = ({ entries, candidate, maxInFlight = 3 }) => {
|
|
1124
|
+
if (!Array.isArray(entries)) fail("entries must be an array.", "INVALID_INPUT");
|
|
1125
|
+
if (!Number.isInteger(maxInFlight) || maxInFlight < 1) fail("maxInFlight must be a positive integer.", "INVALID_INPUT");
|
|
1126
|
+
const candidateId = required2(candidate.issueId, "candidate.issueId");
|
|
1127
|
+
if (candidate.kind !== "new" && candidate.kind !== "resume") fail("candidate.kind must be new or resume.", "INVALID_INPUT");
|
|
1128
|
+
const ids = /* @__PURE__ */ new Set();
|
|
1129
|
+
const counts = Object.fromEntries(WIP_STATES.map((state) => [state, 0]));
|
|
1130
|
+
for (const entry of entries) {
|
|
1131
|
+
const id2 = required2(entry.issueId, "entry.issueId");
|
|
1132
|
+
if (ids.has(id2)) fail("entry issueIds must be unique.", "INVALID_INPUT");
|
|
1133
|
+
ids.add(id2);
|
|
1134
|
+
if (!WIP_STATES.includes(entry.state)) fail(`Unknown WIP state: ${entry.state}.`, "INVALID_INPUT");
|
|
1135
|
+
counts[entry.state] += 1;
|
|
1136
|
+
}
|
|
1137
|
+
const inFlight = entries.filter((entry) => !terminal.has(entry.state));
|
|
1138
|
+
const existing = entries.find((entry) => entry.issueId === candidateId);
|
|
1139
|
+
if (candidate.kind === "resume") {
|
|
1140
|
+
if (!existing || terminal.has(existing.state)) return { decision: "hold", inFlight, counts, reason: "A resume requires an existing non-terminal issue." };
|
|
1141
|
+
return { decision: "admit", inFlight, counts, reason: "A resume keeps its existing WIP reservation and takes priority over new work." };
|
|
1142
|
+
}
|
|
1143
|
+
if (existing) return { decision: "hold", inFlight, counts, reason: "A new admission cannot reuse an existing issue id." };
|
|
1144
|
+
if (inFlight.length >= maxInFlight) return { decision: "hold", inFlight, counts, reason: `WIP limit ${maxInFlight} reached; blocked and awaiting-human work still count.` };
|
|
1145
|
+
return { decision: "admit", inFlight, counts, reason: `WIP slot available (${inFlight.length}/${maxInFlight}).` };
|
|
1146
|
+
};
|
|
1147
|
+
|
|
1148
|
+
// src/experiment.ts
|
|
1149
|
+
var required3 = (value, label) => {
|
|
1150
|
+
if (typeof value !== "string" || !value.trim()) fail(`${label} is required.`, "INVALID_INPUT");
|
|
1151
|
+
return value.trim();
|
|
1152
|
+
};
|
|
1153
|
+
var comparable = (candidate, baseline) => {
|
|
1154
|
+
for (const key of ["sourceRevision", "contractHash", "provider", "model", "configurationHash"]) {
|
|
1155
|
+
if (candidate[key] !== baseline[key]) fail(`Candidates must share ${key}.`, "INVALID_INPUT");
|
|
1156
|
+
}
|
|
1157
|
+
};
|
|
1158
|
+
var selectRuntime = (candidates) => {
|
|
1159
|
+
if (!Array.isArray(candidates) || candidates.length < 2) fail("At least two runtime candidates are required.", "INVALID_INPUT");
|
|
1160
|
+
const names = /* @__PURE__ */ new Set();
|
|
1161
|
+
for (const candidate of candidates) {
|
|
1162
|
+
const runtime = required3(candidate.runtime, "candidate.runtime");
|
|
1163
|
+
if (names.has(runtime)) fail("candidate.runtime values must be unique.", "INVALID_INPUT");
|
|
1164
|
+
names.add(runtime);
|
|
1165
|
+
for (const key of ["sourceRevision", "contractHash", "provider", "model", "configurationHash"]) required3(candidate[key], `candidate.${key}`);
|
|
1166
|
+
for (const key of ["humanMinutes", "durationMs", "cost"]) if (!Number.isFinite(candidate[key]) || candidate[key] < 0) fail(`candidate.${key} must be a non-negative number.`, "INVALID_INPUT");
|
|
1167
|
+
comparable(candidate, candidates[0]);
|
|
1168
|
+
}
|
|
1169
|
+
const eligible = candidates.filter((candidate) => candidate.hardGatesPassed);
|
|
1170
|
+
if (!eligible.length) return { decision: "blocked", eligible, reason: "No runtime passed every hard gate." };
|
|
1171
|
+
const selected = [...eligible].sort((left, right) => left.humanMinutes - right.humanMinutes || left.durationMs - right.durationMs || left.cost - right.cost || (left.runtime === "orca" ? -1 : right.runtime === "orca" ? 1 : left.runtime.localeCompare(right.runtime)))[0];
|
|
1172
|
+
return { decision: "selected", selected, eligible, reason: "Selected by human minutes, duration, cost, then Orca tie-break." };
|
|
1173
|
+
};
|
|
1174
|
+
|
|
1175
|
+
// src/delivery.ts
|
|
1176
|
+
var required4 = (value, label) => {
|
|
1177
|
+
if (typeof value !== "string" || !value.trim()) fail(`${label} is required.`, "INVALID_INPUT");
|
|
1178
|
+
return value.trim();
|
|
1179
|
+
};
|
|
1180
|
+
var criteriaFor = (criteria, gate) => {
|
|
1181
|
+
if (!Array.isArray(criteria)) fail("criteria must be an array.", "INVALID_INPUT");
|
|
1182
|
+
const ids = /* @__PURE__ */ new Set();
|
|
1183
|
+
for (const criterion of criteria) {
|
|
1184
|
+
const id2 = required4(criterion.id, "criterion.id");
|
|
1185
|
+
if (ids.has(id2)) fail("criterion ids must be unique.", "INVALID_INPUT");
|
|
1186
|
+
ids.add(id2);
|
|
1187
|
+
if (!["G2", "G3", "G4", "G5"].includes(criterion.gate)) fail("criterion.gate is invalid.", "INVALID_INPUT");
|
|
1188
|
+
if (!["passed", "failed", "pending", "not-applicable"].includes(criterion.status)) fail("criterion.status is invalid.", "INVALID_INPUT");
|
|
1189
|
+
if (criterion.status === "not-applicable" && !criterion.reason?.trim()) fail("not-applicable criteria require a reason.", "INVALID_INPUT");
|
|
1190
|
+
}
|
|
1191
|
+
return criteria.filter((criterion) => criterion.gate === gate);
|
|
1192
|
+
};
|
|
1193
|
+
var binding = (value) => ({ candidateRevision: required4(value.candidateRevision, "binding.candidateRevision"), contractHash: required4(value.contractHash, "binding.contractHash"), configHash: required4(value.configHash, "binding.configHash") });
|
|
1194
|
+
var assessed = (gate, decision, reasons, current) => {
|
|
1195
|
+
const base = { gate, decision, reasons, binding: binding(current) };
|
|
1196
|
+
return { ...base, digest: hashJson(base) };
|
|
1197
|
+
};
|
|
1198
|
+
var assessPreflight = ({ criteria, repairAttempts = 0, implementerId, reviewerId, reviewKind, reviewApproved, binding: current }) => {
|
|
1199
|
+
required4(implementerId, "implementerId");
|
|
1200
|
+
if (!Number.isInteger(repairAttempts) || repairAttempts < 0) fail("repairAttempts must be a non-negative integer.", "INVALID_INPUT");
|
|
1201
|
+
const g2 = criteriaFor(criteria, "G2");
|
|
1202
|
+
const reasons = [
|
|
1203
|
+
...g2.length ? [] : ["No G2 criteria are defined."],
|
|
1204
|
+
...g2.filter((criterion) => criterion.status === "failed" || criterion.status === "pending").map((criterion) => `${criterion.id} is ${criterion.status}.`),
|
|
1205
|
+
...reviewApproved && reviewerId && reviewerId !== implementerId && reviewKind === "adversarial" ? [] : ["An approved adversarial review by a reviewer different from the implementer is required."],
|
|
1206
|
+
...repairAttempts <= 2 ? [] : ["The two-repair limit was exceeded; preserve diagnostics and return blocked."]
|
|
1207
|
+
];
|
|
1208
|
+
return assessed("G2", reasons.length ? "blocked" : "approved", reasons, current);
|
|
1209
|
+
};
|
|
1210
|
+
var composePullRequest = ({ draft, g2, remote }) => {
|
|
1211
|
+
for (const [label, value] of Object.entries({ issueId: draft.issueId, candidateRevision: draft.candidateRevision, contractHash: draft.contractHash, configHash: draft.configHash, g2Digest: draft.g2Digest, risk: draft.risk, rollback: draft.rollback })) required4(value, `draft.${label}`);
|
|
1212
|
+
if (g2.gate !== "G2" || g2.decision !== "approved" || g2.digest !== draft.g2Digest || g2.binding.candidateRevision !== draft.candidateRevision || g2.binding.contractHash !== draft.contractHash || g2.binding.configHash !== draft.configHash) return { decision: "blocked", reason: "A current approved G2 assessment is required before a PR can be created.", idempotencyKey: hashJson(draft) };
|
|
1213
|
+
const idempotencyKey = hashJson({ issueId: draft.issueId, contractHash: draft.contractHash, action: "pull-request", revision: draft.candidateRevision });
|
|
1214
|
+
if (remote?.state === "uncertain") return { decision: "blocked", reason: "Remote PR state is uncertain; reconcile before retrying.", idempotencyKey };
|
|
1215
|
+
if (remote?.state === "confirmed") {
|
|
1216
|
+
if (remote.candidateRevision !== draft.candidateRevision || !remote.url) return { decision: "blocked", reason: "Confirmed remote PR does not match the candidate revision.", idempotencyKey };
|
|
1217
|
+
return { decision: "reuse", reason: "The idempotent remote PR already exists for this candidate revision.", idempotencyKey };
|
|
1218
|
+
}
|
|
1219
|
+
const body2 = [`## Contract`, `- Issue: ${draft.issueId}`, `- Candidate: ${draft.candidateRevision}`, `- Contract: ${draft.contractHash}`, `- Configuration: ${draft.configHash}`, "", "## G2 evidence", ...draft.evidence.map((item) => `- ${item}`), "", "## Documentation", ...draft.documentation.map((item) => `- ${item}`), "", "## Risk and rollback", `- Risk: ${draft.risk}`, `- Rollback: ${draft.rollback}`, "", "## Later gates", ...draft.pendingCriteria.length ? draft.pendingCriteria.map((item) => `- Pending: ${item}`) : ["- None."]].join("\n");
|
|
1220
|
+
return { decision: "create", body: body2, reason: "G2 is current and the remote PR is absent.", idempotencyKey };
|
|
1221
|
+
};
|
|
1222
|
+
var assessIntegration = ({ g2, candidateRevision, evidenceRevision, contractHash, configHash, ci }) => {
|
|
1223
|
+
required4(candidateRevision, "candidateRevision");
|
|
1224
|
+
required4(evidenceRevision, "evidenceRevision");
|
|
1225
|
+
if (!["passed", "failed", "pending", "not-applicable"].includes(ci)) fail("ci is invalid.", "INVALID_INPUT");
|
|
1226
|
+
const reasons = [
|
|
1227
|
+
...g2.gate === "G2" && g2.decision === "approved" ? [] : ["G2 is not approved."],
|
|
1228
|
+
...g2.binding.candidateRevision === candidateRevision && g2.binding.contractHash === contractHash && g2.binding.configHash === configHash ? [] : ["G2 is not bound to the current candidate, contract, and configuration."],
|
|
1229
|
+
...candidateRevision === evidenceRevision ? [] : ["Candidate revision changed; G3 evidence must be revalidated."],
|
|
1230
|
+
...ci === "passed" ? [] : [`Integration CI is ${ci}.`]
|
|
1231
|
+
];
|
|
1232
|
+
return assessed("G3", reasons.length ? "blocked" : "approved", reasons, { candidateRevision, contractHash, configHash });
|
|
1233
|
+
};
|
|
1234
|
+
var assessWorktreeCleanup = ({ branch, candidateRevision, contractHash, configHash, remoteBranchRevision, remotePr, integration }) => {
|
|
1235
|
+
required4(branch, "branch");
|
|
1236
|
+
required4(candidateRevision, "candidateRevision");
|
|
1237
|
+
if (remotePr === "uncertain") return { decision: "preserve", reason: "Remote PR state is uncertain; preserve the worktree for reconciliation." };
|
|
1238
|
+
if (remotePr !== "confirmed") return { decision: "preserve", reason: "No confirmed remote PR exists; preserve the worktree." };
|
|
1239
|
+
if (remoteBranchRevision !== candidateRevision) return { decision: "preserve", reason: "Remote branch SHA does not match the candidate revision." };
|
|
1240
|
+
if (integration.gate !== "G3" || integration.decision !== "approved") return { decision: "preserve", reason: "G3 is not approved." };
|
|
1241
|
+
if (integration.binding.candidateRevision !== candidateRevision || integration.binding.contractHash !== contractHash || integration.binding.configHash !== configHash) return { decision: "preserve", reason: "G3 is not bound to the current candidate, contract, and configuration." };
|
|
1242
|
+
return { decision: "clean", reason: "Remote branch, PR, and G3 evidence are confirmed for the candidate revision." };
|
|
1243
|
+
};
|
|
1244
|
+
var profileReasons = (profile) => Object.entries(profile).flatMap(([key, value]) => Array.isArray(value) ? value.length ? [] : [`RepositoryProfile.${key} is required.`] : typeof value === "string" && value.trim() ? [] : [`RepositoryProfile.${key} is required.`]);
|
|
1245
|
+
var assessProduction = ({ profile, integration, artifact, isolated, acceptanceArtifact, lowRisk = true, observationMinutes, technicalPassed, evidence, containmentPreauthorized, containmentAction, linkedDefect }) => {
|
|
1246
|
+
required4(artifact, "artifact");
|
|
1247
|
+
if (!Number.isFinite(observationMinutes) || observationMinutes < 0) fail("observationMinutes must be non-negative.", "INVALID_INPUT");
|
|
1248
|
+
const evidenceReasons = [required4(evidence.tenant, "evidence.tenant"), required4(evidence.realFlow, "evidence.realFlow"), ...Array.isArray(evidence.logs) && evidence.logs.length ? [] : ["Production evidence requires logs."], ...Array.isArray(evidence.metrics) && evidence.metrics.length ? [] : ["Production evidence requires metrics."]].filter((item) => item.startsWith("Production evidence"));
|
|
1249
|
+
const reasons = [
|
|
1250
|
+
...profileReasons(profile),
|
|
1251
|
+
...integration.gate === "G3" && integration.decision === "approved" ? [] : ["G3 is not approved."],
|
|
1252
|
+
...evidenceReasons,
|
|
1253
|
+
...isolated || acceptanceArtifact === artifact ? [] : ["Exposure requires isolation or acceptance linked to this artifact version."],
|
|
1254
|
+
...technicalPassed ? [] : [containmentPreauthorized && containmentAction?.trim() && linkedDefect?.trim() ? "Technical validation failed; pre-authorized containment and linked defect are recorded." : containmentPreauthorized ? "Technical validation failed; containment action and linked defect are required." : "Technical validation failed."],
|
|
1255
|
+
...lowRisk && observationMinutes < 15 ? ["Low-risk production validation requires a 15-minute observation window."] : []
|
|
1256
|
+
];
|
|
1257
|
+
return assessed("G4", reasons.length ? "blocked" : "approved", reasons, integration.binding);
|
|
1258
|
+
};
|
|
1259
|
+
var assessAcceptance = ({ production, acceptanceRequired, accepted, notApplicableReason, materialChange }) => {
|
|
1260
|
+
const reasons = [
|
|
1261
|
+
...production.gate === "G4" && production.decision === "approved" ? [] : ["G4 is not approved."],
|
|
1262
|
+
...materialChange ? ["A material change invalidated acceptance; return to the affected gate."] : []
|
|
1263
|
+
];
|
|
1264
|
+
if (reasons.length) return assessed("G5", "blocked", reasons, production.binding);
|
|
1265
|
+
if (acceptanceRequired && !accepted) return assessed("G5", "awaiting-acceptance", ["Business or UX acceptance is still required."], production.binding);
|
|
1266
|
+
if (!acceptanceRequired && !notApplicableReason?.trim()) return assessed("G5", "blocked", ["Acceptance marked not applicable requires a contractual reason."], production.binding);
|
|
1267
|
+
return assessed("G5", "approved", acceptanceRequired ? [] : [`Acceptance is not applicable: ${notApplicableReason}.`], production.binding);
|
|
1268
|
+
};
|
|
1269
|
+
|
|
1270
|
+
// src/pilot.ts
|
|
1271
|
+
var required5 = (value, label) => {
|
|
1272
|
+
if (typeof value !== "string" || !value.trim()) fail(`${label} is required.`, "INVALID_INPUT");
|
|
1273
|
+
return value.trim();
|
|
1274
|
+
};
|
|
1275
|
+
var assessPilot = (manifest) => {
|
|
1276
|
+
required5(manifest.policyHash, "policyHash");
|
|
1277
|
+
required5(manifest.baselineReference, "baselineReference");
|
|
1278
|
+
if (!Array.isArray(manifest.entries)) fail("entries must be an array.", "INVALID_INPUT");
|
|
1279
|
+
const ids = /* @__PURE__ */ new Set();
|
|
1280
|
+
const reasons = [];
|
|
1281
|
+
const included = [];
|
|
1282
|
+
for (const entry of manifest.entries) {
|
|
1283
|
+
const issueId = required5(entry.issueId, "entry.issueId");
|
|
1284
|
+
if (ids.has(issueId)) fail("entry issueIds must be unique; an issue cannot be substituted in the same pilot.", "INVALID_INPUT");
|
|
1285
|
+
ids.add(issueId);
|
|
1286
|
+
if (!["normal", "incident", "sensitive"].includes(entry.classification)) fail("entry.classification is invalid.", "INVALID_INPUT");
|
|
1287
|
+
if (!["included", "excluded", "aborted"].includes(entry.status)) fail("entry.status is invalid.", "INVALID_INPUT");
|
|
1288
|
+
if (entry.status !== "included" && !entry.reason?.trim()) reasons.push(`${issueId} is ${entry.status} without an auditable reason.`);
|
|
1289
|
+
if (entry.status === "included") {
|
|
1290
|
+
included.push(issueId);
|
|
1291
|
+
if (entry.classification !== "normal") reasons.push(`${issueId} is ${entry.classification}; only normal issues can enter the pilot.`);
|
|
1292
|
+
}
|
|
1293
|
+
}
|
|
1294
|
+
if (included.length !== 10) reasons.push(`Pilot requires exactly 10 included issues; found ${included.length}.`);
|
|
1295
|
+
const base = { decision: reasons.length ? "blocked" : "ready", included, reasons };
|
|
1296
|
+
return { ...base, digest: hashJson({ ...manifest, ...base }) };
|
|
1297
|
+
};
|
|
1298
|
+
var IMPROVEMENT_CYCLE_STEPS = ["adversarial-review", "g2-preflight", "baseline-record", "pilot-execution", "comparison"];
|
|
1299
|
+
var nonEmpty = (value, label) => {
|
|
1300
|
+
if (typeof value !== "string" || !value.trim()) return fail(`${label} must be a non-empty string.`, "INVALID_INPUT");
|
|
1301
|
+
return value.trim();
|
|
1302
|
+
};
|
|
1303
|
+
var validateMetrics = (metrics, index2) => {
|
|
1304
|
+
if (metrics === void 0) return void 0;
|
|
1305
|
+
for (const [key, value] of Object.entries(metrics)) {
|
|
1306
|
+
if (value !== void 0 && (typeof value !== "number" || !Number.isFinite(value) || value < 0)) return fail(`iterations[${index2}].metrics.${key} must be a non-negative number.`, "INVALID_INPUT");
|
|
1307
|
+
}
|
|
1308
|
+
return metrics;
|
|
1309
|
+
};
|
|
1310
|
+
var validateIteration = (iteration, index2) => {
|
|
1311
|
+
if (typeof iteration !== "object" || iteration === null || Array.isArray(iteration)) return fail(`iterations[${index2}] must be an object.`, "INVALID_INPUT");
|
|
1312
|
+
if (!Number.isInteger(iteration.iteration) || iteration.iteration < 1) return fail(`iterations[${index2}].iteration must be a positive integer.`, "INVALID_INPUT");
|
|
1313
|
+
if (!Array.isArray(iteration.steps) || iteration.steps.length !== IMPROVEMENT_CYCLE_STEPS.length) return fail(`iterations[${index2}].steps must contain the five cycle steps exactly once, in order.`, "INVALID_INPUT");
|
|
1314
|
+
iteration.steps.forEach((result, stepIndex) => {
|
|
1315
|
+
if (typeof result !== "object" || result === null || Array.isArray(result)) return fail(`iterations[${index2}].steps[${stepIndex}] must be an object.`, "INVALID_INPUT");
|
|
1316
|
+
if (result.step !== IMPROVEMENT_CYCLE_STEPS[stepIndex]) return fail(`iterations[${index2}].steps[${stepIndex}] must be ${IMPROVEMENT_CYCLE_STEPS[stepIndex]}.`, "INVALID_INPUT");
|
|
1317
|
+
if (!["passed", "failed", "blocked", "pending"].includes(result.status)) return fail(`iterations[${index2}].steps[${stepIndex}].status is invalid.`, "INVALID_INPUT");
|
|
1318
|
+
if (result.status !== "passed" && !nonEmpty(result.reason, `iterations[${index2}].steps[${stepIndex}].reason`)) return fail(`iterations[${index2}].steps[${stepIndex}].reason is required when the step does not pass.`, "INVALID_INPUT");
|
|
1319
|
+
});
|
|
1320
|
+
if (iteration.adjustment !== void 0) nonEmpty(iteration.adjustment, `iterations[${index2}].adjustment`);
|
|
1321
|
+
return { ...iteration, metrics: validateMetrics(iteration.metrics, index2) };
|
|
1322
|
+
};
|
|
1323
|
+
var assessImprovementCycle = (input) => {
|
|
1324
|
+
if (typeof input !== "object" || input === null || Array.isArray(input)) return fail("cycle input must be an object.", "INVALID_INPUT");
|
|
1325
|
+
const cycleId = nonEmpty(input.cycleId, "cycleId");
|
|
1326
|
+
if (!Number.isInteger(input.maxIterations) || input.maxIterations < 1) return fail("maxIterations must be a positive integer.", "INVALID_INPUT");
|
|
1327
|
+
if (!Array.isArray(input.iterations) || input.iterations.length < 1) return fail("iterations must be non-empty.", "INVALID_INPUT");
|
|
1328
|
+
if (input.iterations.length > input.maxIterations) return fail("iterations cannot exceed maxIterations.", "INVALID_INPUT");
|
|
1329
|
+
const iterations = input.iterations.map(validateIteration);
|
|
1330
|
+
iterations.forEach((iteration, index2) => {
|
|
1331
|
+
if (iteration.iteration !== index2 + 1) return fail("iterations must be sequential and start at 1.", "INVALID_INPUT");
|
|
1332
|
+
if (index2 > 0 && iterations[index2 - 1]?.steps.every((step) => step.status === "passed")) return fail("a completed cycle cannot have later iterations.", "INVALID_INPUT");
|
|
1333
|
+
if (index2 < iterations.length - 1 && !iteration.adjustment) return fail(`iterations[${index2}].adjustment is required before repeating.`, "INVALID_INPUT");
|
|
1334
|
+
});
|
|
1335
|
+
const matrix = iterations.map((iteration) => {
|
|
1336
|
+
const statuses = Object.fromEntries(iteration.steps.map((step) => [step.step, step.status]));
|
|
1337
|
+
const passedSteps = iteration.steps.filter((step) => step.status === "passed").length;
|
|
1338
|
+
return { iteration: iteration.iteration, passedSteps, totalSteps: IMPROVEMENT_CYCLE_STEPS.length, passRate: Number((passedSteps / IMPROVEMENT_CYCLE_STEPS.length).toFixed(4)), statuses, ...iteration.adjustment ? { adjustment: iteration.adjustment } : {}, ...iteration.metrics ? { metrics: iteration.metrics } : {} };
|
|
1339
|
+
});
|
|
1340
|
+
const latest = iterations[iterations.length - 1];
|
|
1341
|
+
const complete = latest.steps.every((step) => step.status === "passed");
|
|
1342
|
+
const reasons = complete ? ["All five cycle steps passed."] : iterations.length >= input.maxIterations ? ["Maximum cycle iterations reached; human adjustment is required."] : latest.adjustment ? ["A failed or blocked step remains; repeat with the recorded adjustment."] : ["A failed or blocked step remains; an explicit adjustment is required before repeating."];
|
|
1343
|
+
const decision = complete ? "complete" : iterations.length >= input.maxIterations || !latest.adjustment ? "blocked" : "repeat";
|
|
1344
|
+
const result = { type: "agentskit-harness-improvement-cycle", cycleId, decision, ...decision === "repeat" ? { nextIteration: latest.iteration + 1 } : {}, reasons, matrix };
|
|
1345
|
+
const digest3 = createHash("sha256").update(JSON.stringify(result)).digest("hex");
|
|
1346
|
+
return { ...result, digest: digest3 };
|
|
1347
|
+
};
|
|
1348
|
+
|
|
1349
|
+
// src/eval.ts
|
|
1350
|
+
var pass = (expected, output) => typeof expected === "string" ? output === expected : expected(output);
|
|
1351
|
+
var runAgentEval = async ({ suite, agent, concurrency = 1 }) => {
|
|
1352
|
+
if (!suite.name.trim() || !suite.cases.length) fail("Eval suite must have a name and at least one case.", "INVALID_INPUT");
|
|
1353
|
+
if (!Number.isInteger(concurrency) || concurrency < 1) fail("Eval concurrency must be a positive integer.", "INVALID_INPUT");
|
|
1354
|
+
const failures = [];
|
|
1355
|
+
let passed = 0;
|
|
1356
|
+
for (let offset = 0; offset < suite.cases.length; offset += concurrency) {
|
|
1357
|
+
const batch = suite.cases.slice(offset, offset + concurrency);
|
|
1358
|
+
const outputs = await Promise.all(batch.map((testCase) => agent(testCase.input)));
|
|
1359
|
+
batch.forEach((testCase, index2) => {
|
|
1360
|
+
if (pass(testCase.expected, outputs[index2])) passed += 1;
|
|
1361
|
+
else failures.push(testCase.id);
|
|
1362
|
+
});
|
|
1363
|
+
}
|
|
1364
|
+
return { suite: suite.name, total: suite.cases.length, passed, failed: suite.cases.length - passed, accuracy: passed / suite.cases.length, failures };
|
|
1365
|
+
};
|
|
1366
|
+
var assessAgentEval = (report, minimumAccuracy) => {
|
|
1367
|
+
if (!Number.isFinite(minimumAccuracy) || minimumAccuracy < 0 || minimumAccuracy > 1) fail("minimumAccuracy must be between 0 and 1.", "INVALID_INPUT");
|
|
1368
|
+
if (report.total < 1 || report.passed + report.failed !== report.total || report.accuracy !== report.passed / report.total) fail("Eval report is inconsistent.", "INVALID_INPUT");
|
|
1369
|
+
return report.accuracy >= minimumAccuracy ? { status: "passed", reason: `Accuracy ${report.accuracy.toFixed(4)} meets ${minimumAccuracy.toFixed(4)}.`, report } : { status: "blocked", reason: `Accuracy ${report.accuracy.toFixed(4)} is below ${minimumAccuracy.toFixed(4)}.`, report };
|
|
1370
|
+
};
|
|
1371
|
+
|
|
1372
|
+
// src/cache.ts
|
|
1373
|
+
var validateCacheableOperation = (operation) => {
|
|
1374
|
+
if (operation !== "context" && operation !== "read-only") fail("Only context and read-only operations may use the LLM cache.", "POLICY_BLOCKED");
|
|
1375
|
+
return operation;
|
|
1376
|
+
};
|
|
1377
|
+
var createLlmCacheKey = (input) => {
|
|
1378
|
+
validateCacheableOperation(input.operation);
|
|
1379
|
+
return hashJson(input);
|
|
1380
|
+
};
|
|
1381
|
+
var createLlmCache = () => {
|
|
1382
|
+
const values = /* @__PURE__ */ new Map();
|
|
1383
|
+
let hits = 0;
|
|
1384
|
+
let misses = 0;
|
|
1385
|
+
let invalidations = 0;
|
|
1386
|
+
return {
|
|
1387
|
+
async getOrCompute(key, compute) {
|
|
1388
|
+
const cached = values.get(key);
|
|
1389
|
+
if (cached !== void 0) {
|
|
1390
|
+
hits += 1;
|
|
1391
|
+
return cached;
|
|
1392
|
+
}
|
|
1393
|
+
misses += 1;
|
|
1394
|
+
const value = await compute();
|
|
1395
|
+
values.set(key, value);
|
|
1396
|
+
return value;
|
|
1397
|
+
},
|
|
1398
|
+
invalidate(key) {
|
|
1399
|
+
if (key === void 0) {
|
|
1400
|
+
invalidations += values.size;
|
|
1401
|
+
values.clear();
|
|
1402
|
+
return;
|
|
1403
|
+
}
|
|
1404
|
+
if (values.delete(key)) invalidations += 1;
|
|
1405
|
+
},
|
|
1406
|
+
stats: () => ({ hits, misses, invalidations })
|
|
1407
|
+
};
|
|
1408
|
+
};
|
|
1409
|
+
|
|
1410
|
+
// src/optimization.ts
|
|
1411
|
+
var nonNegative = (value, label) => {
|
|
1412
|
+
if (!Number.isFinite(value) || value < 0) fail(`${label} must be a non-negative number.`, "INVALID_INPUT");
|
|
1413
|
+
return value;
|
|
1414
|
+
};
|
|
1415
|
+
var nonNegativeInteger = (value, label) => {
|
|
1416
|
+
nonNegative(value, label);
|
|
1417
|
+
if (!Number.isInteger(value)) fail(`${label} must be an integer.`, "INVALID_INPUT");
|
|
1418
|
+
return value;
|
|
1419
|
+
};
|
|
1420
|
+
var required6 = (value, label) => {
|
|
1421
|
+
if (typeof value !== "string" || !value.trim()) fail(`${label} is required.`, "INVALID_INPUT");
|
|
1422
|
+
return value.trim();
|
|
1423
|
+
};
|
|
1424
|
+
var validateOptimizationObservation = (observation) => {
|
|
1425
|
+
required6(observation.sourceRevision, "sourceRevision");
|
|
1426
|
+
required6(observation.contractHash, "contractHash");
|
|
1427
|
+
required6(observation.configHash, "configHash");
|
|
1428
|
+
required6(observation.provider, "provider");
|
|
1429
|
+
required6(observation.model, "model");
|
|
1430
|
+
nonNegative(observation.durationMs, "durationMs");
|
|
1431
|
+
if (observation.accuracy !== void 0 && (!Number.isFinite(observation.accuracy) || observation.accuracy < 0 || observation.accuracy > 1)) fail("accuracy must be between 0 and 1.", "INVALID_INPUT");
|
|
1432
|
+
if (observation.tokens) {
|
|
1433
|
+
const tokens = observation.tokens;
|
|
1434
|
+
for (const key of ["inputTokens", "outputTokens", "totalTokens"]) nonNegativeInteger(tokens[key], `tokens.${key}`);
|
|
1435
|
+
if (tokens.totalTokens !== tokens.inputTokens + tokens.outputTokens) fail("tokens.totalTokens must equal inputTokens + outputTokens.", "INVALID_INPUT");
|
|
1436
|
+
for (const key of ["cacheReadTokens", "cacheWriteTokens"]) if (tokens[key] !== void 0) nonNegativeInteger(tokens[key], `tokens.${key}`);
|
|
1437
|
+
}
|
|
1438
|
+
if (observation.memory) for (const key of ["reads", "writes", "relevantHits", "staleHits"]) nonNegativeInteger(observation.memory[key], `memory.${key}`);
|
|
1439
|
+
if (observation.cache) {
|
|
1440
|
+
for (const key of ["hits", "misses", "invalidations"]) nonNegativeInteger(observation.cache[key], `cache.${key}`);
|
|
1441
|
+
if (observation.cache.tokensSaved !== void 0) nonNegativeInteger(observation.cache.tokensSaved, "cache.tokensSaved");
|
|
1442
|
+
}
|
|
1443
|
+
if (observation.parallelism) {
|
|
1444
|
+
for (const key of ["tasks", "peakConcurrency"]) nonNegativeInteger(observation.parallelism[key], `parallelism.${key}`);
|
|
1445
|
+
nonNegative(observation.parallelism.criticalPathMs, "parallelism.criticalPathMs");
|
|
1446
|
+
if (observation.parallelism.queueWaitMs !== void 0) nonNegative(observation.parallelism.queueWaitMs, "parallelism.queueWaitMs");
|
|
1447
|
+
if (observation.parallelism.tasks > 0 && observation.parallelism.peakConcurrency < 1) fail("parallelism.peakConcurrency must be positive when tasks exist.", "INVALID_INPUT");
|
|
1448
|
+
}
|
|
1449
|
+
return observation;
|
|
1450
|
+
};
|
|
1451
|
+
var rate = (hits, total) => total ? Number((hits / total).toFixed(4)) : void 0;
|
|
1452
|
+
var compareOptimization = (baseline, candidate) => {
|
|
1453
|
+
validateOptimizationObservation(baseline);
|
|
1454
|
+
validateOptimizationObservation(candidate);
|
|
1455
|
+
for (const key of ["sourceRevision", "contractHash", "configHash", "provider", "model"]) if (baseline[key] !== candidate[key]) return { comparable: false, reason: `Bindings differ: ${key}.`, digest: hashJson({ baseline, candidate }) };
|
|
1456
|
+
const result = {
|
|
1457
|
+
comparable: true,
|
|
1458
|
+
reason: "Observations share source, contract, configuration, provider, and model bindings.",
|
|
1459
|
+
digest: hashJson({ baseline, candidate }),
|
|
1460
|
+
durationDeltaMs: candidate.durationMs - baseline.durationMs,
|
|
1461
|
+
...baseline.accuracy !== void 0 && candidate.accuracy !== void 0 ? { accuracyDelta: Number((candidate.accuracy - baseline.accuracy).toFixed(4)) } : {},
|
|
1462
|
+
...baseline.tokens && candidate.tokens ? { tokenDelta: candidate.tokens.totalTokens - baseline.tokens.totalTokens } : {},
|
|
1463
|
+
...baseline.cache && candidate.cache ? { cacheHitRateDelta: (rate(candidate.cache.hits, candidate.cache.hits + candidate.cache.misses) ?? 0) - (rate(baseline.cache.hits, baseline.cache.hits + baseline.cache.misses) ?? 0) } : {},
|
|
1464
|
+
...baseline.memory && candidate.memory ? { memoryRelevantHitRateDelta: (rate(candidate.memory.relevantHits, candidate.memory.reads) ?? 0) - (rate(baseline.memory.relevantHits, baseline.memory.reads) ?? 0) } : {},
|
|
1465
|
+
...baseline.parallelism && candidate.parallelism ? { peakConcurrencyDelta: candidate.parallelism.peakConcurrency - baseline.parallelism.peakConcurrency } : {}
|
|
1466
|
+
};
|
|
1467
|
+
return result;
|
|
1468
|
+
};
|
|
1469
|
+
|
|
1470
|
+
// src/memory.ts
|
|
1471
|
+
var MEMORY_SCOPES = ["issue", "project", "global"];
|
|
1472
|
+
var text2 = (value, label) => {
|
|
1473
|
+
if (typeof value !== "string" || !value.trim()) fail(label + " must be a non-empty string.", "INVALID_INPUT");
|
|
1474
|
+
return value.trim();
|
|
1475
|
+
};
|
|
1476
|
+
var validateMemoryRecord = (record3) => {
|
|
1477
|
+
text2(record3.id, "memory.id");
|
|
1478
|
+
if (!MEMORY_SCOPES.includes(record3.scope)) fail("memory.scope is invalid.", "INVALID_INPUT");
|
|
1479
|
+
text2(record3.summary, "memory.summary");
|
|
1480
|
+
text2(record3.source, "memory.source");
|
|
1481
|
+
text2(record3.sourceRevision, "memory.sourceRevision");
|
|
1482
|
+
text2(record3.contentHash, "memory.contentHash");
|
|
1483
|
+
if (record3.approved !== true) fail("Only approved memory may enter the shared store.", "POLICY_BLOCKED");
|
|
1484
|
+
return record3;
|
|
1485
|
+
};
|
|
1486
|
+
var createInMemoryMemoryAdapter = (options = {}) => {
|
|
1487
|
+
const records = /* @__PURE__ */ new Map();
|
|
1488
|
+
return {
|
|
1489
|
+
id: options.id ?? "in-memory",
|
|
1490
|
+
version: options.version ?? "1",
|
|
1491
|
+
async remember(record3) {
|
|
1492
|
+
records.set(validateMemoryRecord(record3).id, record3);
|
|
1493
|
+
},
|
|
1494
|
+
async recall({ query, issueId, project, sourceRevision }) {
|
|
1495
|
+
const needle = query.trim().toLowerCase();
|
|
1496
|
+
return [...records.values()].filter((record3) => {
|
|
1497
|
+
const scopeMatch = record3.scope === "global" || (record3.scope === "issue" ? Boolean(issueId && record3.source.includes(issueId)) : Boolean(project && record3.source.includes(project)));
|
|
1498
|
+
return scopeMatch && (!needle || `${record3.summary} ${record3.source}`.toLowerCase().includes(needle));
|
|
1499
|
+
}).map((record3) => ({ record: record3, relevant: true, stale: sourceRevision !== void 0 && record3.sourceRevision !== sourceRevision }));
|
|
1500
|
+
}
|
|
1501
|
+
};
|
|
1502
|
+
};
|
|
1503
|
+
var createKvMemoryAdapter = (store, options = {}) => {
|
|
1504
|
+
const indexKey = "agentskit-harness:memory:index";
|
|
1505
|
+
const matches2 = (record3, query, issueId, project) => {
|
|
1506
|
+
const scopeMatch = record3.scope === "global" || (record3.scope === "issue" ? Boolean(issueId && record3.source.includes(issueId)) : Boolean(project && record3.source.includes(project)));
|
|
1507
|
+
return scopeMatch && (!query || `${record3.summary} ${record3.source}`.toLowerCase().includes(query));
|
|
1508
|
+
};
|
|
1509
|
+
return {
|
|
1510
|
+
id: options.id ?? "agentskit-kv",
|
|
1511
|
+
version: options.version ?? "1",
|
|
1512
|
+
async remember(record3) {
|
|
1513
|
+
const valid = validateMemoryRecord(record3);
|
|
1514
|
+
const ids = await store.get(indexKey);
|
|
1515
|
+
const index2 = Array.isArray(ids) ? ids.filter((id2) => typeof id2 === "string") : [];
|
|
1516
|
+
if (!index2.includes(valid.id)) await store.set(indexKey, [...index2, valid.id].sort());
|
|
1517
|
+
await store.set(`agentskit-harness:memory:${valid.id}`, valid);
|
|
1518
|
+
},
|
|
1519
|
+
async recall({ query, issueId, project, sourceRevision }) {
|
|
1520
|
+
const ids = await store.get(indexKey);
|
|
1521
|
+
const records = Array.isArray(ids) ? await Promise.all(ids.filter((id2) => typeof id2 === "string").map((id2) => store.get(`agentskit-harness:memory:${id2}`))) : [];
|
|
1522
|
+
return records.filter((record3) => Boolean(record3 && typeof record3 === "object" && record3.approved === true)).filter((record3) => matches2(record3, query.trim().toLowerCase(), issueId, project)).map((record3) => ({ record: record3, relevant: true, stale: sourceRevision !== void 0 && record3.sourceRevision !== sourceRevision }));
|
|
1523
|
+
}
|
|
1524
|
+
};
|
|
1525
|
+
};
|
|
1526
|
+
|
|
1527
|
+
// src/workflow.ts
|
|
1528
|
+
var validId2 = (id2) => {
|
|
1529
|
+
if (typeof id2 !== "string" || !id2.trim()) fail("Workflow node id must be non-empty.", "INVALID_INPUT");
|
|
1530
|
+
return id2.trim();
|
|
1531
|
+
};
|
|
1532
|
+
var levels = (nodes) => {
|
|
1533
|
+
const byId = new Map(nodes.map((node) => [validId2(node.id), node]));
|
|
1534
|
+
if (byId.size !== nodes.length) fail("Workflow node ids must be unique.", "INVALID_INPUT");
|
|
1535
|
+
const remaining = new Set(byId.keys());
|
|
1536
|
+
const completed = /* @__PURE__ */ new Set();
|
|
1537
|
+
const result = [];
|
|
1538
|
+
while (remaining.size) {
|
|
1539
|
+
const ready = [...remaining].sort().map((id2) => byId.get(id2)).filter((node) => (node.dependsOn ?? []).every((dependency) => completed.has(dependency)));
|
|
1540
|
+
if (!ready.length) fail("Workflow contains an unknown dependency or cycle.", "INVALID_INPUT");
|
|
1541
|
+
result.push(ready);
|
|
1542
|
+
for (const node of ready) {
|
|
1543
|
+
remaining.delete(node.id);
|
|
1544
|
+
completed.add(node.id);
|
|
1545
|
+
}
|
|
1546
|
+
}
|
|
1547
|
+
return result;
|
|
1548
|
+
};
|
|
1549
|
+
var runWorkflow = async (nodes, options) => {
|
|
1550
|
+
if (!Number.isInteger(options.maxConcurrency) || options.maxConcurrency < 1) fail("maxConcurrency must be a positive integer.", "INVALID_INPUT");
|
|
1551
|
+
const started = Date.now();
|
|
1552
|
+
const results = {};
|
|
1553
|
+
const order = [];
|
|
1554
|
+
let peakConcurrency = 0;
|
|
1555
|
+
for (const level of levels(nodes)) {
|
|
1556
|
+
const remaining = [...level];
|
|
1557
|
+
while (remaining.length) {
|
|
1558
|
+
const batch = [];
|
|
1559
|
+
const keys = /* @__PURE__ */ new Set();
|
|
1560
|
+
const limit = options.currentConcurrency ? options.currentConcurrency() : options.maxConcurrency;
|
|
1561
|
+
if (!Number.isInteger(limit) || limit < 1) fail("currentConcurrency must return a positive integer.", "INVALID_INPUT");
|
|
1562
|
+
for (const node of remaining) {
|
|
1563
|
+
const key = node.mutationKey?.trim();
|
|
1564
|
+
if (batch.length >= limit || key && keys.has(key)) continue;
|
|
1565
|
+
batch.push(node);
|
|
1566
|
+
if (key) keys.add(key);
|
|
1567
|
+
}
|
|
1568
|
+
if (!batch.length) fail("Workflow could not schedule a mutation batch.", "INVALID_INPUT");
|
|
1569
|
+
peakConcurrency = Math.max(peakConcurrency, batch.length);
|
|
1570
|
+
const values = await Promise.all(batch.map((node) => node.run()));
|
|
1571
|
+
batch.forEach((node, index2) => {
|
|
1572
|
+
results[node.id] = values[index2];
|
|
1573
|
+
order.push(node.id);
|
|
1574
|
+
});
|
|
1575
|
+
for (const node of batch) remaining.splice(remaining.indexOf(node), 1);
|
|
1576
|
+
}
|
|
1577
|
+
}
|
|
1578
|
+
return { results, order, peakConcurrency, criticalPathMs: Date.now() - started };
|
|
1579
|
+
};
|
|
907
1580
|
var BENCHMARK_SCHEMA_VERSION = 1;
|
|
908
1581
|
var percentage = (part, total) => total ? Number((part / total).toFixed(4)) : null;
|
|
909
|
-
var DEFAULT_POLICY = { minComparableTasks: 3, maxDurationRegressionRate: 0.2, minCompletedRunsPerTask: 3, minBaselineSamplesPerTask: 3, requireZeroEscapedIncomplete: true };
|
|
910
1582
|
var improvementRate = (baseline, current) => baseline === void 0 || current === void 0 || baseline === 0 ? null : Number(((baseline - current) / baseline).toFixed(4));
|
|
911
|
-
var increaseRate = (baseline, current) => baseline === void 0 || current === void 0 || baseline === 0 ? null : Number(((current - baseline) / baseline).toFixed(4));
|
|
912
|
-
var increaseDelta = (baseline, current) => baseline === void 0 || current === null ? null : Number((current - baseline).toFixed(4));
|
|
913
|
-
var increaseDirection = (rate2, delta) => delta === null ? improvementDirection(rate2) : delta > 0 ? "improved" : delta < 0 ? "regressed" : "unchanged";
|
|
914
1583
|
var improvementDirection = (rate2) => rate2 === null ? "unavailable" : rate2 > 0 ? "improved" : rate2 < 0 ? "regressed" : "unchanged";
|
|
915
1584
|
var count = (items, predicate) => items.filter(predicate).length;
|
|
916
1585
|
var median = (values) => {
|
|
@@ -926,25 +1595,13 @@ var reviewMinutes = (run) => {
|
|
|
926
1595
|
const elapsed = Date.parse(run.humanApproval.at) - Date.parse(reviewStart);
|
|
927
1596
|
return Number.isFinite(elapsed) && elapsed >= 0 ? Number((elapsed / 6e4).toFixed(2)) : void 0;
|
|
928
1597
|
};
|
|
929
|
-
var confidence = (comparable, completedRuns, policy) => !comparable ? "insufficient" : completedRuns >= policy.minCompletedRunsPerTask ? "reliable" : "directional";
|
|
930
|
-
var artifactAcceptanceRate = (run) => {
|
|
931
|
-
const rates = run.checks.flatMap((check) => {
|
|
932
|
-
const evidence = check.evidence;
|
|
933
|
-
if (!evidence) return [];
|
|
934
|
-
const direct = typeof evidence["artifactAcceptanceRate"] === "number" ? [evidence["artifactAcceptanceRate"]] : [];
|
|
935
|
-
const reports = Array.isArray(evidence["reports"]) ? evidence["reports"].flatMap((report) => typeof report === "object" && report !== null && typeof report["artifactAcceptanceRate"] === "number" ? [report["artifactAcceptanceRate"]] : []) : [];
|
|
936
|
-
return [...direct, ...reports].filter((rate2) => Number.isFinite(rate2) && rate2 >= 0 && rate2 <= 1);
|
|
937
|
-
});
|
|
938
|
-
return rates.length ? Number((rates.reduce((total, rate2) => total + rate2, 0) / rates.length).toFixed(4)) : void 0;
|
|
939
|
-
};
|
|
940
1598
|
var projectRun = (run) => {
|
|
941
1599
|
const checks = { total: run.checks.length, passed: count(run.checks, (check) => check.status === "passed"), failed: count(run.checks, (check) => check.status === "failed") };
|
|
942
1600
|
const outcomes = { total: run.outcomes.length, passed: count(run.outcomes, (outcome) => outcome.status === "passed"), failed: count(run.outcomes, (outcome) => outcome.status === "failed") };
|
|
943
1601
|
const evidence = { total: run.checks.length, attached: count(run.checks, (check) => check.evidence !== void 0) };
|
|
944
|
-
const acceptanceRate = artifactAcceptanceRate(run);
|
|
945
1602
|
const humanReviewMinutes = reviewMinutes(run);
|
|
946
1603
|
const escapedIncomplete = run.state === "COMPLETE" && checks.failed === 0 && outcomes.failed === 0 && evidence.attached === evidence.total ? 0 : void 0;
|
|
947
|
-
return { runId: run.runId, state: run.state, sourceRevision: run.sourceRevision, configHash: run.configHash, contractHash: run.contractHash, ...run.supersedes ? { supersedes: run.supersedes } : {}, ...run.metrics ? { durationMs: run.metrics.totalDurationMs } : {}, checks, outcomes, evidence, ...
|
|
1604
|
+
return { runId: run.runId, state: run.state, sourceRevision: run.sourceRevision, configHash: run.configHash, contractHash: run.contractHash, ...run.supersedes ? { supersedes: run.supersedes } : {}, ...run.metrics ? { durationMs: run.metrics.totalDurationMs } : {}, checks, outcomes, evidence, ...escapedIncomplete === void 0 ? {} : { escapedIncomplete }, humanApproved: run.humanApproval !== void 0, ...humanReviewMinutes === void 0 ? {} : { humanReviewMinutes }, authorized: run.authorization !== void 0, ...run.metrics?.machine ? { machine: run.metrics.machine } : {}, ...run.benchmark ? { benchmark: run.benchmark } : {} };
|
|
948
1605
|
};
|
|
949
1606
|
var summarize = (runs) => {
|
|
950
1607
|
const stateCounts = Object.fromEntries(RUN_STATES.map((state) => [state, count(runs, (run) => run.state === state)]));
|
|
@@ -955,14 +1612,6 @@ var summarize = (runs) => {
|
|
|
955
1612
|
const evidenceTotal = runs.reduce((total, run) => total + run.evidence.total, 0);
|
|
956
1613
|
const evidenceAttached = runs.reduce((total, run) => total + run.evidence.attached, 0);
|
|
957
1614
|
const firstAttempts = runs.filter((run) => !run.supersedes);
|
|
958
|
-
const superseded = new Set(runs.flatMap((run) => run.supersedes ? [run.supersedes] : []));
|
|
959
|
-
const effectiveRuns = runs.filter((run) => !superseded.has(run.runId));
|
|
960
|
-
const effectiveChecksTotal = effectiveRuns.reduce((total, run) => total + run.checks.total, 0);
|
|
961
|
-
const effectiveChecksPassed = effectiveRuns.reduce((total, run) => total + run.checks.passed, 0);
|
|
962
|
-
const effectiveOutcomesTotal = effectiveRuns.reduce((total, run) => total + run.outcomes.total, 0);
|
|
963
|
-
const effectiveOutcomesPassed = effectiveRuns.reduce((total, run) => total + run.outcomes.passed, 0);
|
|
964
|
-
const effectiveEvidenceTotal = effectiveRuns.reduce((total, run) => total + run.evidence.total, 0);
|
|
965
|
-
const effectiveEvidenceAttached = effectiveRuns.reduce((total, run) => total + run.evidence.attached, 0);
|
|
966
1615
|
const durations = runs.flatMap((run) => run.durationMs === void 0 ? [] : [run.durationMs]);
|
|
967
1616
|
return {
|
|
968
1617
|
totalRuns: runs.length,
|
|
@@ -973,12 +1622,6 @@ var summarize = (runs) => {
|
|
|
973
1622
|
firstAttemptRuns: firstAttempts.length,
|
|
974
1623
|
humanApprovedRuns: count(runs, (run) => run.humanApproved),
|
|
975
1624
|
authorizedRuns: count(runs, (run) => run.authorized),
|
|
976
|
-
effectiveRunCount: effectiveRuns.length,
|
|
977
|
-
effectiveCompleteRuns: count(effectiveRuns, (run) => run.state === "COMPLETE"),
|
|
978
|
-
effectiveCompletionRate: percentage(count(effectiveRuns, (run) => run.state === "COMPLETE"), effectiveRuns.length),
|
|
979
|
-
effectiveCheckPassRate: percentage(effectiveChecksPassed, effectiveChecksTotal),
|
|
980
|
-
effectiveOutcomePassRate: percentage(effectiveOutcomesPassed, effectiveOutcomesTotal),
|
|
981
|
-
effectiveEvidenceCoverageRate: percentage(effectiveEvidenceAttached, effectiveEvidenceTotal),
|
|
982
1625
|
checkPassRate: percentage(checksPassed, checksTotal),
|
|
983
1626
|
outcomePassRate: percentage(outcomesPassed, outcomesTotal),
|
|
984
1627
|
evidenceCoverageRate: percentage(evidenceAttached, evidenceTotal),
|
|
@@ -1026,73 +1669,16 @@ var nonNegativeNumber = (value, label) => {
|
|
|
1026
1669
|
const result = value;
|
|
1027
1670
|
return result;
|
|
1028
1671
|
};
|
|
1029
|
-
var
|
|
1672
|
+
var nonNegativeInteger2 = (value, label) => {
|
|
1030
1673
|
const result = nonNegativeNumber(value, label);
|
|
1031
1674
|
if (result !== void 0 && !Number.isInteger(result)) return fail(`${label} must be an integer.`, "INVALID_CONFIG");
|
|
1032
1675
|
return result;
|
|
1033
1676
|
};
|
|
1034
|
-
var rate = (value, label) => {
|
|
1035
|
-
const result = nonNegativeNumber(value, label);
|
|
1036
|
-
if (result !== void 0 && result > 1) return fail(`${label} must be between 0 and 1.`, "INVALID_CONFIG");
|
|
1037
|
-
return result;
|
|
1038
|
-
};
|
|
1039
1677
|
var timestamp = (value, label) => {
|
|
1040
1678
|
const result = nonEmptyString(value, label);
|
|
1041
1679
|
if (!Number.isFinite(Date.parse(result))) return fail(`${label} must be a valid timestamp.`, "INVALID_CONFIG");
|
|
1042
1680
|
return result;
|
|
1043
1681
|
};
|
|
1044
|
-
var relativePath = (value, label) => {
|
|
1045
|
-
const result = nonEmptyString(value, label);
|
|
1046
|
-
if (result.startsWith("/") || result.split("/").includes("..")) return fail(`${label} must be a repository-relative path.`, "INVALID_CONFIG");
|
|
1047
|
-
return result;
|
|
1048
|
-
};
|
|
1049
|
-
var taskFile = (value, label) => {
|
|
1050
|
-
if (value === void 0) return void 0;
|
|
1051
|
-
if (typeof value !== "object" || value === null || Array.isArray(value)) return fail(`${label} must be an object.`, "INVALID_CONFIG");
|
|
1052
|
-
const raw = value;
|
|
1053
|
-
return { path: relativePath(raw["path"], `${label}.path`), sha256: sha2562(raw["sha256"], `${label}.sha256`) ?? fail(`${label}.sha256 is required.`, "INVALID_CONFIG") };
|
|
1054
|
-
};
|
|
1055
|
-
var taskSource = (value, label) => {
|
|
1056
|
-
if (value === void 0) return void 0;
|
|
1057
|
-
if (typeof value !== "object" || value === null || Array.isArray(value)) return fail(`${label} must be an object.`, "INVALID_CONFIG");
|
|
1058
|
-
const raw = value;
|
|
1059
|
-
return { repository: nonEmptyString(raw["repository"], `${label}.repository`), path: relativePath(raw["path"], `${label}.path`), revision: nonEmptyString(raw["revision"], `${label}.revision`) };
|
|
1060
|
-
};
|
|
1061
|
-
var suiteSource = (value, label) => {
|
|
1062
|
-
if (value === void 0) return void 0;
|
|
1063
|
-
if (typeof value !== "object" || value === null || Array.isArray(value)) return fail(`${label} must be an object.`, "INVALID_CONFIG");
|
|
1064
|
-
const raw = value;
|
|
1065
|
-
return { repository: nonEmptyString(raw["repository"], `${label}.repository`), revision: nonEmptyString(raw["revision"], `${label}.revision`), taskDefinition: relativePath(raw["taskDefinition"], `${label}.taskDefinition`) };
|
|
1066
|
-
};
|
|
1067
|
-
var taskScope = (value, label) => {
|
|
1068
|
-
if (value === void 0) return void 0;
|
|
1069
|
-
if (typeof value !== "object" || value === null || Array.isArray(value)) return fail(`${label} must be an object.`, "INVALID_CONFIG");
|
|
1070
|
-
const raw = value;
|
|
1071
|
-
return { read: stringList(raw["read"], `${label}.read`), write: stringList(raw["write"], `${label}.write`) };
|
|
1072
|
-
};
|
|
1073
|
-
var taskSurfaces = (value, label) => {
|
|
1074
|
-
if (value === void 0) return void 0;
|
|
1075
|
-
if (!Array.isArray(value) || !value.length) return fail(`${label} must be a non-empty array.`, "INVALID_CONFIG");
|
|
1076
|
-
const surfaces = value.map((item, index2) => nonEmptyString(item, `${label}[${index2}]`));
|
|
1077
|
-
if (surfaces.some((surface2) => !SURFACE_NAMES.includes(surface2))) fail(`${label} contains an unknown surface.`, "INVALID_CONFIG");
|
|
1078
|
-
if (new Set(surfaces).size !== surfaces.length) fail(`${label} must contain unique surfaces.`, "INVALID_CONFIG");
|
|
1079
|
-
return surfaces;
|
|
1080
|
-
};
|
|
1081
|
-
var benchmarkPolicy = (value) => {
|
|
1082
|
-
if (value === void 0) return void 0;
|
|
1083
|
-
if (typeof value !== "object" || value === null || Array.isArray(value)) return fail("benchmark.policy must be an object.", "INVALID_CONFIG");
|
|
1084
|
-
const raw = value;
|
|
1085
|
-
const minComparableTasks = nonNegativeInteger(raw["minComparableTasks"], "benchmark.policy.minComparableTasks");
|
|
1086
|
-
const maxDurationRegressionRate = nonNegativeNumber(raw["maxDurationRegressionRate"], "benchmark.policy.maxDurationRegressionRate");
|
|
1087
|
-
const minCompletedRunsPerTask = nonNegativeInteger(raw["minCompletedRunsPerTask"], "benchmark.policy.minCompletedRunsPerTask");
|
|
1088
|
-
const minBaselineSamplesPerTask = nonNegativeInteger(raw["minBaselineSamplesPerTask"] ?? 1, "benchmark.policy.minBaselineSamplesPerTask");
|
|
1089
|
-
if (minComparableTasks === void 0 || minComparableTasks < 1) return fail("benchmark.policy.minComparableTasks must be at least 1.", "INVALID_CONFIG");
|
|
1090
|
-
if (maxDurationRegressionRate === void 0 || maxDurationRegressionRate > 1) return fail("benchmark.policy.maxDurationRegressionRate must be between 0 and 1.", "INVALID_CONFIG");
|
|
1091
|
-
if (minCompletedRunsPerTask === void 0 || minCompletedRunsPerTask < 1) return fail("benchmark.policy.minCompletedRunsPerTask must be at least 1.", "INVALID_CONFIG");
|
|
1092
|
-
if (minBaselineSamplesPerTask === void 0 || minBaselineSamplesPerTask < 1) return fail("benchmark.policy.minBaselineSamplesPerTask must be at least 1.", "INVALID_CONFIG");
|
|
1093
|
-
if (typeof raw["requireZeroEscapedIncomplete"] !== "boolean") return fail("benchmark.policy.requireZeroEscapedIncomplete must be boolean.", "INVALID_CONFIG");
|
|
1094
|
-
return { minComparableTasks, maxDurationRegressionRate, minCompletedRunsPerTask, minBaselineSamplesPerTask, requireZeroEscapedIncomplete: raw["requireZeroEscapedIncomplete"] };
|
|
1095
|
-
};
|
|
1096
1682
|
var validateBenchmarkManifest = (value) => {
|
|
1097
1683
|
if (typeof value !== "object" || value === null || Array.isArray(value)) fail("benchmark manifest must be an object.", "INVALID_CONFIG");
|
|
1098
1684
|
const raw = value;
|
|
@@ -1101,12 +1687,7 @@ var validateBenchmarkManifest = (value) => {
|
|
|
1101
1687
|
const tasks = rawTasks.map((item, index2) => {
|
|
1102
1688
|
if (typeof item !== "object" || item === null || Array.isArray(item)) fail(`benchmark.tasks[${index2}] must be an object.`, "INVALID_CONFIG");
|
|
1103
1689
|
const task = item;
|
|
1104
|
-
|
|
1105
|
-
const prompt = taskFile(task["prompt"], `benchmark.tasks[${index2}].prompt`);
|
|
1106
|
-
const source = taskSource(task["source"], `benchmark.tasks[${index2}].source`);
|
|
1107
|
-
const scope = taskScope(task["scope"], `benchmark.tasks[${index2}].scope`);
|
|
1108
|
-
const surfaces = taskSurfaces(task["surfaces"], `benchmark.tasks[${index2}].surfaces`);
|
|
1109
|
-
return { id: nonEmptyString(task["id"], `benchmark.tasks[${index2}].id`), title: nonEmptyString(task["title"], `benchmark.tasks[${index2}].title`), acceptanceCriteria: stringList(task["acceptanceCriteria"], `benchmark.tasks[${index2}].acceptanceCriteria`), ...surfaces === void 0 ? {} : { surfaces }, ...kind === void 0 ? {} : { kind }, ...prompt === void 0 ? {} : { prompt }, ...source === void 0 ? {} : { source }, ...scope === void 0 ? {} : { scope } };
|
|
1690
|
+
return { id: nonEmptyString(task["id"], `benchmark.tasks[${index2}].id`), title: nonEmptyString(task["title"], `benchmark.tasks[${index2}].title`), acceptanceCriteria: stringList(task["acceptanceCriteria"], `benchmark.tasks[${index2}].acceptanceCriteria`) };
|
|
1110
1691
|
});
|
|
1111
1692
|
if (new Set(tasks.map((task) => task.id)).size !== tasks.length) fail("benchmark task ids must be unique.", "INVALID_CONFIG");
|
|
1112
1693
|
const taskIds = new Set(tasks.map((task) => task.id));
|
|
@@ -1119,14 +1700,10 @@ var validateBenchmarkManifest = (value) => {
|
|
|
1119
1700
|
const taskId = nonEmptyString(observation["taskId"], `benchmark.observations[${index2}].taskId`);
|
|
1120
1701
|
if (!taskIds.has(taskId)) fail(`benchmark observation references unknown task: ${taskId}.`, "INVALID_CONFIG");
|
|
1121
1702
|
const task = tasks.find((candidate) => candidate.id === taskId) ?? fail(`benchmark observation references unknown task: ${taskId}.`, "INVALID_CONFIG");
|
|
1122
|
-
const attempts =
|
|
1703
|
+
const attempts = nonNegativeInteger2(observation["attempts"], `benchmark.observations[${index2}].attempts`);
|
|
1123
1704
|
const durationMs = nonNegativeNumber(observation["durationMs"], `benchmark.observations[${index2}].durationMs`);
|
|
1124
|
-
const durationSamplesMs = observation["durationSamplesMs"] === void 0 ? void 0 : Array.isArray(observation["durationSamplesMs"]) ? observation["durationSamplesMs"].map((sample, sampleIndex) => nonNegativeNumber(sample, `benchmark.observations[${index2}].durationSamplesMs[${sampleIndex}]`) ?? fail(`benchmark.observations[${index2}].durationSamplesMs must contain numbers.`, "INVALID_CONFIG")) : fail(`benchmark.observations[${index2}].durationSamplesMs must be an array.`, "INVALID_CONFIG");
|
|
1125
|
-
if (durationSamplesMs && !durationSamplesMs.length) fail(`benchmark.observations[${index2}].durationSamplesMs must not be empty.`, "INVALID_CONFIG");
|
|
1126
|
-
const artifactAcceptanceRate2 = rate(observation["artifactAcceptanceRate"], `benchmark.observations[${index2}].artifactAcceptanceRate`);
|
|
1127
|
-
const protocolCompletionRate = rate(observation["protocolCompletionRate"], `benchmark.observations[${index2}].protocolCompletionRate`);
|
|
1128
1705
|
const reviewMinutes2 = nonNegativeNumber(observation["reviewMinutes"], `benchmark.observations[${index2}].reviewMinutes`);
|
|
1129
|
-
const escapedIncomplete =
|
|
1706
|
+
const escapedIncomplete = nonNegativeInteger2(observation["escapedIncomplete"], `benchmark.observations[${index2}].escapedIncomplete`);
|
|
1130
1707
|
const evidenceDigest = sha2562(observation["evidenceDigest"], `benchmark.observations[${index2}].evidenceDigest`);
|
|
1131
1708
|
const rawEvidence = observation["evidence"] === void 0 ? void 0 : Array.isArray(observation["evidence"]) ? observation["evidence"] : fail(`benchmark.observations[${index2}].evidence must be an array.`, "INVALID_CONFIG");
|
|
1132
1709
|
const evidence = rawEvidence?.map((item2, evidenceIndex) => {
|
|
@@ -1139,12 +1716,10 @@ var validateBenchmarkManifest = (value) => {
|
|
|
1139
1716
|
return { criterion, status: evidenceStatus, source: nonEmptyString(entry["source"], `benchmark.observations[${index2}].evidence[${evidenceIndex}].source`) };
|
|
1140
1717
|
});
|
|
1141
1718
|
if (evidence && new Set(evidence.map((entry) => entry.criterion)).size !== evidence.length) fail(`benchmark.observations[${index2}].evidence criteria must be unique.`, "INVALID_CONFIG");
|
|
1142
|
-
return { taskId, mode: "baseline", status, source: nonEmptyString(observation["source"], `benchmark.observations[${index2}].source`), recordedAt: timestamp(observation["recordedAt"], `benchmark.observations[${index2}].recordedAt`), ...attempts === void 0 ? {} : { attempts }, ...durationMs === void 0 ? {} : { durationMs }, ...
|
|
1719
|
+
return { taskId, mode: "baseline", status, source: nonEmptyString(observation["source"], `benchmark.observations[${index2}].source`), recordedAt: timestamp(observation["recordedAt"], `benchmark.observations[${index2}].recordedAt`), ...attempts === void 0 ? {} : { attempts }, ...durationMs === void 0 ? {} : { durationMs }, ...reviewMinutes2 === void 0 ? {} : { reviewMinutes: reviewMinutes2 }, ...escapedIncomplete === void 0 ? {} : { escapedIncomplete }, ...evidence === void 0 ? {} : { evidence }, ...evidenceDigest === void 0 ? {} : { evidenceDigest } };
|
|
1143
1720
|
});
|
|
1144
1721
|
if (new Set(observations.map((observation) => observation.taskId)).size !== observations.length) fail("benchmark allows at most one baseline observation per task.", "INVALID_CONFIG");
|
|
1145
|
-
|
|
1146
|
-
const policy = benchmarkPolicy(raw["policy"]);
|
|
1147
|
-
return { type: "agentskit-harness-benchmark-manifest", schemaVersion: BENCHMARK_SCHEMA_VERSION, suiteId: nonEmptyString(raw["suiteId"], "benchmark.suiteId"), name: nonEmptyString(raw["name"], "benchmark.name"), ...provenance === void 0 ? {} : { provenance }, tasks, observations, ...policy === void 0 ? {} : { policy } };
|
|
1722
|
+
return { type: "agentskit-harness-benchmark-manifest", schemaVersion: BENCHMARK_SCHEMA_VERSION, suiteId: nonEmptyString(raw["suiteId"], "benchmark.suiteId"), name: nonEmptyString(raw["name"], "benchmark.name"), tasks, observations };
|
|
1148
1723
|
};
|
|
1149
1724
|
var loadBenchmarkManifest = (path) => {
|
|
1150
1725
|
try {
|
|
@@ -1170,9 +1745,6 @@ var recordBenchmarkObservation = (path, input) => {
|
|
|
1170
1745
|
recordedAt: input.recordedAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
1171
1746
|
...input.attempts === void 0 ? {} : { attempts: input.attempts },
|
|
1172
1747
|
...input.durationMs === void 0 ? {} : { durationMs: input.durationMs },
|
|
1173
|
-
...input.durationSamplesMs === void 0 ? {} : { durationSamplesMs: input.durationSamplesMs },
|
|
1174
|
-
...input.artifactAcceptanceRate === void 0 ? {} : { artifactAcceptanceRate: input.artifactAcceptanceRate },
|
|
1175
|
-
...input.protocolCompletionRate === void 0 ? {} : { protocolCompletionRate: input.protocolCompletionRate },
|
|
1176
1748
|
...input.reviewMinutes === void 0 ? {} : { reviewMinutes: input.reviewMinutes },
|
|
1177
1749
|
...input.escapedIncomplete === void 0 ? {} : { escapedIncomplete: input.escapedIncomplete },
|
|
1178
1750
|
...input.evidence === void 0 ? {} : { evidence: input.evidence },
|
|
@@ -1191,99 +1763,27 @@ var recordBenchmarkObservation = (path, input) => {
|
|
|
1191
1763
|
}
|
|
1192
1764
|
return observation;
|
|
1193
1765
|
};
|
|
1194
|
-
var comparisons = (runs, manifest
|
|
1766
|
+
var comparisons = (runs, manifest) => manifest.tasks.map((task) => {
|
|
1195
1767
|
const taskRuns = runs.filter((run) => run.benchmark?.suiteId === manifest.suiteId && run.benchmark.taskId === task.id);
|
|
1196
1768
|
const latest = taskRuns.at(-1);
|
|
1197
1769
|
const baseline = manifest.observations.find((observation) => observation.taskId === task.id);
|
|
1198
1770
|
const coveredCriteria = new Set((baseline?.evidence ?? []).map((entry) => entry.criterion));
|
|
1199
1771
|
const baselineEvidenceCoverageRate = baseline ? percentage(coveredCriteria.size, task.acceptanceCriteria.length) : null;
|
|
1200
1772
|
const baselineEvidenceComplete = baselineEvidenceCoverageRate === 1;
|
|
1201
|
-
const
|
|
1202
|
-
const
|
|
1203
|
-
const
|
|
1204
|
-
const
|
|
1205
|
-
const
|
|
1206
|
-
const
|
|
1207
|
-
|
|
1208
|
-
const completedTaskRuns = taskRuns.filter((run) => run.state === "COMPLETE");
|
|
1209
|
-
const durationSamplesMs = completedTaskRuns.flatMap((run) => run.durationMs === void 0 ? [] : [run.durationMs]);
|
|
1210
|
-
const medianDurationMs = median(durationSamplesMs);
|
|
1211
|
-
const retryCount = count(taskRuns, (run) => run.supersedes !== void 0);
|
|
1212
|
-
const attempts = retryCount + (taskRuns.length ? 1 : 0);
|
|
1213
|
-
const durationRate = comparable ? improvementRate(baselineMedianDurationMs ?? void 0, medianDurationMs ?? void 0) : null;
|
|
1214
|
-
const attemptsRate = comparable ? improvementRate(baseline?.attempts, attempts) : null;
|
|
1215
|
-
const reviewRate = comparable ? improvementRate(baseline?.reviewMinutes, latest?.humanReviewMinutes) : null;
|
|
1216
|
-
const acceptanceSamples = taskRuns.flatMap((run) => run.artifactAcceptanceRate === void 0 ? [] : [run.artifactAcceptanceRate]);
|
|
1217
|
-
const harnessArtifactAcceptanceRate = acceptanceSamples.length ? Number((acceptanceSamples.reduce((total, rate2) => total + rate2, 0) / acceptanceSamples.length).toFixed(4)) : null;
|
|
1218
|
-
const artifactAcceptanceImprovementRate = increaseRate(baseline?.artifactAcceptanceRate, harnessArtifactAcceptanceRate ?? void 0);
|
|
1219
|
-
const artifactAcceptanceDelta = increaseDelta(baseline?.artifactAcceptanceRate, harnessArtifactAcceptanceRate);
|
|
1220
|
-
const completedRuns = completedTaskRuns.length;
|
|
1221
|
-
const harnessProtocolCompletionRate = taskRuns.length ? percentage(completedRuns, taskRuns.length) : null;
|
|
1222
|
-
const protocolCompletionImprovementRate = increaseRate(baseline?.protocolCompletionRate, harnessProtocolCompletionRate ?? void 0);
|
|
1223
|
-
const protocolCompletionDelta = increaseDelta(baseline?.protocolCompletionRate, harnessProtocolCompletionRate);
|
|
1224
|
-
const escapedIncompleteRate = improvementRate(baseline?.escapedIncomplete, latest?.escapedIncomplete);
|
|
1225
|
-
return { taskId: task.id, title: task.title, comparability, comparable, baselineDeliveryComplete, baselineEvidenceCoverageRate, baselineSampleCount: baselineDurationSamples.length, baselineArtifactAcceptanceRate: baseline?.artifactAcceptanceRate ?? null, baselineProtocolCompletionRate: baseline?.protocolCompletionRate ?? null, ...baselineMedianDurationMs === null ? {} : { baselineMedianDurationMs }, improvement: { durationRate, duration: improvementDirection(durationRate), attemptsRate, attempts: improvementDirection(attemptsRate), reviewRate, review: improvementDirection(reviewRate), artifactAcceptanceRate: artifactAcceptanceImprovementRate, artifactAcceptance: increaseDirection(artifactAcceptanceImprovementRate, artifactAcceptanceDelta), artifactAcceptanceDelta, protocolCompletionRate: protocolCompletionImprovementRate, protocolCompletion: increaseDirection(protocolCompletionImprovementRate, protocolCompletionDelta), protocolCompletionDelta, escapedIncompleteRate, escapedIncomplete: improvementDirection(escapedIncompleteRate) }, ...baseline ? { baseline } : {}, harness: { attempts, retryCount, completedRuns, durationSamplesMs, ...medianDurationMs === null ? {} : { medianDurationMs }, ...harnessArtifactAcceptanceRate === null ? {} : { artifactAcceptanceRate: harnessArtifactAcceptanceRate }, artifactAcceptanceSampleCount: acceptanceSamples.length, protocolCompletionRate: harnessProtocolCompletionRate, protocolCompletionSampleCount: taskRuns.length, latestState: latest?.state ?? "NOT_RUN", ...latest ? { latestRunId: latest.runId } : {}, ...latest?.durationMs === void 0 ? {} : { latestDurationMs: latest.durationMs }, checkPassRate: latest ? percentage(latest.checks.passed, latest.checks.total) : null, outcomePassRate: latest ? percentage(latest.outcomes.passed, latest.outcomes.total) : null, evidenceCoverageRate: latest ? percentage(latest.evidence.attached, latest.evidence.total) : null, ...latest?.escapedIncomplete === void 0 ? {} : { escapedIncomplete: latest.escapedIncomplete }, ...latest?.humanReviewMinutes === void 0 ? {} : { humanReviewMinutes: latest.humanReviewMinutes }, humanApproved: latest?.humanApproved ?? false }, confidence: confidence(comparable, completedRuns, policy), ...comparable && baselineMedianDurationMs !== null && medianDurationMs !== null ? { durationDeltaMs: medianDurationMs - baselineMedianDurationMs } : {}, ...comparable && baseline?.attempts !== void 0 ? { attemptDelta: attempts - baseline.attempts } : {}, ...comparable && baseline?.reviewMinutes !== void 0 && latest?.humanReviewMinutes !== void 0 ? { reviewDeltaMinutes: latest.humanReviewMinutes - baseline.reviewMinutes } : {}, ...baseline?.escapedIncomplete !== void 0 && latest?.escapedIncomplete !== void 0 ? { escapedIncompleteDelta: latest.escapedIncomplete - baseline.escapedIncomplete } : {} };
|
|
1773
|
+
const comparable2 = baseline !== void 0 && baseline.status !== "not-run" && baselineEvidenceComplete && latest?.state === "COMPLETE";
|
|
1774
|
+
const comparability = comparable2 ? "comparable" : baseline === void 0 ? "missing-baseline" : baseline.status === "not-run" ? "baseline-not-run" : !baselineEvidenceComplete ? "baseline-evidence-missing" : latest === void 0 ? "harness-not-run" : "harness-not-complete";
|
|
1775
|
+
const durationRate = comparable2 ? improvementRate(baseline?.durationMs, latest?.durationMs) : null;
|
|
1776
|
+
const attemptsRate = comparable2 ? improvementRate(baseline?.attempts, taskRuns.length) : null;
|
|
1777
|
+
const reviewRate = comparable2 ? improvementRate(baseline?.reviewMinutes, latest?.humanReviewMinutes) : null;
|
|
1778
|
+
const escapedIncompleteRate = comparable2 ? improvementRate(baseline?.escapedIncomplete, latest?.escapedIncomplete) : null;
|
|
1779
|
+
return { taskId: task.id, title: task.title, comparability, comparable: comparable2, baselineEvidenceCoverageRate, improvement: { durationRate, duration: improvementDirection(durationRate), attemptsRate, attempts: improvementDirection(attemptsRate), reviewRate, review: improvementDirection(reviewRate), escapedIncompleteRate, escapedIncomplete: improvementDirection(escapedIncompleteRate) }, ...baseline ? { baseline } : {}, harness: { attempts: taskRuns.length, latestState: latest?.state ?? "NOT_RUN", ...latest ? { latestRunId: latest.runId } : {}, ...latest?.durationMs === void 0 ? {} : { latestDurationMs: latest.durationMs }, checkPassRate: latest ? percentage(latest.checks.passed, latest.checks.total) : null, outcomePassRate: latest ? percentage(latest.outcomes.passed, latest.outcomes.total) : null, evidenceCoverageRate: latest ? percentage(latest.evidence.attached, latest.evidence.total) : null, ...latest?.escapedIncomplete === void 0 ? {} : { escapedIncomplete: latest.escapedIncomplete }, ...latest?.humanReviewMinutes === void 0 ? {} : { humanReviewMinutes: latest.humanReviewMinutes }, humanApproved: latest?.humanApproved ?? false }, ...comparable2 && baseline?.durationMs !== void 0 && latest?.durationMs !== void 0 ? { durationDeltaMs: latest.durationMs - baseline.durationMs } : {}, ...comparable2 && baseline?.attempts !== void 0 ? { attemptDelta: taskRuns.length - baseline.attempts } : {}, ...comparable2 && baseline?.reviewMinutes !== void 0 && latest?.humanReviewMinutes !== void 0 ? { reviewDeltaMinutes: latest.humanReviewMinutes - baseline.reviewMinutes } : {}, ...comparable2 && baseline?.escapedIncomplete !== void 0 && latest?.escapedIncomplete !== void 0 ? { escapedIncompleteDelta: latest.escapedIncomplete - baseline.escapedIncomplete } : {} };
|
|
1226
1780
|
});
|
|
1227
1781
|
var benchmarkRuns = (stateDir, manifest) => {
|
|
1228
1782
|
const runs = readRuns(stateDir).map(projectRun).sort((left, right) => left.runId.localeCompare(right.runId));
|
|
1229
|
-
const
|
|
1230
|
-
|
|
1231
|
-
const comparable = reportComparisons.filter((comparison) => comparison.comparable);
|
|
1232
|
-
const durationRegressionTaskIds = comparable.filter((comparison) => (comparison.improvement.durationRate ?? 0) < -policy.maxDurationRegressionRate).map((comparison) => comparison.taskId);
|
|
1233
|
-
const escapedIncompleteTaskIds = comparable.filter((comparison) => comparison.harness.escapedIncomplete !== 0).map((comparison) => comparison.taskId);
|
|
1234
|
-
const reasons = [];
|
|
1235
|
-
if (comparable.length < policy.minComparableTasks) reasons.push(`requires at least ${policy.minComparableTasks} comparable tasks`);
|
|
1236
|
-
const incompleteBaselineTaskIds = reportComparisons.filter((comparison) => comparison.comparability === "baseline-incomplete").map((comparison) => comparison.taskId);
|
|
1237
|
-
if (incompleteBaselineTaskIds.length) reasons.push(`baseline delivery incomplete: ${incompleteBaselineTaskIds.join(", ")}`);
|
|
1238
|
-
const baselineSampleGaps = reportComparisons.filter((comparison) => comparison.baselineSampleCount < policy.minBaselineSamplesPerTask).map((comparison) => `${comparison.taskId} (${comparison.baselineSampleCount}/${policy.minBaselineSamplesPerTask})`);
|
|
1239
|
-
if (baselineSampleGaps.length) reasons.push(`requires ${policy.minBaselineSamplesPerTask} baseline samples per task: ${baselineSampleGaps.join(", ")}`);
|
|
1240
|
-
if (durationRegressionTaskIds.length) reasons.push(`duration regression exceeds ${policy.maxDurationRegressionRate * 100}%: ${durationRegressionTaskIds.join(", ")}`);
|
|
1241
|
-
if (policy.requireZeroEscapedIncomplete && escapedIncompleteTaskIds.length) reasons.push(`escaped incomplete delivery: ${escapedIncompleteTaskIds.join(", ")}`);
|
|
1242
|
-
const confidenceLevel = comparable.length < policy.minComparableTasks ? "insufficient" : comparable.every((comparison) => comparison.confidence === "reliable") ? "reliable" : "directional";
|
|
1243
|
-
const qualityGate = { status: comparable.length < policy.minComparableTasks ? "insufficient-data" : reasons.length ? "failed" : "passed", confidence: confidenceLevel, comparableTaskCount: comparable.length, policy, durationRegressionTaskIds, escapedIncompleteTaskIds, reasons };
|
|
1244
|
-
return { type: "agentskit-harness-benchmark", schemaVersion: BENCHMARK_SCHEMA_VERSION, stateDir, generatedAt: (/* @__PURE__ */ new Date()).toISOString(), runs, summary: summarize(runs), comparisons: reportComparisons, qualityGate, ...manifest ? { manifest: { suiteId: manifest.suiteId, taskCount: manifest.tasks.length, baselineCount: manifest.observations.length, comparableTaskCount: comparable.length } } : {} };
|
|
1245
|
-
};
|
|
1246
|
-
|
|
1247
|
-
// src/coding-benchmark.ts
|
|
1248
|
-
var text2 = (value, label) => {
|
|
1249
|
-
if (typeof value !== "string" || !value.trim()) fail(`${label} must be a non-empty string.`, "INVALID_INPUT");
|
|
1250
|
-
return value.trim();
|
|
1251
|
-
};
|
|
1252
|
-
var numberValue = (value, label, integer = false) => {
|
|
1253
|
-
if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || integer && !Number.isInteger(value)) fail(`${label} must be a non-negative ${integer ? "integer" : "number"}.`, "INVALID_INPUT");
|
|
1254
|
-
return value;
|
|
1255
|
-
};
|
|
1256
|
-
var validateExternalCodingBenchmarkReport = (value) => {
|
|
1257
|
-
if (typeof value !== "object" || value === null || Array.isArray(value)) fail("coding benchmark report must be an object.", "INVALID_INPUT");
|
|
1258
|
-
const raw = value;
|
|
1259
|
-
if (!["edit", "fix-bug", "add-feature", "refactor", "add-test", "review-pr", "free-form"].includes(text2(raw["kind"], "report.kind"))) fail("report.kind is not a supported coding task kind.", "INVALID_INPUT");
|
|
1260
|
-
if (typeof raw["dryRun"] !== "boolean" || typeof raw["isolateWorktrees"] !== "boolean") fail("report.dryRun and report.isolateWorktrees must be booleans.", "INVALID_INPUT");
|
|
1261
|
-
if (!Array.isArray(raw["rows"]) || raw["rows"].length === 0) fail("report.rows must contain at least one provider result.", "INVALID_INPUT");
|
|
1262
|
-
const rows = raw["rows"].map((item, index2) => {
|
|
1263
|
-
if (typeof item !== "object" || item === null || Array.isArray(item)) fail(`report.rows[${index2}] must be an object.`, "INVALID_INPUT");
|
|
1264
|
-
const row = item;
|
|
1265
|
-
const status = text2(row["status"], `report.rows[${index2}].status`);
|
|
1266
|
-
if (!["ok", "partial", "fail", "timeout"].includes(status)) fail(`report.rows[${index2}].status is invalid.`, "INVALID_INPUT");
|
|
1267
|
-
const completenessScore = numberValue(row["completenessScore"], `report.rows[${index2}].completenessScore`);
|
|
1268
|
-
if (completenessScore > 100) fail(`report.rows[${index2}].completenessScore must be between 0 and 100.`, "INVALID_INPUT");
|
|
1269
|
-
const optional = (key) => row[key] === void 0 ? void 0 : numberValue(row[key], `report.rows[${index2}].${key}`);
|
|
1270
|
-
return {
|
|
1271
|
-
providerId: text2(row["providerId"], `report.rows[${index2}].providerId`),
|
|
1272
|
-
status,
|
|
1273
|
-
completenessScore,
|
|
1274
|
-
fileEditCount: numberValue(row["fileEditCount"], `report.rows[${index2}].fileEditCount`, true),
|
|
1275
|
-
summary: text2(row["summary"], `report.rows[${index2}].summary`),
|
|
1276
|
-
...optional("durationMs") === void 0 ? {} : { durationMs: optional("durationMs") },
|
|
1277
|
-
...optional("inputTokens") === void 0 ? {} : { inputTokens: optional("inputTokens") },
|
|
1278
|
-
...optional("outputTokens") === void 0 ? {} : { outputTokens: optional("outputTokens") },
|
|
1279
|
-
...optional("costUsd") === void 0 ? {} : { costUsd: optional("costUsd") },
|
|
1280
|
-
...row["successPassed"] === void 0 ? {} : typeof row["successPassed"] !== "boolean" ? fail(`report.rows[${index2}].successPassed must be a boolean.`, "INVALID_INPUT") : { successPassed: row["successPassed"] }
|
|
1281
|
-
};
|
|
1282
|
-
});
|
|
1283
|
-
if (new Set(rows.map((row) => row.providerId)).size !== rows.length) fail("coding benchmark provider ids must be unique.", "INVALID_INPUT");
|
|
1284
|
-
return { kind: text2(raw["kind"], "report.kind"), prompt: text2(raw["prompt"], "report.prompt"), dryRun: raw["dryRun"], isolateWorktrees: raw["isolateWorktrees"], repoRoot: text2(raw["repoRoot"], "report.repoRoot"), rows };
|
|
1783
|
+
const reportComparisons = manifest ? comparisons(runs, manifest) : [];
|
|
1784
|
+
return { type: "agentskit-harness-benchmark", schemaVersion: BENCHMARK_SCHEMA_VERSION, stateDir, generatedAt: (/* @__PURE__ */ new Date()).toISOString(), runs, summary: summarize(runs), comparisons: reportComparisons, ...manifest ? { manifest: { suiteId: manifest.suiteId, taskCount: manifest.tasks.length, baselineCount: manifest.observations.length, comparableTaskCount: reportComparisons.filter((comparison) => comparison.comparable).length } } : {} };
|
|
1285
1785
|
};
|
|
1286
|
-
var
|
|
1786
|
+
var required7 = (value, label) => {
|
|
1287
1787
|
if (typeof value !== "string" || !value.trim()) fail(`${label} is required.`, "INVALID_INPUT");
|
|
1288
1788
|
return value.trim();
|
|
1289
1789
|
};
|
|
@@ -1293,9 +1793,9 @@ var duration = (value) => {
|
|
|
1293
1793
|
};
|
|
1294
1794
|
var createSessionRecorder = ({ stateDir, run, adapter, policy, runtime, sessionId = randomUUID(), resume = false }) => {
|
|
1295
1795
|
if (run.state !== "IMPLEMENTING") fail(`Agent sessions can only start during IMPLEMENTING, not ${run.state}.`, "INVALID_STATE");
|
|
1296
|
-
const id2 =
|
|
1297
|
-
const adapterId =
|
|
1298
|
-
const adapterVersion =
|
|
1796
|
+
const id2 = required7(sessionId, "sessionId");
|
|
1797
|
+
const adapterId = required7(adapter.id, "adapter.id");
|
|
1798
|
+
const adapterVersion = required7(adapter.version, "adapter.version");
|
|
1299
1799
|
if (!policy || typeof policy.evaluate !== "function") fail("policy.evaluate is required.", "INVALID_INPUT");
|
|
1300
1800
|
if (!runtime || typeof runtime.execute !== "function") fail("runtime.execute is required.", "INVALID_INPUT");
|
|
1301
1801
|
if (!Array.isArray(adapter.capabilities) || adapter.capabilities.some((capability) => typeof capability !== "string" || !capability.trim())) fail("adapter.capabilities must contain non-empty strings.", "INVALID_INPUT");
|
|
@@ -1346,18 +1846,18 @@ var createSessionRecorder = ({ stateDir, run, adapter, policy, runtime, sessionI
|
|
|
1346
1846
|
};
|
|
1347
1847
|
const complete = (input) => {
|
|
1348
1848
|
open();
|
|
1349
|
-
const actionId =
|
|
1849
|
+
const actionId = required7(input.actionId, "actionId");
|
|
1350
1850
|
if (!pending.has(actionId)) fail(`Tool action is not pending: ${actionId}.`, "INVALID_STATE");
|
|
1351
|
-
const event = append("tool.completed", { actionId, resultHash:
|
|
1851
|
+
const event = append("tool.completed", { actionId, resultHash: required7(input.resultHash, "resultHash"), durationMs: duration(input.durationMs), ...input.runtimeEvidence ? { runtimeEvidence: input.runtimeEvidence } : {} });
|
|
1352
1852
|
pending.delete(actionId);
|
|
1353
1853
|
return event;
|
|
1354
1854
|
};
|
|
1355
1855
|
const failAction = (input) => {
|
|
1356
1856
|
open();
|
|
1357
|
-
const actionId =
|
|
1857
|
+
const actionId = required7(input.actionId, "actionId");
|
|
1358
1858
|
if (!pending.has(actionId)) fail(`Tool action is not pending: ${actionId}.`, "INVALID_STATE");
|
|
1359
1859
|
if (typeof input.retryable !== "boolean") fail("retryable must be boolean.", "INVALID_INPUT");
|
|
1360
|
-
const event = append("tool.failed", { actionId, errorCode:
|
|
1860
|
+
const event = append("tool.failed", { actionId, errorCode: required7(input.errorCode, "errorCode"), retryable: input.retryable, durationMs: duration(input.durationMs), ...input.runtimeEvidence ? { runtimeEvidence: input.runtimeEvidence } : {} });
|
|
1361
1861
|
pending.delete(actionId);
|
|
1362
1862
|
return event;
|
|
1363
1863
|
};
|
|
@@ -1365,24 +1865,24 @@ var createSessionRecorder = ({ stateDir, run, adapter, policy, runtime, sessionI
|
|
|
1365
1865
|
sessionId: id2,
|
|
1366
1866
|
startTurn: (inputHash, turnId = randomUUID()) => {
|
|
1367
1867
|
open();
|
|
1368
|
-
const turn =
|
|
1868
|
+
const turn = required7(turnId, "turnId");
|
|
1369
1869
|
if (turns.has(turn)) fail(`Turn already exists: ${turn}.`, "INVALID_STATE");
|
|
1370
|
-
const event = append("agent.turn.started", { turnId: turn, inputHash:
|
|
1870
|
+
const event = append("agent.turn.started", { turnId: turn, inputHash: required7(inputHash, "inputHash") });
|
|
1371
1871
|
turns.add(turn);
|
|
1372
1872
|
return event;
|
|
1373
1873
|
},
|
|
1374
1874
|
requestTool: (input) => {
|
|
1375
1875
|
open();
|
|
1376
|
-
const turnId =
|
|
1876
|
+
const turnId = required7(input.turnId, "turnId");
|
|
1377
1877
|
if (!turns.has(turnId)) fail(`Turn does not exist: ${turnId}.`, "INVALID_STATE");
|
|
1378
|
-
const actionId =
|
|
1878
|
+
const actionId = required7(input.actionId ?? randomUUID(), "actionId");
|
|
1379
1879
|
if (actions.has(actionId)) fail(`Tool action already exists: ${actionId}.`, "INVALID_STATE");
|
|
1380
|
-
const toolId =
|
|
1381
|
-
const argumentsHash =
|
|
1880
|
+
const toolId = required7(input.toolId, "toolId");
|
|
1881
|
+
const argumentsHash = required7(input.argumentsHash, "argumentsHash");
|
|
1382
1882
|
const decision = policy.evaluate({ actionId, turnId, toolId, argumentsHash });
|
|
1383
1883
|
if (!decision || decision.decision !== "allow" && decision.decision !== "block" && decision.decision !== "approve") fail("Policy decision is invalid.", "HARNESS_ERROR");
|
|
1384
|
-
const policyId =
|
|
1385
|
-
const reason =
|
|
1884
|
+
const policyId = required7(decision.policyId, "policyId");
|
|
1885
|
+
const reason = required7(decision.reason, "policy reason");
|
|
1386
1886
|
append("policy.evaluated", { actionId, turnId, toolId, decision: decision.decision, policyId, reason });
|
|
1387
1887
|
actions.add(actionId);
|
|
1388
1888
|
if (decision.decision === "block") {
|
|
@@ -1400,7 +1900,7 @@ var createSessionRecorder = ({ stateDir, run, adapter, policy, runtime, sessionI
|
|
|
1400
1900
|
},
|
|
1401
1901
|
approveTool: (input) => {
|
|
1402
1902
|
open();
|
|
1403
|
-
const actionId =
|
|
1903
|
+
const actionId = required7(input.actionId, "actionId");
|
|
1404
1904
|
const approval = approvals.get(actionId) ?? fail(`Tool action is not awaiting human approval: ${actionId}.`, "INVALID_STATE");
|
|
1405
1905
|
if (input.actor !== void 0 && input.actor !== "human") fail("Tool approval requires a human actor.", "HUMAN_APPROVAL_REQUIRED");
|
|
1406
1906
|
const decision = input.decision;
|
|
@@ -1414,7 +1914,7 @@ var createSessionRecorder = ({ stateDir, run, adapter, policy, runtime, sessionI
|
|
|
1414
1914
|
},
|
|
1415
1915
|
recoverTool: (input) => {
|
|
1416
1916
|
open();
|
|
1417
|
-
const actionId =
|
|
1917
|
+
const actionId = required7(input.actionId, "actionId");
|
|
1418
1918
|
const action = pending.get(actionId) ?? fail(`Tool action is not pending: ${actionId}.`, "INVALID_STATE");
|
|
1419
1919
|
if (!action.executionStarted) fail(`Tool action does not require recovery: ${actionId}.`, "INVALID_STATE");
|
|
1420
1920
|
if (input.actor !== void 0 && input.actor !== "human") fail("Tool recovery requires a human actor.", "HUMAN_APPROVAL_REQUIRED");
|
|
@@ -1432,7 +1932,7 @@ var createSessionRecorder = ({ stateDir, run, adapter, policy, runtime, sessionI
|
|
|
1432
1932
|
failTool: failAction,
|
|
1433
1933
|
executeTool: async (input) => {
|
|
1434
1934
|
open();
|
|
1435
|
-
const actionId =
|
|
1935
|
+
const actionId = required7(input.actionId, "actionId");
|
|
1436
1936
|
const action = pending.get(actionId) ?? fail(`Tool action is not pending: ${actionId}.`, "INVALID_STATE");
|
|
1437
1937
|
if (action.executionStarted) fail(`Tool action requires human recovery decision: ${actionId}.`, "HUMAN_APPROVAL_REQUIRED");
|
|
1438
1938
|
if (executing.has(actionId)) fail(`Tool action is already executing: ${actionId}.`, "INVALID_STATE");
|
|
@@ -1476,7 +1976,7 @@ var createSessionRecorder = ({ stateDir, run, adapter, policy, runtime, sessionI
|
|
|
1476
1976
|
};
|
|
1477
1977
|
|
|
1478
1978
|
// src/policy.ts
|
|
1479
|
-
var
|
|
1979
|
+
var required8 = (value, label) => {
|
|
1480
1980
|
if (typeof value !== "string" || !value.trim()) fail(`${label} is required.`, "INVALID_INPUT");
|
|
1481
1981
|
return value.trim();
|
|
1482
1982
|
};
|
|
@@ -1484,25 +1984,26 @@ var createPolicyGate = ({ rules }) => {
|
|
|
1484
1984
|
if (!Array.isArray(rules)) fail("Policy rules must be an array.", "INVALID_INPUT");
|
|
1485
1985
|
const normalized = rules.map((rule, index2) => {
|
|
1486
1986
|
if (typeof rule !== "object" || rule === null || Array.isArray(rule)) fail(`rules[${index2}] must be an object.`, "INVALID_INPUT");
|
|
1487
|
-
const id2 =
|
|
1987
|
+
const id2 = required8(rule.id, `rules[${index2}].id`);
|
|
1488
1988
|
if (rule.effect !== "allow" && rule.effect !== "block" && rule.effect !== "approve") fail(`rules[${index2}].effect is invalid.`, "INVALID_INPUT");
|
|
1489
1989
|
if (!Array.isArray(rule.toolIds) || !rule.toolIds.length || rule.toolIds.some((toolId) => typeof toolId !== "string" || !toolId.trim())) fail(`rules[${index2}].toolIds must contain non-empty strings.`, "INVALID_INPUT");
|
|
1490
|
-
return { id: id2, effect: rule.effect, toolIds: rule.toolIds.map((toolId) =>
|
|
1990
|
+
return { id: id2, effect: rule.effect, toolIds: rule.toolIds.map((toolId) => required8(toolId, `rules[${index2}].toolIds`)), reason: required8(rule.reason, `rules[${index2}].reason`) };
|
|
1491
1991
|
});
|
|
1492
1992
|
if (new Set(normalized.map((rule) => rule.id)).size !== normalized.length) fail("Policy rules must have unique ids.", "INVALID_INPUT");
|
|
1493
1993
|
return {
|
|
1494
1994
|
evaluate: (request) => {
|
|
1495
1995
|
if (typeof request !== "object" || request === null || Array.isArray(request)) fail("Policy request must be an object.", "INVALID_INPUT");
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
const toolId =
|
|
1499
|
-
|
|
1996
|
+
required8(request.actionId, "request.actionId");
|
|
1997
|
+
required8(request.turnId, "request.turnId");
|
|
1998
|
+
const toolId = required8(request.toolId, "request.toolId");
|
|
1999
|
+
required8(request.argumentsHash, "request.argumentsHash");
|
|
1500
2000
|
const rule = normalized.find((candidate) => candidate.toolIds.includes(toolId));
|
|
1501
2001
|
return rule ? { decision: rule.effect, policyId: rule.id, reason: rule.reason } : { decision: "block", policyId: "default-deny", reason: `No policy rule allows tool: ${toolId}.` };
|
|
1502
2002
|
}
|
|
1503
2003
|
};
|
|
1504
2004
|
};
|
|
1505
|
-
var
|
|
2005
|
+
var createConfiguredToolRuntime = ({ runtime, process: process2, docker }) => runtime.kind === "docker" ? createDockerToolRuntime(docker) : createProcessToolRuntime(process2);
|
|
2006
|
+
var required9 = (value, label) => {
|
|
1506
2007
|
if (typeof value !== "string" || !value.trim()) fail(`${label} is required.`, "INVALID_INPUT");
|
|
1507
2008
|
return value.trim();
|
|
1508
2009
|
};
|
|
@@ -1516,7 +2017,7 @@ var positiveNumber = (value, label) => {
|
|
|
1516
2017
|
return normalized;
|
|
1517
2018
|
};
|
|
1518
2019
|
var absolutePath = (value, label) => {
|
|
1519
|
-
const normalized =
|
|
2020
|
+
const normalized = required9(value, label);
|
|
1520
2021
|
if (!normalized.startsWith("/") || normalized.includes(",")) fail(`${label} must be an absolute path without commas.`, "INVALID_INPUT");
|
|
1521
2022
|
return normalized;
|
|
1522
2023
|
};
|
|
@@ -1534,7 +2035,7 @@ var createToolRuntime = ({ tools, timeoutMs = 3e4 }) => {
|
|
|
1534
2035
|
if (!Number.isInteger(timeoutMs) || timeoutMs < 1) fail("Runtime timeoutMs must be a positive integer.", "INVALID_INPUT");
|
|
1535
2036
|
const normalized = tools.map((tool, index2) => {
|
|
1536
2037
|
if (typeof tool !== "object" || tool === null || Array.isArray(tool)) fail(`tools[${index2}] must be an object.`, "INVALID_INPUT");
|
|
1537
|
-
const toolId =
|
|
2038
|
+
const toolId = required9(tool.toolId, `tools[${index2}].toolId`);
|
|
1538
2039
|
if (typeof tool.execute !== "function") fail(`tools[${index2}].execute is required.`, "INVALID_INPUT");
|
|
1539
2040
|
return { toolId, execute: tool.execute };
|
|
1540
2041
|
});
|
|
@@ -1542,10 +2043,10 @@ var createToolRuntime = ({ tools, timeoutMs = 3e4 }) => {
|
|
|
1542
2043
|
return {
|
|
1543
2044
|
execute: async (request) => {
|
|
1544
2045
|
const started = Date.now();
|
|
1545
|
-
const actionId =
|
|
1546
|
-
const turnId =
|
|
1547
|
-
const toolId =
|
|
1548
|
-
const argumentsHash =
|
|
2046
|
+
const actionId = required9(request.actionId, "request.actionId");
|
|
2047
|
+
const turnId = required9(request.turnId, "request.turnId");
|
|
2048
|
+
const toolId = required9(request.toolId, "request.toolId");
|
|
2049
|
+
const argumentsHash = required9(request.argumentsHash, "request.argumentsHash");
|
|
1549
2050
|
const tool = normalized.find((candidate) => candidate.toolId === toolId);
|
|
1550
2051
|
if (!tool) return { status: "failed", errorCode: "TOOL_NOT_FOUND", retryable: false, durationMs: duration2(Date.now() - started) };
|
|
1551
2052
|
const controller = new AbortController();
|
|
@@ -1575,8 +2076,8 @@ var createProcessToolRuntime = ({ tools, timeoutMs = 3e4, maxOutputBytes = 10485
|
|
|
1575
2076
|
if (!Number.isInteger(maxOutputBytes) || maxOutputBytes < 1) fail("Process runtime maxOutputBytes must be a positive integer.", "INVALID_INPUT");
|
|
1576
2077
|
const normalized = tools.map((tool, index2) => {
|
|
1577
2078
|
if (typeof tool !== "object" || tool === null || Array.isArray(tool)) fail(`tools[${index2}] must be an object.`, "INVALID_INPUT");
|
|
1578
|
-
const toolId =
|
|
1579
|
-
const command =
|
|
2079
|
+
const toolId = required9(tool.toolId, `tools[${index2}].toolId`);
|
|
2080
|
+
const command = required9(tool.command, `tools[${index2}].command`);
|
|
1580
2081
|
if (tool.args !== void 0 && (!Array.isArray(tool.args) || tool.args.some((arg) => typeof arg !== "string"))) fail(`tools[${index2}].args must contain strings.`, "INVALID_INPUT");
|
|
1581
2082
|
if (tool.env !== void 0 && (typeof tool.env !== "object" || tool.env === null || Array.isArray(tool.env) || Object.values(tool.env).some((value) => typeof value !== "string"))) fail(`tools[${index2}].env must contain string values.`, "INVALID_INPUT");
|
|
1582
2083
|
return { toolId, command, args: tool.args ? [...tool.args] : [], ...tool.cwd ? { cwd: tool.cwd } : {}, env: tool.env ? { ...tool.env } : { PATH: process.env["PATH"] ?? "" } };
|
|
@@ -1585,10 +2086,10 @@ var createProcessToolRuntime = ({ tools, timeoutMs = 3e4, maxOutputBytes = 10485
|
|
|
1585
2086
|
return {
|
|
1586
2087
|
execute: async (request) => {
|
|
1587
2088
|
const started = Date.now();
|
|
1588
|
-
const actionId =
|
|
1589
|
-
const turnId =
|
|
1590
|
-
const toolId =
|
|
1591
|
-
const argumentsHash =
|
|
2089
|
+
const actionId = required9(request.actionId, "request.actionId");
|
|
2090
|
+
const turnId = required9(request.turnId, "request.turnId");
|
|
2091
|
+
const toolId = required9(request.toolId, "request.toolId");
|
|
2092
|
+
const argumentsHash = required9(request.argumentsHash, "request.argumentsHash");
|
|
1592
2093
|
const tool = normalized.find((candidate) => candidate.toolId === toolId);
|
|
1593
2094
|
if (!tool) return { status: "failed", errorCode: "TOOL_NOT_FOUND", retryable: false, durationMs: Date.now() - started };
|
|
1594
2095
|
let input;
|
|
@@ -1598,7 +2099,7 @@ var createProcessToolRuntime = ({ tools, timeoutMs = 3e4, maxOutputBytes = 10485
|
|
|
1598
2099
|
return { status: "failed", errorCode: "SERIALIZATION_ERROR", retryable: false, durationMs: Date.now() - started };
|
|
1599
2100
|
}
|
|
1600
2101
|
return new Promise((resolve6) => {
|
|
1601
|
-
const child = spawn(tool.command, tool.args, { cwd: tool.cwd, env: tool.env, shell: false, stdio: ["pipe", "pipe", "pipe"] });
|
|
2102
|
+
const child = spawn(tool.command, [...tool.args], { cwd: tool.cwd, env: tool.env, shell: false, stdio: ["pipe", "pipe", "pipe"] });
|
|
1602
2103
|
let stdout = "";
|
|
1603
2104
|
let timedOut = false;
|
|
1604
2105
|
let outputLimit = false;
|
|
@@ -1658,17 +2159,17 @@ var createDockerToolRuntime = ({
|
|
|
1658
2159
|
pull = "never"
|
|
1659
2160
|
}) => {
|
|
1660
2161
|
if (!Array.isArray(tools)) fail("Docker runtime tools must be an array.", "INVALID_INPUT");
|
|
1661
|
-
const command =
|
|
1662
|
-
const memory =
|
|
2162
|
+
const command = required9(dockerCommand, "dockerCommand");
|
|
2163
|
+
const memory = required9(memoryLimit, "memoryLimit");
|
|
1663
2164
|
const cpu = positiveNumber(cpus, "cpus");
|
|
1664
2165
|
if (!Number.isInteger(pidsLimit) || pidsLimit < 1) fail("pidsLimit must be a positive integer.", "INVALID_INPUT");
|
|
1665
|
-
const normalizedUser =
|
|
2166
|
+
const normalizedUser = required9(user, "user");
|
|
1666
2167
|
if (normalizedUser.includes(" ")) fail("user must not contain spaces.", "INVALID_INPUT");
|
|
1667
2168
|
if (pull !== "never" && pull !== "missing" && pull !== "always") fail("pull must be never, missing, or always.", "INVALID_INPUT");
|
|
1668
2169
|
const normalized = tools.map((tool, index2) => {
|
|
1669
2170
|
if (typeof tool !== "object" || tool === null || Array.isArray(tool)) fail(`tools[${index2}] must be an object.`, "INVALID_INPUT");
|
|
1670
|
-
const toolId =
|
|
1671
|
-
const image =
|
|
2171
|
+
const toolId = required9(tool.toolId, `tools[${index2}].toolId`);
|
|
2172
|
+
const image = required9(tool.image, `tools[${index2}].image`);
|
|
1672
2173
|
if (!Array.isArray(tool.command) || tool.command.length === 0 || tool.command.some((part) => typeof part !== "string" || !part.trim())) fail(`tools[${index2}].command must be a non-empty string array.`, "INVALID_INPUT");
|
|
1673
2174
|
if (tool.args !== void 0 && (!Array.isArray(tool.args) || tool.args.some((arg) => typeof arg !== "string"))) fail(`tools[${index2}].args must contain strings.`, "INVALID_INPUT");
|
|
1674
2175
|
const env = dockerEnvironment(tool.env, `tools[${index2}].env`);
|
|
@@ -1738,6 +2239,381 @@ var createDockerToolRuntime = ({
|
|
|
1738
2239
|
}
|
|
1739
2240
|
};
|
|
1740
2241
|
};
|
|
2242
|
+
var required10 = (value, label) => {
|
|
2243
|
+
if (typeof value !== "string" || !value.trim()) fail(`${label} is required.`, "INVALID_INPUT");
|
|
2244
|
+
return value.trim();
|
|
2245
|
+
};
|
|
2246
|
+
var safeKey = (identity) => hashJson(identity);
|
|
2247
|
+
var now3 = () => (/* @__PURE__ */ new Date()).toISOString();
|
|
2248
|
+
var parse = (value, label) => {
|
|
2249
|
+
try {
|
|
2250
|
+
const raw = JSON.parse(value);
|
|
2251
|
+
const identity = {
|
|
2252
|
+
tracker: required10(raw["tracker"], `${label}.tracker`),
|
|
2253
|
+
repository: required10(raw["repository"], `${label}.repository`),
|
|
2254
|
+
issue: required10(raw["issue"], `${label}.issue`),
|
|
2255
|
+
worktree: required10(raw["worktree"], `${label}.worktree`),
|
|
2256
|
+
branch: required10(raw["branch"], `${label}.branch`)
|
|
2257
|
+
};
|
|
2258
|
+
return { ...identity, key: required10(raw["key"], `${label}.key`), leaseId: required10(raw["leaseId"], `${label}.leaseId`), owner: required10(raw["owner"], `${label}.owner`), claimedAt: required10(raw["claimedAt"], `${label}.claimedAt`) };
|
|
2259
|
+
} catch (error) {
|
|
2260
|
+
if (error instanceof SyntaxError) fail(`${label} contains invalid JSON.`, "HARNESS_ERROR");
|
|
2261
|
+
throw error;
|
|
2262
|
+
}
|
|
2263
|
+
};
|
|
2264
|
+
var createDispatchLedger = (stateDir) => {
|
|
2265
|
+
const root = required10(stateDir, "stateDir");
|
|
2266
|
+
const claimsDir = join(root, "coordination", "claims");
|
|
2267
|
+
const ledgerPath = join(root, "coordination", "dispatch-ledger.ndjson");
|
|
2268
|
+
mkdirSync(claimsDir, { recursive: true });
|
|
2269
|
+
const claimPath = (key) => join(claimsDir, `${key}.json`);
|
|
2270
|
+
const append = (record3) => appendFileSync(ledgerPath, `${JSON.stringify(record3)}
|
|
2271
|
+
`, "utf8");
|
|
2272
|
+
const records = () => {
|
|
2273
|
+
if (!existsSync(ledgerPath)) return [];
|
|
2274
|
+
return readFileSync(ledgerPath, "utf8").split(/\r?\n/).map((line) => line.trim()).filter(Boolean).map((line, index2) => {
|
|
2275
|
+
try {
|
|
2276
|
+
return JSON.parse(line);
|
|
2277
|
+
} catch {
|
|
2278
|
+
return fail(`Dispatch ledger record ${index2 + 1} is invalid JSON.`, "HARNESS_ERROR");
|
|
2279
|
+
}
|
|
2280
|
+
});
|
|
2281
|
+
};
|
|
2282
|
+
const active = () => {
|
|
2283
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
2284
|
+
for (const record3 of records()) {
|
|
2285
|
+
if (record3.action === "release" || record3.action === "recover") byKey.delete(record3.key);
|
|
2286
|
+
else if (record3.action === "dispatch") byKey.set(record3.key, record3);
|
|
2287
|
+
}
|
|
2288
|
+
return [...byKey.values()];
|
|
2289
|
+
};
|
|
2290
|
+
return {
|
|
2291
|
+
claim: (input) => {
|
|
2292
|
+
const identity = {
|
|
2293
|
+
tracker: required10(input.tracker, "tracker"),
|
|
2294
|
+
repository: required10(input.repository, "repository"),
|
|
2295
|
+
issue: required10(input.issue, "issue"),
|
|
2296
|
+
worktree: required10(input.worktree, "worktree"),
|
|
2297
|
+
branch: required10(input.branch, "branch")
|
|
2298
|
+
};
|
|
2299
|
+
const owner = required10(input.owner, "owner");
|
|
2300
|
+
const key = safeKey(identity);
|
|
2301
|
+
const path = claimPath(key);
|
|
2302
|
+
if (existsSync(path)) return { decision: "already-claimed", lease: parse(readFileSync(path, "utf8"), "claim") };
|
|
2303
|
+
const lease = { ...identity, key, leaseId: randomUUID(), owner, claimedAt: now3() };
|
|
2304
|
+
let fd;
|
|
2305
|
+
try {
|
|
2306
|
+
fd = openSync(path, "wx");
|
|
2307
|
+
} catch (error) {
|
|
2308
|
+
if (error.code === "EEXIST") return { decision: "already-claimed", lease: parse(readFileSync(path, "utf8"), "claim") };
|
|
2309
|
+
throw error;
|
|
2310
|
+
}
|
|
2311
|
+
try {
|
|
2312
|
+
writeFileSync(fd, JSON.stringify(lease), "utf8");
|
|
2313
|
+
} finally {
|
|
2314
|
+
closeSync(fd);
|
|
2315
|
+
}
|
|
2316
|
+
append({ ...lease, action: "dispatch", at: lease.claimedAt });
|
|
2317
|
+
return { decision: "claimed", lease };
|
|
2318
|
+
},
|
|
2319
|
+
recordDispatch: ({ lease, idempotencyKey, commandDigest }) => {
|
|
2320
|
+
const id2 = required10(idempotencyKey, "idempotencyKey");
|
|
2321
|
+
const digest3 = required10(commandDigest, "commandDigest");
|
|
2322
|
+
const existing = records().find((record4) => record4.action === "dispatch" && record4.idempotencyKey === id2);
|
|
2323
|
+
if (existing) return { decision: "duplicate", record: existing };
|
|
2324
|
+
const record3 = { ...lease, action: "dispatch", at: now3(), idempotencyKey: id2, commandDigest: digest3 };
|
|
2325
|
+
append(record3);
|
|
2326
|
+
return { decision: "recorded", record: record3 };
|
|
2327
|
+
},
|
|
2328
|
+
release: (lease, reason = "lease released") => {
|
|
2329
|
+
const path = claimPath(required10(lease.key, "lease.key"));
|
|
2330
|
+
if (!existsSync(path)) fail("Dispatch lease is not active.", "INVALID_STATE");
|
|
2331
|
+
const current = parse(readFileSync(path, "utf8"), "claim");
|
|
2332
|
+
if (current.leaseId !== lease.leaseId) fail("Dispatch lease owner does not match.", "INVALID_STATE");
|
|
2333
|
+
unlinkSync(path);
|
|
2334
|
+
const record3 = { ...current, action: "release", at: now3(), reason: required10(reason, "reason") };
|
|
2335
|
+
append(record3);
|
|
2336
|
+
return record3;
|
|
2337
|
+
},
|
|
2338
|
+
recover: (key, input) => {
|
|
2339
|
+
if (input.actor !== "human") fail("Dispatch lease recovery requires a human actor.", "HUMAN_APPROVAL_REQUIRED");
|
|
2340
|
+
const normalizedKey = required10(key, "key");
|
|
2341
|
+
const maxAgeMs = input.maxAgeMs ?? 3e5;
|
|
2342
|
+
if (!Number.isInteger(maxAgeMs) || maxAgeMs < 0) fail("maxAgeMs must be a non-negative integer.", "INVALID_INPUT");
|
|
2343
|
+
const path = claimPath(normalizedKey);
|
|
2344
|
+
if (!existsSync(path)) fail("Dispatch lease is not active.", "INVALID_STATE");
|
|
2345
|
+
const current = parse(readFileSync(path, "utf8"), "claim");
|
|
2346
|
+
if (Date.now() - Date.parse(current.claimedAt) < maxAgeMs) fail("Dispatch lease is not old enough to recover.", "HARNESS_ERROR");
|
|
2347
|
+
unlinkSync(path);
|
|
2348
|
+
const record3 = { ...current, action: "recover", at: now3(), reason: required10(input.reason, "reason") };
|
|
2349
|
+
append(record3);
|
|
2350
|
+
return record3;
|
|
2351
|
+
},
|
|
2352
|
+
active,
|
|
2353
|
+
records
|
|
2354
|
+
};
|
|
2355
|
+
};
|
|
2356
|
+
|
|
2357
|
+
// src/resilience.ts
|
|
2358
|
+
var positiveInteger = (value, label) => {
|
|
2359
|
+
if (!Number.isInteger(value) || value < 1) fail(`${label} must be a positive integer.`, "INVALID_INPUT");
|
|
2360
|
+
return value;
|
|
2361
|
+
};
|
|
2362
|
+
var nonNegativeInteger3 = (value, label) => {
|
|
2363
|
+
if (!Number.isInteger(value) || value < 0) fail(`${label} must be a non-negative integer.`, "INVALID_INPUT");
|
|
2364
|
+
return value;
|
|
2365
|
+
};
|
|
2366
|
+
var classifyFailure = (error) => {
|
|
2367
|
+
const value = error;
|
|
2368
|
+
const code = typeof value?.code === "string" ? value.code.toUpperCase() : "";
|
|
2369
|
+
const message = typeof value?.message === "string" ? value.message : String(error);
|
|
2370
|
+
const text5 = `${code} ${message}`.toLowerCase();
|
|
2371
|
+
if (/quota|rate.?limit|too many requests|429/.test(text5)) return { class: "quota", retryable: true, reason: message };
|
|
2372
|
+
if (/timeout|timed out|deadline/.test(text5)) return { class: "timeout", retryable: true, reason: message };
|
|
2373
|
+
if (/policy|forbidden|permission|approval/.test(text5)) return { class: "policy", retryable: false, reason: message };
|
|
2374
|
+
if (/invalid|schema|argument|config|validation/.test(text5)) return { class: "validation", retryable: false, reason: message };
|
|
2375
|
+
if (/network|connection|econn|503|502|external/.test(text5)) return { class: "external", retryable: true, reason: message };
|
|
2376
|
+
return { class: "unknown", retryable: false, reason: message };
|
|
2377
|
+
};
|
|
2378
|
+
var recoveryDelayMs = (attempt, policy) => {
|
|
2379
|
+
positiveInteger(attempt, "attempt");
|
|
2380
|
+
nonNegativeInteger3(policy.baseDelayMs, "baseDelayMs");
|
|
2381
|
+
nonNegativeInteger3(policy.maxDelayMs, "maxDelayMs");
|
|
2382
|
+
if (policy.maxDelayMs < policy.baseDelayMs) fail("maxDelayMs must be greater than or equal to baseDelayMs.", "INVALID_INPUT");
|
|
2383
|
+
return Math.min(policy.maxDelayMs, policy.baseDelayMs * 2 ** Math.max(0, attempt - 1));
|
|
2384
|
+
};
|
|
2385
|
+
var wait = (delayMs, sleep) => delayMs > 0 ? sleep(delayMs) : Promise.resolve();
|
|
2386
|
+
var runWithRecovery = async (operation, options) => {
|
|
2387
|
+
const maxAttempts = positiveInteger(options.maxAttempts, "maxAttempts");
|
|
2388
|
+
const baseDelayMs = nonNegativeInteger3(options.baseDelayMs, "baseDelayMs");
|
|
2389
|
+
const maxDelayMs = nonNegativeInteger3(options.maxDelayMs, "maxDelayMs");
|
|
2390
|
+
if (maxDelayMs < baseDelayMs) fail("maxDelayMs must be greater than or equal to baseDelayMs.", "INVALID_INPUT");
|
|
2391
|
+
if (options.timeoutMs !== void 0) positiveInteger(options.timeoutMs, "timeoutMs");
|
|
2392
|
+
const sleep = options.sleep ?? ((delayMs) => new Promise((resolve6) => setTimeout(resolve6, delayMs)));
|
|
2393
|
+
const observations = [];
|
|
2394
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
2395
|
+
const controller = new AbortController();
|
|
2396
|
+
let timer;
|
|
2397
|
+
try {
|
|
2398
|
+
const operationPromise = operation(controller.signal, attempt);
|
|
2399
|
+
const value = options.timeoutMs === void 0 ? await operationPromise : await Promise.race([
|
|
2400
|
+
operationPromise,
|
|
2401
|
+
new Promise((_, reject) => {
|
|
2402
|
+
timer = setTimeout(() => {
|
|
2403
|
+
controller.abort();
|
|
2404
|
+
reject(new Error("operation timed out"));
|
|
2405
|
+
}, options.timeoutMs);
|
|
2406
|
+
})
|
|
2407
|
+
]);
|
|
2408
|
+
return { status: "completed", attempts: attempt, observations, value };
|
|
2409
|
+
} catch (error) {
|
|
2410
|
+
const failure = classifyFailure(error);
|
|
2411
|
+
const delayMs = failure.retryable && attempt < maxAttempts ? recoveryDelayMs(attempt, { baseDelayMs, maxDelayMs }) : 0;
|
|
2412
|
+
const observation = { attempt, failure, delayMs };
|
|
2413
|
+
observations.push(observation);
|
|
2414
|
+
options.onObservation?.(observation);
|
|
2415
|
+
if (!failure.retryable || attempt >= maxAttempts) return { status: "failed", attempts: attempt, observations, failure };
|
|
2416
|
+
await wait(delayMs, sleep);
|
|
2417
|
+
} finally {
|
|
2418
|
+
if (timer) clearTimeout(timer);
|
|
2419
|
+
}
|
|
2420
|
+
}
|
|
2421
|
+
return fail("Recovery loop exhausted unexpectedly.", "HARNESS_ERROR");
|
|
2422
|
+
};
|
|
2423
|
+
var DOC_EXTENSIONS = /* @__PURE__ */ new Set([".md", ".mdx", ".txt", ".adoc", ".rst"]);
|
|
2424
|
+
var TEST_SUFFIXES = [".test.", ".spec.", "__tests__"];
|
|
2425
|
+
var SHELL_META = /[;&|`$()<>\n\r]/;
|
|
2426
|
+
var normalizedPath = (value, label) => {
|
|
2427
|
+
if (typeof value !== "string" || !value.trim()) fail(`${label} must be a non-empty path.`, "INVALID_INPUT");
|
|
2428
|
+
const path = value.trim().replaceAll("\\", "/");
|
|
2429
|
+
if (path.startsWith("/") || path.split("/").includes("..")) fail(`${label} must be repository-relative.`, "INVALID_INPUT");
|
|
2430
|
+
return path;
|
|
2431
|
+
};
|
|
2432
|
+
var validateSafeCommand = (command) => {
|
|
2433
|
+
if (typeof command !== "string" || !command.trim()) fail("command must be a non-empty string.", "INVALID_INPUT");
|
|
2434
|
+
const value = command.trim();
|
|
2435
|
+
if (SHELL_META.test(value)) fail("command contains shell metacharacters; use argv-based execution.", "POLICY_BLOCKED");
|
|
2436
|
+
return { valid: true, command: value };
|
|
2437
|
+
};
|
|
2438
|
+
var isTest = (path) => TEST_SUFFIXES.some((suffix) => path.includes(suffix)) || /(^|\/)(test|tests|__tests__)\//.test(path);
|
|
2439
|
+
var isDoc = (path) => DOC_EXTENSIONS.has(extname(path).toLowerCase());
|
|
2440
|
+
var planFilePreflight = (files, options = {}) => {
|
|
2441
|
+
if (!Array.isArray(files)) fail("files must be an array.", "INVALID_INPUT");
|
|
2442
|
+
const unique2 = [...new Set(files.map((file, index2) => normalizedPath(file.path, `files[${index2}].path`)))].sort();
|
|
2443
|
+
const codeFiles = unique2.filter((path) => !isDoc(path) && !isTest(path));
|
|
2444
|
+
const existingTests = unique2.filter(isTest);
|
|
2445
|
+
const roots = (options.testRoots ?? ["test", "tests", "__tests__"]).map((root, index2) => normalizedPath(root, `testRoots[${index2}]`));
|
|
2446
|
+
const colocated = options.includeTests === false ? [] : codeFiles.flatMap((path) => {
|
|
2447
|
+
const file = basename(path);
|
|
2448
|
+
const directory = dirname(path);
|
|
2449
|
+
const stem = file.includes(".") ? file.slice(0, file.lastIndexOf(".")) : file;
|
|
2450
|
+
return [join(directory, `${stem}.test.ts`), join(directory, `${stem}.spec.ts`)].filter((candidate) => unique2.includes(candidate));
|
|
2451
|
+
});
|
|
2452
|
+
const testFiles = [...new Set([...existingTests, ...colocated, ...unique2.filter((path) => roots.some((root) => path === root || path.startsWith(`${root}/`)))].sort())];
|
|
2453
|
+
const docsOnly = unique2.length > 0 && codeFiles.length === 0 && existingTests.length === 0;
|
|
2454
|
+
return { files: unique2, codeFiles, testFiles, docsOnly, checks: docsOnly ? [] : ["lint", "typecheck", ...testFiles.length ? ["test"] : []] };
|
|
2455
|
+
};
|
|
2456
|
+
|
|
2457
|
+
// src/block.ts
|
|
2458
|
+
var BLOCK_STATUSES = ["todo", "picked", "development", "validation", "pr-open", "merged", "post-merge", "done", "blocked", "scope-cut"];
|
|
2459
|
+
var text3 = (value, label) => {
|
|
2460
|
+
return typeof value === "string" && value.trim() ? value.trim() : fail(`${label} must be a non-empty string.`, "INVALID_INPUT");
|
|
2461
|
+
};
|
|
2462
|
+
var list = (value, label) => {
|
|
2463
|
+
if (!Array.isArray(value)) return fail(`${label} must be an array of non-empty strings.`, "INVALID_INPUT");
|
|
2464
|
+
if (!value.every((item) => typeof item === "string" && Boolean(item.trim()))) return fail(`${label} must be an array of non-empty strings.`, "INVALID_INPUT");
|
|
2465
|
+
return [...new Set(value.map((item) => item.trim()))];
|
|
2466
|
+
};
|
|
2467
|
+
var validateBlockManifest = (value) => {
|
|
2468
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) fail("block manifest must be an object.", "INVALID_INPUT");
|
|
2469
|
+
const raw = value;
|
|
2470
|
+
if (raw["schemaVersion"] !== 1) fail("block manifest schemaVersion must be 1.", "INVALID_INPUT");
|
|
2471
|
+
const criteria = list(raw["acceptanceCriteria"], "acceptanceCriteria");
|
|
2472
|
+
if (!criteria.length) fail("acceptanceCriteria must not be empty.", "INVALID_INPUT");
|
|
2473
|
+
const dependencies = list(raw["dependencies"] ?? [], "dependencies");
|
|
2474
|
+
const wave = raw["wave"];
|
|
2475
|
+
if (!Number.isInteger(wave) || wave < 1) fail("wave must be a positive integer.", "INVALID_INPUT");
|
|
2476
|
+
const status = raw["status"];
|
|
2477
|
+
if (!BLOCK_STATUSES.includes(status)) fail("status is invalid.", "INVALID_INPUT");
|
|
2478
|
+
const budgetRaw = raw["budget"];
|
|
2479
|
+
let budget;
|
|
2480
|
+
if (budgetRaw !== void 0) {
|
|
2481
|
+
if (typeof budgetRaw !== "object" || budgetRaw === null || Array.isArray(budgetRaw)) fail("budget must be an object.", "INVALID_INPUT");
|
|
2482
|
+
const candidate = budgetRaw;
|
|
2483
|
+
for (const key of ["maxMinutes", "maxAttempts"]) if (candidate[key] !== void 0 && (!Number.isInteger(candidate[key]) || candidate[key] < 1)) fail(`budget.${key} must be a positive integer.`, "INVALID_INPUT");
|
|
2484
|
+
budget = { ...candidate["maxMinutes"] === void 0 ? {} : { maxMinutes: candidate["maxMinutes"] }, ...candidate["maxAttempts"] === void 0 ? {} : { maxAttempts: candidate["maxAttempts"] } };
|
|
2485
|
+
}
|
|
2486
|
+
return { schemaVersion: 1, id: text3(raw["id"], "id"), title: text3(raw["title"], "title"), tracker: text3(raw["tracker"], "tracker"), repository: text3(raw["repository"], "repository"), acceptanceCriteria: criteria, dependencies, wave, status, ...budget ? { budget } : {}, ...raw["humanGates"] === void 0 ? {} : { humanGates: list(raw["humanGates"], "humanGates") }, ...raw["sourceHash"] === void 0 ? {} : { sourceHash: text3(raw["sourceHash"], "sourceHash") } };
|
|
2487
|
+
};
|
|
2488
|
+
var assessBlock = (manifest, completedDependencies = []) => {
|
|
2489
|
+
const value = validateBlockManifest(manifest);
|
|
2490
|
+
const completed = new Set(completedDependencies.map((item) => text3(item, "completedDependencies[]")));
|
|
2491
|
+
const blockers = value.dependencies.filter((dependency) => !completed.has(dependency));
|
|
2492
|
+
const next = blockers.length ? [`Complete dependencies: ${blockers.join(", ")}`] : value.status === "blocked" ? ["Resolve the recorded blocker before dispatch."] : ["Dispatch the block with the frozen acceptance criteria."];
|
|
2493
|
+
return { status: blockers.length || value.status === "blocked" ? "blocked" : "ready", manifestHash: hashJson(value), blockers, next };
|
|
2494
|
+
};
|
|
2495
|
+
var LEARNING_STATUSES = ["proposed", "promoted", "rejected"];
|
|
2496
|
+
var text4 = (value, label) => {
|
|
2497
|
+
return typeof value === "string" && value.trim() ? value.trim() : fail(`${label} must be a non-empty string.`, "INVALID_INPUT");
|
|
2498
|
+
};
|
|
2499
|
+
var category = (heading) => {
|
|
2500
|
+
const value = heading.toLowerCase();
|
|
2501
|
+
if (/went well|success|worked/.test(value)) return "worked";
|
|
2502
|
+
if (/problem|failed|blocker|pain/.test(value)) return "problem";
|
|
2503
|
+
if (/adjust|action|next|improv/.test(value)) return "adjustment";
|
|
2504
|
+
return "other";
|
|
2505
|
+
};
|
|
2506
|
+
var parseRetro = (markdown, source, recordedAt = (/* @__PURE__ */ new Date()).toISOString()) => {
|
|
2507
|
+
const input = text4(markdown, "markdown");
|
|
2508
|
+
const origin = text4(source, "source");
|
|
2509
|
+
if (!Number.isFinite(Date.parse(recordedAt))) fail("recordedAt must be a valid timestamp.", "INVALID_INPUT");
|
|
2510
|
+
const records = [];
|
|
2511
|
+
let current = "other";
|
|
2512
|
+
for (const line of input.split(/\r?\n/)) {
|
|
2513
|
+
const heading = line.match(/^#{1,6}\s+(.+)$/);
|
|
2514
|
+
if (heading) {
|
|
2515
|
+
current = category(heading[1] ?? "");
|
|
2516
|
+
continue;
|
|
2517
|
+
}
|
|
2518
|
+
const item = line.match(/^\s*[-*]\s+(?:\[[ xX]\]\s+)?(.+?)\s*$/);
|
|
2519
|
+
if (!item?.[1]?.trim()) continue;
|
|
2520
|
+
const value = item[1].trim();
|
|
2521
|
+
const id2 = `L-${createHash("sha256").update(`${origin}|${current}|${value}`).digest("hex").slice(0, 12)}`;
|
|
2522
|
+
if (!records.some((record3) => record3.id === id2)) records.push({ id: id2, source: origin, category: current, text: value, status: "proposed", recordedAt });
|
|
2523
|
+
}
|
|
2524
|
+
return records;
|
|
2525
|
+
};
|
|
2526
|
+
var promoteLearnings = (records, input) => {
|
|
2527
|
+
if (input.actor !== "human") fail("Learning promotion requires a human actor.", "HUMAN_APPROVAL_REQUIRED");
|
|
2528
|
+
const ids = new Set(input.ids.map((id2) => text4(id2, "ids[]")));
|
|
2529
|
+
const status = input.status ?? "promoted";
|
|
2530
|
+
const result = records.map((record3) => ids.has(record3.id) ? { ...record3, status } : record3);
|
|
2531
|
+
const unknown = [...ids].filter((id2) => !records.some((record3) => record3.id === id2));
|
|
2532
|
+
if (unknown.length) fail(`Unknown learning IDs: ${unknown.join(", ")}`, "INVALID_INPUT");
|
|
2533
|
+
return result;
|
|
2534
|
+
};
|
|
2535
|
+
|
|
2536
|
+
// src/status.ts
|
|
2537
|
+
var required11 = (value, label) => {
|
|
2538
|
+
return typeof value === "string" && value.trim() ? value.trim() : fail(`${label} must be a non-empty string.`, "INVALID_INPUT");
|
|
2539
|
+
};
|
|
2540
|
+
var createStatusSnapshot = (input) => {
|
|
2541
|
+
const sourceRevision = required11(input.sourceRevision, "sourceRevision");
|
|
2542
|
+
if (!Number.isFinite(Date.parse(input.generatedAt))) fail("generatedAt must be a valid timestamp.", "INVALID_INPUT");
|
|
2543
|
+
if (!Array.isArray(input.blocks)) fail("blocks must be an array.", "INVALID_INPUT");
|
|
2544
|
+
const blocks = input.blocks.map((block, index2) => {
|
|
2545
|
+
if (typeof block !== "object" || block === null || Array.isArray(block)) fail(`blocks[${index2}] must be an object.`, "INVALID_INPUT");
|
|
2546
|
+
const value = block;
|
|
2547
|
+
if (!(typeof value.id === "string" && value.id.trim())) fail(`blocks[${index2}].id is required.`, "INVALID_INPUT");
|
|
2548
|
+
if (!["todo", "picked", "development", "validation", "pr-open", "merged", "post-merge", "done", "blocked", "scope-cut"].includes(value.status)) fail(`blocks[${index2}].status is invalid.`, "INVALID_INPUT");
|
|
2549
|
+
return { ...value, id: value.id.trim() };
|
|
2550
|
+
}).sort((left, right) => left.id.localeCompare(right.id));
|
|
2551
|
+
if (input.metrics !== void 0 && Object.entries(input.metrics).some(([key, value]) => !key.trim() || typeof value !== "number" || !Number.isFinite(value) || value < 0)) fail("metrics must contain finite non-negative numbers.", "INVALID_INPUT");
|
|
2552
|
+
const body2 = { schemaVersion: 1, generatedAt: input.generatedAt, sourceRevision, blocks, ...input.machine ? { machine: input.machine } : {}, ...input.metrics ? { metrics: input.metrics } : {}, ...input.next ? { next: required11(input.next, "next") } : {} };
|
|
2553
|
+
return { ...body2, digest: hashJson(body2) };
|
|
2554
|
+
};
|
|
2555
|
+
var validateStatusSnapshot = (value) => {
|
|
2556
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) fail("status snapshot must be an object.", "INVALID_INPUT");
|
|
2557
|
+
const raw = value;
|
|
2558
|
+
const snapshot = createStatusSnapshot({ generatedAt: required11(raw.generatedAt, "generatedAt"), sourceRevision: required11(raw.sourceRevision, "sourceRevision"), blocks: raw.blocks, ...raw.machine ? { machine: raw.machine } : {}, ...raw.metrics ? { metrics: raw.metrics } : {}, ...raw.next ? { next: raw.next } : {} });
|
|
2559
|
+
if (raw.schemaVersion !== 1 || raw.digest !== snapshot.digest) fail("status snapshot digest or schemaVersion is invalid.", "HARNESS_ERROR");
|
|
2560
|
+
return snapshot;
|
|
2561
|
+
};
|
|
2562
|
+
|
|
2563
|
+
// src/model-policy.ts
|
|
2564
|
+
var MODEL_ROLES = ["orchestrator", "reviewer", "builder", "watcher"];
|
|
2565
|
+
var required12 = (value, label) => {
|
|
2566
|
+
if (typeof value !== "string" || !value.trim()) fail(`${label} is required.`, "INVALID_INPUT");
|
|
2567
|
+
return value.trim();
|
|
2568
|
+
};
|
|
2569
|
+
var createModelPolicy = (bindings) => {
|
|
2570
|
+
if (!Array.isArray(bindings) || !bindings.length) fail("bindings must be a non-empty array.", "INVALID_INPUT");
|
|
2571
|
+
const normalized = bindings.map((binding2, index2) => {
|
|
2572
|
+
if (typeof binding2 !== "object" || binding2 === null || Array.isArray(binding2)) fail(`bindings[${index2}] must be an object.`, "INVALID_INPUT");
|
|
2573
|
+
if (!MODEL_ROLES.includes(binding2.role)) fail(`bindings[${index2}].role is invalid.`, "INVALID_INPUT");
|
|
2574
|
+
if (binding2.maxTokens !== void 0 && (!Number.isInteger(binding2.maxTokens) || binding2.maxTokens < 1)) fail(`bindings[${index2}].maxTokens must be a positive integer.`, "INVALID_INPUT");
|
|
2575
|
+
return { role: binding2.role, provider: required12(binding2.provider, `bindings[${index2}].provider`), model: required12(binding2.model, `bindings[${index2}].model`), ...binding2.maxTokens === void 0 ? {} : { maxTokens: binding2.maxTokens } };
|
|
2576
|
+
});
|
|
2577
|
+
if (new Set(normalized.map((binding2) => binding2.role)).size !== normalized.length) fail("Each model role may be bound only once.", "INVALID_INPUT");
|
|
2578
|
+
return { bindings: normalized, digest: hashJson(normalized) };
|
|
2579
|
+
};
|
|
2580
|
+
var modelFor = (policy, role) => policy.bindings.find((binding2) => binding2.role === role) ?? fail(`No model binding exists for role: ${role}.`, "INVALID_STATE");
|
|
2581
|
+
|
|
2582
|
+
// src/adapters/orca.ts
|
|
2583
|
+
var required13 = (value, label) => {
|
|
2584
|
+
if (typeof value !== "string" || !value.trim()) fail(`${label} is required.`, "INVALID_INPUT");
|
|
2585
|
+
return value.trim();
|
|
2586
|
+
};
|
|
2587
|
+
var createOrcaDispatchPlan = (input) => {
|
|
2588
|
+
const repository = required13(input.repository, "repository");
|
|
2589
|
+
const worktree = required13(input.worktree, "worktree");
|
|
2590
|
+
const branch = required13(input.branch, "branch");
|
|
2591
|
+
const baseBranch = required13(input.baseBranch, "baseBranch");
|
|
2592
|
+
const goalFile = required13(input.goalFile, "goalFile");
|
|
2593
|
+
const agent = required13(input.agent ?? "default", "agent");
|
|
2594
|
+
const argv = ["orca", "worktree", "create", "--repo", repository, "--name", worktree, "--base-branch", baseBranch, "--agent", agent, "--prompt-file", goalFile];
|
|
2595
|
+
validateSafeCommand(argv.join(" "));
|
|
2596
|
+
const identity = { repository, worktree, branch, baseBranch, goalFile, agent };
|
|
2597
|
+
return { argv, commandDigest: hashJson(argv), idempotencyKey: hashJson(identity) };
|
|
2598
|
+
};
|
|
2599
|
+
|
|
2600
|
+
// src/adapters/tracking.ts
|
|
2601
|
+
var required14 = (value, label) => {
|
|
2602
|
+
if (typeof value !== "string" || !value.trim()) fail(`${label} is required.`, "INVALID_INPUT");
|
|
2603
|
+
return value.trim();
|
|
2604
|
+
};
|
|
2605
|
+
var createTrackingTransition = (input) => {
|
|
2606
|
+
const transition2 = { tracker: required14(input.tracker, "tracker"), issue: required14(input.issue, "issue"), ...input.from ? { from: required14(input.from, "from") } : {}, to: required14(input.to, "to"), reason: required14(input.reason, "reason") };
|
|
2607
|
+
return { ...transition2, idempotencyKey: hashJson(transition2) };
|
|
2608
|
+
};
|
|
2609
|
+
var createTrackingAdapter = (id2, handler) => {
|
|
2610
|
+
const adapterId = required14(id2, "id");
|
|
2611
|
+
return { id: adapterId, transition: async (input) => {
|
|
2612
|
+
const transition2 = createTrackingTransition(input);
|
|
2613
|
+
await handler(transition2);
|
|
2614
|
+
return transition2;
|
|
2615
|
+
} };
|
|
2616
|
+
};
|
|
1741
2617
|
var EVIDENCE_BUNDLE_SCHEMA_VERSION = 1;
|
|
1742
2618
|
var body = (bundle) => {
|
|
1743
2619
|
const { payloadHash: _payloadHash, signature: _signature, ...unsigned } = bundle;
|
|
@@ -1764,7 +2640,7 @@ var exportEvidenceBundle = async ({ configPath, runId, outputPath, privateKeyPat
|
|
|
1764
2640
|
const loaded = loadConfig(configPath);
|
|
1765
2641
|
const run = requireRun2(runId ? readJson(join(loaded.stateDir, "runs", runId, "run.json")) : loadLatestRun(loaded.stateDir));
|
|
1766
2642
|
const reconciliation = await reconcileRun({ configPath, runId: run.runId });
|
|
1767
|
-
const
|
|
2643
|
+
const digest3 = run.verificationDigest ?? fail("Only a reconciled COMPLETE run can be exported.", "INVALID_STATE");
|
|
1768
2644
|
if (reconciliation.state !== "COMPLETE") fail("Only a reconciled COMPLETE run can be exported.", "INVALID_STATE");
|
|
1769
2645
|
const eventLog = new FileEventStore(loaded.stateDir);
|
|
1770
2646
|
eventLog.read(run.runId);
|
|
@@ -1779,7 +2655,7 @@ var exportEvidenceBundle = async ({ configPath, runId, outputPath, privateKeyPat
|
|
|
1779
2655
|
return bundleFile(loaded.stateDir, path);
|
|
1780
2656
|
});
|
|
1781
2657
|
const privateKey = createPrivateKey(readFileSync(privateKeyPath));
|
|
1782
|
-
const unsigned = { type: "agentskit-harness-evidence-bundle", schemaVersion: EVIDENCE_BUNDLE_SCHEMA_VERSION, runId: run.runId, signerKeyId: keyId, sourceRevision: run.sourceRevision, configHash: run.configHash, contractHash: run.contractHash, verificationDigest:
|
|
2658
|
+
const unsigned = { type: "agentskit-harness-evidence-bundle", schemaVersion: EVIDENCE_BUNDLE_SCHEMA_VERSION, runId: run.runId, signerKeyId: keyId, sourceRevision: run.sourceRevision, configHash: run.configHash, contractHash: run.contractHash, verificationDigest: digest3, eventLog: eventVerification, files };
|
|
1783
2659
|
const payloadHash = sha256(JSON.stringify(unsigned));
|
|
1784
2660
|
const publicKeyPem = createPublicKey(privateKey).export({ type: "spki", format: "pem" }).toString();
|
|
1785
2661
|
const bundle = { ...unsigned, payloadHash, signature: { algorithm: "ed25519", keyId, publicKeyPem, signatureBase64: sign(null, Buffer.from(payloadHash), privateKey).toString("base64") } };
|
|
@@ -1823,6 +2699,6 @@ var readEvidenceTrustStore = (path) => {
|
|
|
1823
2699
|
});
|
|
1824
2700
|
};
|
|
1825
2701
|
|
|
1826
|
-
export { BENCHMARK_SCHEMA_VERSION, CONTEXT_PROVIDER_SLOT, EVENT_LOG_GENESIS, EVIDENCE_BUNDLE_SCHEMA_VERSION, FileEventStore, HARNESS_EVENT_SCHEMA_VERSION, HARNESS_EVENT_TYPES, HARNESS_PLUGIN_API_VERSION, HarnessError, LEGAL_TRANSITIONS, STATES, approveRun, approvedDecision, assertHuman, authorizeRun, benchmarkRuns, cancelRun, cleanTaskArtifacts, createDocBridgeContextProvider, createDockerToolRuntime, createPluginRegistry, createPluginSlot, createPolicyGate, createProcessToolRuntime, createSessionRecorder, createToolRuntime, exportEvidenceBundle, hashContextSnapshot, hashContextSnapshots, inspectEventLogLock, loadBenchmarkManifest, loadConfig, loadLatestRun, planRun, readContextSnapshots, readEvidenceTrustStore, reconcileRun, recordBenchmarkObservation, recoverEventLogLock, retryRun, startRun, transition, validateBenchmarkManifest, validateConfig, validateContextSnapshot, validateContextSnapshots,
|
|
2702
|
+
export { BENCHMARK_SCHEMA_VERSION, BLOCK_STATUSES, CONTEXT_PROVIDER_SLOT, EVENT_LOG_GENESIS, EVIDENCE_BUNDLE_SCHEMA_VERSION, FileEventStore, HARNESS_EVENT_SCHEMA_VERSION, HARNESS_EVENT_TYPES, HARNESS_PLUGIN_API_VERSION, HarnessError, IMPROVEMENT_CYCLE_STEPS, LEARNING_STATUSES, LEGAL_TRANSITIONS, MEMORY_SCOPES, MODEL_ROLES, STATES, WIP_STATES, adaptiveConcurrency, approveRun, approvedDecision, assertHuman, assessAcceptance, assessAgentEval, assessBlock, assessDiscovery, assessImprovementCycle, assessIntegration, assessPilot, assessPreflight, assessProduction, assessWip, assessWorktreeCleanup, authorizeRun, benchmarkRuns, cancelRun, classifyFailure, cleanTaskArtifacts, compareOptimization, composePullRequest, createConfiguredToolRuntime, createDispatchLedger, createDocBridgeContextProvider, createDockerToolRuntime, createInMemoryMemoryAdapter, createKvMemoryAdapter, createLlmCache, createLlmCacheKey, createMachineMonitor, createModelPolicy, createOrcaDispatchPlan, createPluginRegistry, createPluginSlot, createPolicyGate, createProcessToolRuntime, createSessionRecorder, createStatusSnapshot, createToolRuntime, createTrackingAdapter, createTrackingTransition, exportEvidenceBundle, hashContextSnapshot, hashContextSnapshots, inspectEventLogLock, isDiscoveryCurrent, loadBenchmarkManifest, loadConfig, loadLatestRun, modelFor, parseRetro, planFilePreflight, planRun, promoteLearnings, readContextSnapshots, readEvidenceTrustStore, reconcileRun, recordBenchmarkObservation, recoverEventLogLock, recoveryDelayMs, retryRun, runAgentEval, runWithRecovery, runWorkflow, sampleMachine, selectRuntime, startRun, summarizeMachine, transition, validateBenchmarkManifest, validateBlockManifest, validateCacheableOperation, validateConfig, validateContextSnapshot, validateContextSnapshots, validateMemoryRecord, validateOptimizationObservation, validateSafeCommand, validateStatusSnapshot, verifyEvidenceBundle, verifyRun };
|
|
1827
2703
|
//# sourceMappingURL=index.js.map
|
|
1828
2704
|
//# sourceMappingURL=index.js.map
|