engineering-behavior-observatory 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +151 -0
  3. package/contracts/codex-app-server-0.150.1/manifest.json +25 -0
  4. package/contracts/codex-app-server-0.150.1/schema/InitializeParams.json +84 -0
  5. package/contracts/codex-app-server-0.150.1/schema/JSONRPCMessage.json +137 -0
  6. package/contracts/codex-app-server-0.150.1/schema/v2/ThreadReadParams.json +17 -0
  7. package/contracts/codex-app-server-0.150.1/schema/v2/ThreadTokenUsageUpdatedNotification.json +82 -0
  8. package/contracts/codex-app-server-0.150.1/schema/v2/TurnInterruptParams.json +17 -0
  9. package/contracts/codex-app-server-0.150.1/types/AskForApproval.ts +5 -0
  10. package/contracts/codex-app-server-0.150.1/types/ClientNotification.ts +5 -0
  11. package/contracts/codex-app-server-0.150.1/types/SandboxMode.ts +5 -0
  12. package/contracts/codex-app-server-0.150.1/types/ThreadReadParams.ts +9 -0
  13. package/contracts/codex-app-server-0.150.1/types/TokenUsageBreakdown.ts +5 -0
  14. package/contracts/codex-app-server-0.150.1/types/TurnInterruptParams.ts +5 -0
  15. package/contracts/codex-app-server-0.153.4/manifest.json +27 -0
  16. package/contracts/codex-app-server-0.153.4/schema/JSONRPCMessage.json +137 -0
  17. package/contracts/codex-app-server-0.153.4/schema/v1/InitializeParams.json +84 -0
  18. package/contracts/codex-app-server-0.153.4/schema/v2/ThreadReadParams.json +17 -0
  19. package/contracts/codex-app-server-0.153.4/schema/v2/ThreadStartParams.json +515 -0
  20. package/contracts/codex-app-server-0.153.4/schema/v2/ThreadTokenUsageUpdatedNotification.json +82 -0
  21. package/contracts/codex-app-server-0.153.4/schema/v2/TurnInterruptParams.json +17 -0
  22. package/contracts/codex-app-server-0.153.4/schema/v2/TurnStartParams.json +921 -0
  23. package/contracts/codex-app-server-0.153.4/types/AskForApproval.ts +5 -0
  24. package/contracts/codex-app-server-0.153.4/types/ClientNotification.ts +5 -0
  25. package/contracts/codex-app-server-0.153.4/types/SandboxMode.ts +5 -0
  26. package/contracts/codex-app-server-0.153.4/types/ThreadReadParams.ts +12 -0
  27. package/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.ts +5 -0
  28. package/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.ts +5 -0
  29. package/contracts/openhands-agent-server-v1.44.1.json +65 -0
  30. package/contracts/openhands-agent-server-v1.46.0.json +46 -0
  31. package/dist/contracts/codex-app-server-0.153.4/types/AskForApproval.d.ts +9 -0
  32. package/dist/contracts/codex-app-server-0.153.4/types/AskForApproval.js +2 -0
  33. package/dist/contracts/codex-app-server-0.153.4/types/ClientNotification.d.ts +3 -0
  34. package/dist/contracts/codex-app-server-0.153.4/types/ClientNotification.js +2 -0
  35. package/dist/contracts/codex-app-server-0.153.4/types/SandboxMode.d.ts +1 -0
  36. package/dist/contracts/codex-app-server-0.153.4/types/SandboxMode.js +2 -0
  37. package/dist/contracts/codex-app-server-0.153.4/types/ThreadReadParams.d.ts +10 -0
  38. package/dist/contracts/codex-app-server-0.153.4/types/ThreadReadParams.js +2 -0
  39. package/dist/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.d.ts +8 -0
  40. package/dist/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.js +2 -0
  41. package/dist/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.d.ts +4 -0
  42. package/dist/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.js +2 -0
  43. package/dist/src/agent-sdk-normalizer.d.ts +22 -0
  44. package/dist/src/agent-sdk-normalizer.js +995 -0
  45. package/dist/src/agent-sdk-run.d.ts +31 -0
  46. package/dist/src/agent-sdk-run.js +242 -0
  47. package/dist/src/agent-sdk-runner.d.ts +96 -0
  48. package/dist/src/agent-sdk-runner.js +407 -0
  49. package/dist/src/agent-sdk.d.ts +294 -0
  50. package/dist/src/agent-sdk.js +644 -0
  51. package/dist/src/aggregation.d.ts +183 -0
  52. package/dist/src/aggregation.js +513 -0
  53. package/dist/src/artifacts.d.ts +36 -0
  54. package/dist/src/artifacts.js +1372 -0
  55. package/dist/src/atlas-grafana.d.ts +377 -0
  56. package/dist/src/atlas-grafana.js +47 -0
  57. package/dist/src/atlas-html.d.ts +2 -0
  58. package/dist/src/atlas-html.js +34 -0
  59. package/dist/src/atlas.d.ts +130 -0
  60. package/dist/src/atlas.js +235 -0
  61. package/dist/src/behavior-assertions.d.ts +95 -0
  62. package/dist/src/behavior-assertions.js +154 -0
  63. package/dist/src/cli.d.ts +5 -0
  64. package/dist/src/cli.js +957 -0
  65. package/dist/src/codex-judge.d.ts +44 -0
  66. package/dist/src/codex-judge.js +214 -0
  67. package/dist/src/codex-run.d.ts +114 -0
  68. package/dist/src/codex-run.js +519 -0
  69. package/dist/src/codex.d.ts +187 -0
  70. package/dist/src/codex.js +1028 -0
  71. package/dist/src/contracts.d.ts +155 -0
  72. package/dist/src/contracts.js +866 -0
  73. package/dist/src/corpus.d.ts +67 -0
  74. package/dist/src/corpus.js +540 -0
  75. package/dist/src/cursor-sdk-runner.d.ts +81 -0
  76. package/dist/src/cursor-sdk-runner.js +362 -0
  77. package/dist/src/cursor-sdk.d.ts +81 -0
  78. package/dist/src/cursor-sdk.js +1078 -0
  79. package/dist/src/deepseek-adapter.d.ts +152 -0
  80. package/dist/src/deepseek-adapter.js +777 -0
  81. package/dist/src/exports.d.ts +69 -0
  82. package/dist/src/exports.js +966 -0
  83. package/dist/src/human-calibration.d.ts +167 -0
  84. package/dist/src/human-calibration.js +618 -0
  85. package/dist/src/index.d.ts +71 -0
  86. package/dist/src/index.js +38 -0
  87. package/dist/src/lifecycle.d.ts +196 -0
  88. package/dist/src/lifecycle.js +1889 -0
  89. package/dist/src/normalization-integrity.d.ts +128 -0
  90. package/dist/src/normalization-integrity.js +429 -0
  91. package/dist/src/openhands-run.d.ts +36 -0
  92. package/dist/src/openhands-run.js +415 -0
  93. package/dist/src/openhands.d.ts +130 -0
  94. package/dist/src/openhands.js +859 -0
  95. package/dist/src/pi.d.ts +203 -0
  96. package/dist/src/pi.js +1345 -0
  97. package/dist/src/process-protocol.d.ts +245 -0
  98. package/dist/src/process-protocol.js +1261 -0
  99. package/dist/src/retained-evidence.d.ts +12 -0
  100. package/dist/src/retained-evidence.js +201 -0
  101. package/dist/src/run-bundles.d.ts +193 -0
  102. package/dist/src/run-bundles.js +993 -0
  103. package/dist/src/scheduler.d.ts +109 -0
  104. package/dist/src/scheduler.js +1080 -0
  105. package/dist/src/semantic-judge.d.ts +194 -0
  106. package/dist/src/semantic-judge.js +875 -0
  107. package/dist/src/structural-observations.d.ts +124 -0
  108. package/dist/src/structural-observations.js +671 -0
  109. package/dist/src/task-packets.d.ts +115 -0
  110. package/dist/src/task-packets.js +683 -0
  111. package/dist/src/uniform-events.d.ts +120 -0
  112. package/dist/src/uniform-events.js +158 -0
  113. package/dist/src/verifiers.d.ts +116 -0
  114. package/dist/src/verifiers.js +819 -0
  115. package/dist/src/workspaces.d.ts +56 -0
  116. package/dist/src/workspaces.js +1314 -0
  117. package/docs/README.md +60 -0
  118. package/docs/development/README.md +71 -0
  119. package/docs/development/documentation-sync.md +58 -0
  120. package/docs/development/extension-contracts.md +307 -0
  121. package/docs/evaluation/README.md +48 -0
  122. package/docs/evaluation/aggregation.md +112 -0
  123. package/docs/evaluation/behavior-assertions.md +45 -0
  124. package/docs/evaluation/human-calibration.md +170 -0
  125. package/docs/evaluation/normalization-integrity.md +80 -0
  126. package/docs/evaluation/semantic-judge.md +188 -0
  127. package/docs/evaluation/structural-observations.md +84 -0
  128. package/docs/evaluation/uniform-events.md +131 -0
  129. package/docs/guides/atlas.md +144 -0
  130. package/docs/guides/evidence-and-sharing.md +100 -0
  131. package/docs/guides/operator-guide.md +424 -0
  132. package/docs/guides/quickstart.md +152 -0
  133. package/docs/guides/telemetry.md +66 -0
  134. package/docs/harnesses/README.md +31 -0
  135. package/docs/harnesses/claude-agent-sdk.md +61 -0
  136. package/docs/harnesses/codex-harness.md +174 -0
  137. package/docs/harnesses/cursor-sdk.md +149 -0
  138. package/docs/harnesses/deepseek-harness.md +134 -0
  139. package/docs/harnesses/openhands-agent-server.md +153 -0
  140. package/docs/harnesses/pi-sdk.md +66 -0
  141. package/docs/reference/README.md +19 -0
  142. package/docs/reference/agent-sdk-operational-runner.md +118 -0
  143. package/docs/reference/cli.md +114 -0
  144. package/docs/reference/contracts.md +222 -0
  145. package/docs/reference/run-bundle-contract.md +354 -0
  146. package/docs/reference/run-lifecycle.md +54 -0
  147. package/examples/cursor-sdk/README.md +11 -0
  148. package/examples/cursor-sdk/capture-profile.json +1 -0
  149. package/examples/cursor-sdk/harness.json +1 -0
  150. package/examples/cursor-sdk/model.json +1 -0
  151. package/examples/cursor-sdk/native-limits.json +1 -0
  152. package/examples/cursor-sdk/native-tool-policy.json +1 -0
  153. package/ontology/behavior-categories.v1.json +46 -0
  154. package/package.json +66 -0
  155. package/release/0.1.0/KNOWN_LIMITATIONS.md +35 -0
  156. package/release/0.1.0/README.md +46 -0
  157. package/release/0.1.0/reproducibility.json +61 -0
  158. package/release/0.2.0/KNOWN_LIMITATIONS.md +40 -0
  159. package/release/0.2.0/README.md +42 -0
  160. package/release/0.2.0/reproducibility.json +73 -0
  161. package/release/0.2.1/KNOWN_LIMITATIONS.md +18 -0
  162. package/release/0.2.1/README.md +63 -0
  163. package/release/0.2.1/reproducibility.json +73 -0
  164. package/release/README.md +13 -0
  165. package/schemas/aggregation.v1.json +293 -0
  166. package/schemas/behavior-assertions.v1.json +220 -0
  167. package/schemas/deepseek-runtime-composition.v1.schema.json +122 -0
  168. package/schemas/experiment.v1.schema.json +188 -0
  169. package/schemas/human-calibration.v1.json +336 -0
  170. package/schemas/normalization-integrity.v1.json +302 -0
  171. package/schemas/run-bundles/v1.json +1040 -0
  172. package/schemas/run-queue.v1.schema.json +206 -0
  173. package/schemas/semantic-judge.v1.json +380 -0
  174. package/schemas/structural-observations.v1.json +131 -0
  175. package/schemas/task-packet-freeze.v1.schema.json +106 -0
  176. package/schemas/task-packet.v1.schema.json +234 -0
  177. package/schemas/uniform-events/v1.json +289 -0
  178. package/scripts/atlas-grafana.sh +18 -0
@@ -0,0 +1,67 @@
1
+ import type { PortableExportManifest, PortableExportPolicy } from "./exports.js";
2
+ type DigestString = `sha256:${string}`;
3
+ export type CorpusSourceIssue = {
4
+ field: string;
5
+ message: string;
6
+ };
7
+ export type CorpusIndexEntry = {
8
+ schemaVersion: "ebo.corpus-index-entry/v1";
9
+ manifestKind: "run" | "export" | "unknown";
10
+ manifestPath: string;
11
+ manifestDigest: DigestString;
12
+ bundleId?: string;
13
+ runId?: string;
14
+ trialId?: string;
15
+ attemptId?: string;
16
+ attemptNumber?: number;
17
+ retryOf?: string;
18
+ taskId?: string;
19
+ taskDigest?: DigestString;
20
+ fixtureId?: string;
21
+ fixtureDigest?: DigestString;
22
+ modelProvider?: string;
23
+ modelId?: string;
24
+ modelConfigurationDigest?: DigestString;
25
+ harnessId?: string;
26
+ harnessVersion?: string;
27
+ harnessConfigurationDigest?: DigestString;
28
+ assessmentMode?: string;
29
+ configurationDigest?: DigestString;
30
+ captureProfileDigest?: DigestString;
31
+ budgetDigest?: DigestString;
32
+ toolPolicyDigest?: DigestString;
33
+ terminalState?: string;
34
+ failureClass?: string;
35
+ stopReason?: string;
36
+ verifierLocator?: string;
37
+ verifierDigest?: DigestString;
38
+ verifierArtifactIds: string[];
39
+ verifierStatuses: string[];
40
+ captureArtifactIds: string[];
41
+ captureQualification?: string;
42
+ exportArtifactIds: string[];
43
+ exportStatus?: string;
44
+ sharingClass?: string;
45
+ policyDigest?: DigestString;
46
+ sourceManifestDigest?: DigestString;
47
+ issues: CorpusSourceIssue[];
48
+ };
49
+ export type CorpusIndexQuery = Partial<Pick<CorpusIndexEntry, "manifestKind" | "runId" | "attemptId" | "taskId" | "modelId" | "harnessId" | "assessmentMode" | "terminalState" | "failureClass" | "captureQualification" | "exportStatus" | "sharingClass">> & {
50
+ verifierStatus?: string;
51
+ };
52
+ export type CorpusIndexValidationIssue = {
53
+ kind: "duplicate" | "missing" | "digest-mismatch" | "stale" | "unindexed" | "evidence";
54
+ manifestPath: string;
55
+ message: string;
56
+ };
57
+ /** Rebuild the non-authoritative corpus view from current run and export manifests. */
58
+ export declare function buildCorpusIndex(corpusRoot: string): CorpusIndexEntry[];
59
+ export declare function writeCorpusIndex(path: string, entries: readonly CorpusIndexEntry[]): void;
60
+ export declare function readCorpusIndex(path: string): CorpusIndexEntry[];
61
+ export declare function queryCorpusIndex(entries: readonly CorpusIndexEntry[], query: CorpusIndexQuery): CorpusIndexEntry[];
62
+ export declare function validateCorpusIndex(corpusRoot: string, entries: readonly CorpusIndexEntry[]): CorpusIndexValidationIssue[];
63
+ /** Package exactly one ready/exported derivative; unrelated sibling files are ignored. */
64
+ export declare function packPortableExport(exportRoot: string, archivePath: string, policy: PortableExportPolicy): Promise<DigestString>;
65
+ /** Unpack a bounded archive only when its contents exactly match the approved export manifest. */
66
+ export declare function unpackPortableExport(archivePath: string, destinationRoot: string): PortableExportManifest;
67
+ export {};
@@ -0,0 +1,540 @@
1
+ import { createHash, randomUUID } from "node:crypto";
2
+ import { closeSync, constants, fsyncSync, linkSync, mkdirSync, openSync, readdirSync, renameSync, rmSync, unlinkSync, writeFileSync, } from "node:fs";
3
+ import { dirname, join, relative, resolve, sep } from "node:path";
4
+ import { gzipSync } from "node:zlib";
5
+ import { assertNoDuplicateJsonKeys, assertUniqueArtifactIdentities, canonicalizeMetadata, digestBytes, validateArtifact, validateRunManifestEvidence, } from "./artifacts.js";
6
+ import { isSafeArtifactRelativePath, readTaskArchive, resolveBundleArtifact } from "./contracts.js";
7
+ import { readPortableRunBundleExport } from "./exports.js";
8
+ import { readBoundedFile } from "./scheduler.js";
9
+ const INDEX_SCHEMA_VERSION = "ebo.corpus-index-entry/v1";
10
+ const MAX_CORPUS_MANIFESTS = 100_000;
11
+ const MAX_CORPUS_INDEX_BYTES = 512 * 1024 * 1024;
12
+ const MAX_PORTABLE_ARCHIVE_BYTES = 512 * 1024 * 1024;
13
+ const MAX_PORTABLE_ARCHIVE_MEMBERS = 10_000;
14
+ const TAR_BLOCK_BYTES = 512;
15
+ /** Rebuild the non-authoritative corpus view from current run and export manifests. */
16
+ export function buildCorpusIndex(corpusRoot) {
17
+ const root = resolve(corpusRoot);
18
+ return discoverManifestPaths(root).map((manifestPath) => projectManifest(root, manifestPath));
19
+ }
20
+ export function writeCorpusIndex(path, entries) {
21
+ const bytes = Buffer.from(entries.map((entry) => canonicalizeMetadata(entry)).join("\n") + (entries.length === 0 ? "" : "\n"));
22
+ if (bytes.length > MAX_CORPUS_INDEX_BYTES)
23
+ throw new Error("Corpus index exceeds its local byte limit.");
24
+ writeAtomically(path, bytes, true);
25
+ }
26
+ export function readCorpusIndex(path) {
27
+ const text = new TextDecoder("utf-8", { fatal: true }).decode(readBoundedFile(path, "Corpus index", undefined, MAX_CORPUS_INDEX_BYTES));
28
+ if (text.trim() === "")
29
+ return [];
30
+ return text.trimEnd().split("\n").map((line, index) => {
31
+ assertNoDuplicateJsonKeys(line);
32
+ const entry = JSON.parse(line);
33
+ assertCorpusIndexEntry(entry, index + 1);
34
+ return entry;
35
+ });
36
+ }
37
+ export function queryCorpusIndex(entries, query) {
38
+ const filters = Object.entries(query);
39
+ return entries.filter((entry) => filters.every(([field, value]) => field === "verifierStatus"
40
+ ? entry.verifierStatuses.includes(value)
41
+ : entry[field] === value));
42
+ }
43
+ export function validateCorpusIndex(corpusRoot, entries) {
44
+ const issues = [];
45
+ const indexed = new Map();
46
+ for (const entry of entries) {
47
+ if (indexed.has(entry.manifestPath)) {
48
+ issues.push({ kind: "duplicate", manifestPath: entry.manifestPath, message: "Manifest path occurs more than once in the index." });
49
+ }
50
+ else
51
+ indexed.set(entry.manifestPath, entry);
52
+ }
53
+ const current = new Map(buildCorpusIndex(corpusRoot).map((entry) => [entry.manifestPath, entry]));
54
+ for (const [manifestPath, entry] of indexed) {
55
+ const rebuilt = current.get(manifestPath);
56
+ if (rebuilt === undefined) {
57
+ issues.push({ kind: "missing", manifestPath, message: "Indexed manifest is missing from the corpus." });
58
+ }
59
+ else if (entry.manifestDigest !== rebuilt.manifestDigest) {
60
+ issues.push({ kind: "digest-mismatch", manifestPath, message: "Indexed manifest digest does not match current bytes." });
61
+ }
62
+ else if (canonicalizeMetadata(entry) !== canonicalizeMetadata(rebuilt)) {
63
+ issues.push({ kind: "stale", manifestPath, message: "Indexed fields no longer match the current manifest or evidence." });
64
+ }
65
+ }
66
+ for (const [manifestPath, entry] of current) {
67
+ if (!indexed.has(manifestPath)) {
68
+ issues.push({ kind: "unindexed", manifestPath, message: "Corpus manifest is absent from the index." });
69
+ }
70
+ for (const issue of entry.issues) {
71
+ issues.push({ kind: "evidence", manifestPath, message: `${issue.field}: ${issue.message}` });
72
+ }
73
+ }
74
+ return issues.sort(compareValidationIssues);
75
+ }
76
+ /** Package exactly one ready/exported derivative; unrelated sibling files are ignored. */
77
+ export async function packPortableExport(exportRoot, archivePath, policy) {
78
+ await readPortableRunBundleExport(exportRoot, policy);
79
+ const files = readApprovedPortableExport(exportRoot);
80
+ const archive = gzipSync(createTar(files), { level: 9 });
81
+ if (archive.length > MAX_PORTABLE_ARCHIVE_BYTES)
82
+ throw new Error("Portable archive exceeds its byte limit.");
83
+ writeAtomically(archivePath, archive, false);
84
+ return digestString(digestBytes(archive));
85
+ }
86
+ /** Unpack a bounded archive only when its contents exactly match the approved export manifest. */
87
+ export function unpackPortableExport(archivePath, destinationRoot) {
88
+ const archive = readBoundedFile(archivePath, "Portable archive", undefined, MAX_PORTABLE_ARCHIVE_BYTES);
89
+ const members = readTaskArchive(archive, {
90
+ maxCompressedBytes: MAX_PORTABLE_ARCHIVE_BYTES,
91
+ maxExpandedBytes: MAX_PORTABLE_ARCHIVE_BYTES,
92
+ maxMembers: MAX_PORTABLE_ARCHIVE_MEMBERS,
93
+ }, ["bundle"]);
94
+ if (members.some(({ kind, path }) => kind !== "file" || !path.startsWith("bundle/"))) {
95
+ throw new Error("Portable archive contains an unsupported member.");
96
+ }
97
+ const byPath = new Map(members.map((member) => [member.path.slice("bundle/".length), member.bytes]));
98
+ if (byPath.size !== members.length)
99
+ throw new Error("Portable archive contains duplicate paths.");
100
+ const manifestBytes = byPath.get("manifest.json");
101
+ if (manifestBytes === undefined)
102
+ throw new Error("Portable archive omits manifest.json.");
103
+ const manifest = parseApprovedExportManifest(manifestBytes);
104
+ const expected = new Set(["manifest.json", ...manifest.artifacts.map(({ relativePath }) => relativePath)]);
105
+ if (expected.size !== byPath.size || [...byPath.keys()].some((path) => !expected.has(path))) {
106
+ throw new Error("Portable archive contents do not exactly match its export manifest.");
107
+ }
108
+ verifyPortableArtifacts(manifest, (artifact) => byPath.get(artifact.relativePath));
109
+ const destination = resolve(destinationRoot);
110
+ mkdirSync(dirname(destination), { recursive: true, mode: 0o700 });
111
+ mkdirSync(destination, { mode: 0o700 });
112
+ try {
113
+ for (const [relativePath, bytes] of [...byPath.entries()].sort(([left], [right]) => left.localeCompare(right))) {
114
+ const path = join(destination, ...relativePath.split("/"));
115
+ mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
116
+ writeFileSync(path, bytes, { flag: "wx", mode: 0o600 });
117
+ }
118
+ readApprovedPortableExport(destination);
119
+ return structuredClone(manifest);
120
+ }
121
+ catch (error) {
122
+ rmSync(destination, { recursive: true, force: true });
123
+ throw error;
124
+ }
125
+ }
126
+ function discoverManifestPaths(root) {
127
+ const paths = [];
128
+ const visit = (directory) => {
129
+ for (const entry of readdirSync(directory, { withFileTypes: true }).sort((left, right) => left.name.localeCompare(right.name))) {
130
+ const path = join(directory, entry.name);
131
+ if (entry.isDirectory())
132
+ visit(path);
133
+ else if (entry.name === "manifest.json") {
134
+ if (!entry.isFile())
135
+ throw new Error(`Corpus manifest "${path}" is not a regular file.`);
136
+ const locator = relative(root, path).split(sep).join("/");
137
+ if (!isSafeArtifactRelativePath(locator))
138
+ throw new Error(`Corpus manifest path "${locator}" is unsafe.`);
139
+ paths.push(locator);
140
+ if (paths.length > MAX_CORPUS_MANIFESTS)
141
+ throw new Error("Corpus exceeds the local manifest limit.");
142
+ }
143
+ }
144
+ };
145
+ visit(root);
146
+ return paths.sort();
147
+ }
148
+ function projectManifest(root, manifestPath) {
149
+ const bytes = readBoundedFile(join(root, ...manifestPath.split("/")), "Corpus manifest");
150
+ const manifestRoot = dirname(join(root, ...manifestPath.split("/")));
151
+ const entry = {
152
+ schemaVersion: INDEX_SCHEMA_VERSION,
153
+ manifestKind: "unknown",
154
+ manifestPath,
155
+ manifestDigest: digestString(digestBytes(bytes)),
156
+ verifierArtifactIds: [],
157
+ verifierStatuses: [],
158
+ captureArtifactIds: [],
159
+ exportArtifactIds: [],
160
+ issues: [],
161
+ };
162
+ let document;
163
+ try {
164
+ document = parseJson(bytes, `Corpus manifest ${manifestPath}`);
165
+ }
166
+ catch (error) {
167
+ entry.issues.push({ field: "/", message: errorMessage(error) });
168
+ return entry;
169
+ }
170
+ const schemaVersion = isRecord(document) ? document.schemaVersion : undefined;
171
+ entry.manifestKind = schemaVersion === "run-manifest/v1" ? "run"
172
+ : schemaVersion === "export-manifest/v1" ? "export" : "unknown";
173
+ entry.issues.push(...validateArtifact(manifestPath, document).map(({ field, message }) => ({ field, message })));
174
+ if (!isRecord(document))
175
+ return entry;
176
+ if (typeof document.bundleId === "string")
177
+ entry.bundleId = document.bundleId;
178
+ if (entry.manifestKind === "run")
179
+ projectRun(entry, document, manifestRoot);
180
+ if (entry.manifestKind === "export")
181
+ projectExport(entry, document, manifestRoot);
182
+ entry.issues = uniqueSourceIssues(entry.issues);
183
+ return entry;
184
+ }
185
+ function projectRun(entry, manifest, manifestRoot) {
186
+ entry.issues.push(...validateRunManifestEvidence(entry.manifestPath, manifest, manifestRoot)
187
+ .map(({ field, message }) => ({ field, message })));
188
+ if (isRecord(manifest.run)) {
189
+ assignString(entry, "runId", manifest.run.id);
190
+ if (isRecord(manifest.run.trial)) {
191
+ const trialIndex = numberValue(manifest.run.trial.index);
192
+ if (trialIndex !== undefined)
193
+ entry.trialId = String(trialIndex);
194
+ }
195
+ assignString(entry, "taskId", isRecord(manifest.run.task) ? manifest.run.task.id : undefined);
196
+ if (isRecord(manifest.run.task))
197
+ assignDigest(entry, "taskDigest", manifest.run.task.digest);
198
+ assignString(entry, "fixtureId", isRecord(manifest.run.fixture) ? manifest.run.fixture.id : undefined);
199
+ if (isRecord(manifest.run.fixture))
200
+ assignDigest(entry, "fixtureDigest", manifest.run.fixture.digest);
201
+ assignString(entry, "modelProvider", isRecord(manifest.run.model) ? manifest.run.model.provider : undefined);
202
+ assignString(entry, "modelId", isRecord(manifest.run.model) ? manifest.run.model.id : undefined);
203
+ if (isRecord(manifest.run.model))
204
+ assignDigest(entry, "modelConfigurationDigest", manifest.run.model.configurationDigest);
205
+ assignString(entry, "harnessId", isRecord(manifest.run.harness) ? manifest.run.harness.id : undefined);
206
+ assignString(entry, "harnessVersion", isRecord(manifest.run.harness) ? manifest.run.harness.version : undefined);
207
+ if (isRecord(manifest.run.harness))
208
+ assignDigest(entry, "harnessConfigurationDigest", manifest.run.harness.configurationDigest);
209
+ assignString(entry, "assessmentMode", manifest.run.assessmentMode ?? "verified");
210
+ if (isRecord(manifest.run.verifier)) {
211
+ assignString(entry, "verifierLocator", manifest.run.verifier.locator);
212
+ assignDigest(entry, "verifierDigest", manifest.run.verifier.digest);
213
+ }
214
+ }
215
+ if (isRecord(manifest.attempt)) {
216
+ assignString(entry, "attemptId", manifest.attempt.id);
217
+ const attemptNumber = numberValue(manifest.attempt.number);
218
+ if (attemptNumber !== undefined)
219
+ entry.attemptNumber = attemptNumber;
220
+ assignString(entry, "retryOf", manifest.attempt.retryOf);
221
+ }
222
+ if (isRecord(manifest.configuration)) {
223
+ assignDigest(entry, "configurationDigest", manifest.configuration.digest);
224
+ assignDigest(entry, "captureProfileDigest", manifest.configuration.captureProfileDigest);
225
+ assignDigest(entry, "budgetDigest", manifest.configuration.budgetDigest);
226
+ assignDigest(entry, "toolPolicyDigest", manifest.configuration.toolPolicyDigest);
227
+ }
228
+ if (isRecord(manifest.terminal)) {
229
+ assignString(entry, "terminalState", manifest.terminal.state);
230
+ assignString(entry, "failureClass", manifest.terminal.failureClass);
231
+ assignString(entry, "stopReason", manifest.terminal.stopReason);
232
+ }
233
+ projectEvidence(entry, manifestRoot, Array.isArray(manifest.evidence) ? manifest.evidence : [], isRecord(manifest.terminal) && typeof manifest.terminal.workspaceArtifactId === "string" ? manifest.terminal.workspaceArtifactId : undefined, true);
234
+ }
235
+ function projectExport(entry, manifest, manifestRoot) {
236
+ assignString(entry, "exportStatus", manifest.status);
237
+ assignString(entry, "sharingClass", manifest.sharingClass);
238
+ assignString(entry, "assessmentMode", manifest.assessmentMode ?? "verified");
239
+ assignDigest(entry, "policyDigest", manifest.policyDigest);
240
+ assignDigest(entry, "sourceManifestDigest", manifest.sourceManifestDigest);
241
+ if (isRecord(manifest.correlations)) {
242
+ assignString(entry, "runId", manifest.correlations.runId);
243
+ assignString(entry, "attemptId", manifest.correlations.attemptId);
244
+ }
245
+ const artifacts = Array.isArray(manifest.artifacts) ? manifest.artifacts : [];
246
+ entry.exportArtifactIds = artifacts
247
+ .filter(isRecord)
248
+ .map(({ id }) => id)
249
+ .filter((id) => typeof id === "string");
250
+ projectEvidence(entry, manifestRoot, artifacts);
251
+ if (["ready", "exported"].includes(String(manifest.status))) {
252
+ try {
253
+ readApprovedPortableExport(manifestRoot);
254
+ }
255
+ catch (error) {
256
+ entry.issues.push({ field: "/artifacts", message: errorMessage(error) });
257
+ }
258
+ }
259
+ }
260
+ function projectEvidence(entry, root, descriptors, terminalWorkspaceArtifactId, terminalVerifierOnly = false) {
261
+ for (const candidate of descriptors) {
262
+ if (!isRecord(candidate) || typeof candidate.id !== "string")
263
+ continue;
264
+ const descriptor = candidate;
265
+ if (descriptor.kind === "verifier") {
266
+ entry.verifierArtifactIds.push(descriptor.id);
267
+ const document = readDescriptorJson(root, candidate, entry.issues);
268
+ const workspace = isRecord(document) && isRecord(document.workspace) ? document.workspace : undefined;
269
+ if (isRecord(document) && typeof document.status === "string" && (!terminalVerifierOnly
270
+ || !isRecord(candidate.sanitizedFrom) && terminalWorkspaceArtifactId !== undefined
271
+ && workspace?.artifactId === terminalWorkspaceArtifactId)) {
272
+ entry.verifierStatuses.push(document.status);
273
+ }
274
+ }
275
+ else if (descriptor.kind === "capture-report") {
276
+ entry.captureArtifactIds.push(descriptor.id);
277
+ const document = readDescriptorJson(root, candidate, entry.issues);
278
+ if (isRecord(document)) {
279
+ const structural = document.structuralQualification;
280
+ // Structural qualification supersedes the legacy coarse capture summary.
281
+ const status = isRecord(structural) ? structural.status : structural === undefined ? document.qualification : undefined;
282
+ if (["qualified", "qualified-with-gaps", "incomplete", "unqualified", "unavailable"].includes(String(status))) {
283
+ assignString(entry, "captureQualification", status);
284
+ }
285
+ else
286
+ entry.captureQualification = "unavailable";
287
+ }
288
+ }
289
+ else if (descriptor.kind === "export-manifest")
290
+ entry.exportArtifactIds.push(descriptor.id);
291
+ }
292
+ entry.verifierArtifactIds.sort();
293
+ entry.verifierStatuses.sort();
294
+ entry.captureArtifactIds.sort();
295
+ entry.exportArtifactIds.sort();
296
+ }
297
+ function readDescriptorJson(root, descriptor, issues) {
298
+ const { id, relativePath, digest } = descriptor;
299
+ if (typeof id !== "string" || typeof relativePath !== "string" || typeof digest !== "string"
300
+ || digestValue(digest) === undefined)
301
+ return undefined;
302
+ try {
303
+ return parseJson(resolveBundleArtifact(root, {
304
+ locator: relativePath,
305
+ digest: parseDigest(digest),
306
+ }), `Evidence ${id}`);
307
+ }
308
+ catch (error) {
309
+ issues.push({ field: `/evidence/${id}`, message: errorMessage(error) });
310
+ return undefined;
311
+ }
312
+ }
313
+ function readApprovedPortableExport(exportRoot) {
314
+ const root = resolve(exportRoot);
315
+ const manifestBytes = readBoundedFile(join(root, "manifest.json"), "Portable export manifest");
316
+ const manifest = parseApprovedExportManifest(manifestBytes);
317
+ const files = new Map([["bundle/manifest.json", manifestBytes]]);
318
+ verifyPortableArtifacts(manifest, (artifact) => {
319
+ const bytes = resolveBundleArtifact(root, {
320
+ locator: artifact.relativePath,
321
+ digest: parseDigest(artifact.digest),
322
+ }, MAX_PORTABLE_ARCHIVE_BYTES);
323
+ files.set(`bundle/${artifact.relativePath}`, bytes);
324
+ return bytes;
325
+ });
326
+ if (files.size > MAX_PORTABLE_ARCHIVE_MEMBERS)
327
+ throw new Error("Portable export exceeds the archive member limit.");
328
+ const totalBytes = [...files.values()].reduce((total, bytes) => total + bytes.length, 0);
329
+ if (!Number.isSafeInteger(totalBytes) || totalBytes > MAX_PORTABLE_ARCHIVE_BYTES) {
330
+ throw new Error("Portable export exceeds the archive byte limit.");
331
+ }
332
+ return files;
333
+ }
334
+ function parseApprovedExportManifest(bytes) {
335
+ const manifest = parseJson(bytes, "Portable export manifest");
336
+ const errors = validateArtifact("manifest.json", manifest);
337
+ if (errors.length > 0)
338
+ throw new Error(errors.map(({ field, message }) => `${field}: ${message}`).join("\n"));
339
+ if (!isRecord(manifest) || !["ready", "exported"].includes(String(manifest.status))) {
340
+ throw new Error("Portable export manifest must be ready or exported.");
341
+ }
342
+ if (!["partner", "public"].includes(String(manifest.sharingClass))) {
343
+ throw new Error("Portable export manifest must use partner or public sharing.");
344
+ }
345
+ if (!Array.isArray(manifest.artifacts))
346
+ throw new Error("Portable export manifest has no artifacts.");
347
+ assertUniqueArtifactIdentities(manifest.artifacts.map(({ id, relativePath }) => ({ id, relativePath })));
348
+ if (JSON.stringify(manifest.artifactIds) !== JSON.stringify(manifest.artifacts.map(({ id }) => id))) {
349
+ throw new Error("Portable export artifact index does not match its descriptors.");
350
+ }
351
+ return manifest;
352
+ }
353
+ function verifyPortableArtifacts(manifest, read) {
354
+ for (const artifact of manifest.artifacts) {
355
+ const bytes = read(artifact);
356
+ if (bytes === undefined)
357
+ throw new Error(`Portable export omits artifact ${artifact.id}.`);
358
+ if (bytes.length !== artifact.sizeBytes || digestString(digestBytes(bytes)) !== artifact.digest) {
359
+ throw new Error(`Portable artifact ${artifact.id} does not match its descriptor.`);
360
+ }
361
+ }
362
+ }
363
+ function createTar(files) {
364
+ const blocks = [];
365
+ for (const [path, bytes] of [...files.entries()].sort(([left], [right]) => left.localeCompare(right))) {
366
+ const split = splitUstarPath(path);
367
+ let headerPath = path;
368
+ if (split === undefined) {
369
+ const pax = paxPathRecord(path);
370
+ blocks.push(tarHeader(`PaxHeaders/${createHash("sha256").update(path).digest("hex").slice(0, 24)}`, pax.length, "x"));
371
+ blocks.push(pax, padding(pax.length));
372
+ headerPath = `files/${createHash("sha256").update(path).digest("hex")}`;
373
+ }
374
+ blocks.push(tarHeader(headerPath, bytes.length, "0"), bytes, padding(bytes.length));
375
+ }
376
+ blocks.push(Buffer.alloc(TAR_BLOCK_BYTES * 2));
377
+ return Buffer.concat(blocks);
378
+ }
379
+ function tarHeader(path, size, type) {
380
+ const split = splitUstarPath(path);
381
+ if (split === undefined)
382
+ throw new Error(`Portable archive path "${path}" cannot be encoded.`);
383
+ const header = Buffer.alloc(TAR_BLOCK_BYTES);
384
+ header.write(split.name, 0, 100, "utf8");
385
+ writeOctal(header, 100, 8, 0o600);
386
+ writeOctal(header, 108, 8, 0);
387
+ writeOctal(header, 116, 8, 0);
388
+ writeOctal(header, 124, 12, size);
389
+ writeOctal(header, 136, 12, 0);
390
+ header.fill(0x20, 148, 156);
391
+ header[156] = type.charCodeAt(0);
392
+ header.write("ustar\0", 257, 6, "ascii");
393
+ header.write("00", 263, 2, "ascii");
394
+ header.write("EBO", 265, 32, "ascii");
395
+ header.write("EBO", 297, 32, "ascii");
396
+ if (split.prefix !== "")
397
+ header.write(split.prefix, 345, 155, "utf8");
398
+ const checksum = header.reduce((sum, byte) => sum + byte, 0).toString(8).padStart(6, "0");
399
+ header.write(checksum, 148, 6, "ascii");
400
+ header[154] = 0;
401
+ header[155] = 0x20;
402
+ return header;
403
+ }
404
+ function splitUstarPath(path) {
405
+ if (Buffer.byteLength(path) <= 100)
406
+ return { name: path, prefix: "" };
407
+ for (let boundary = path.lastIndexOf("/"); boundary > 0; boundary = path.lastIndexOf("/", boundary - 1)) {
408
+ const prefix = path.slice(0, boundary);
409
+ const name = path.slice(boundary + 1);
410
+ if (Buffer.byteLength(prefix) <= 155 && Buffer.byteLength(name) <= 100)
411
+ return { name, prefix };
412
+ }
413
+ return undefined;
414
+ }
415
+ function paxPathRecord(path) {
416
+ const content = `path=${path}\n`;
417
+ let length = Buffer.byteLength(content) + 2;
418
+ while (true) {
419
+ const record = `${length} ${content}`;
420
+ const actual = Buffer.byteLength(record);
421
+ if (actual === length)
422
+ return Buffer.from(record);
423
+ length = actual;
424
+ }
425
+ }
426
+ function writeOctal(buffer, offset, length, value) {
427
+ const octal = value.toString(8);
428
+ if (octal.length > length - 1)
429
+ throw new Error("Portable archive value exceeds TAR field limits.");
430
+ buffer.write(`${octal.padStart(length - 1, "0")}\0`, offset, length, "ascii");
431
+ }
432
+ function padding(size) {
433
+ return Buffer.alloc((TAR_BLOCK_BYTES - (size % TAR_BLOCK_BYTES)) % TAR_BLOCK_BYTES);
434
+ }
435
+ function writeAtomically(path, bytes, overwrite) {
436
+ const destination = resolve(path);
437
+ const parent = dirname(destination);
438
+ const temporary = join(parent, `.${randomUUID()}.tmp`);
439
+ mkdirSync(parent, { recursive: true, mode: 0o700 });
440
+ let descriptor;
441
+ try {
442
+ descriptor = openSync(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600);
443
+ writeFileSync(descriptor, bytes);
444
+ fsyncSync(descriptor);
445
+ closeSync(descriptor);
446
+ descriptor = undefined;
447
+ if (overwrite)
448
+ renameSync(temporary, destination);
449
+ else {
450
+ linkSync(temporary, destination);
451
+ unlinkSync(temporary);
452
+ }
453
+ const parentDescriptor = openSync(parent, constants.O_RDONLY);
454
+ try {
455
+ fsyncSync(parentDescriptor);
456
+ }
457
+ finally {
458
+ closeSync(parentDescriptor);
459
+ }
460
+ }
461
+ finally {
462
+ if (descriptor !== undefined)
463
+ closeSync(descriptor);
464
+ rmSync(temporary, { force: true });
465
+ }
466
+ }
467
+ function assertCorpusIndexEntry(value, line) {
468
+ if (!isRecord(value))
469
+ throw new Error(`Corpus index line ${line} is invalid.`);
470
+ const stringArrays = [value.verifierArtifactIds, value.verifierStatuses, value.captureArtifactIds, value.exportArtifactIds];
471
+ const optionalStrings = [
472
+ value.bundleId, value.runId, value.trialId, value.attemptId, value.retryOf, value.taskId, value.fixtureId,
473
+ value.modelProvider, value.modelId, value.harnessId, value.harnessVersion, value.assessmentMode, value.terminalState,
474
+ value.failureClass, value.stopReason, value.verifierLocator, value.captureQualification, value.exportStatus,
475
+ value.sharingClass,
476
+ ];
477
+ if (value.schemaVersion !== INDEX_SCHEMA_VERSION
478
+ || !["run", "export", "unknown"].includes(String(value.manifestKind))
479
+ || typeof value.manifestPath !== "string" || !isSafeArtifactRelativePath(value.manifestPath)
480
+ || digestValue(value.manifestDigest) === undefined
481
+ || stringArrays.some((items) => !Array.isArray(items) || items.some((item) => typeof item !== "string"))
482
+ || optionalStrings.some((item) => item !== undefined && typeof item !== "string")
483
+ || (value.attemptNumber !== undefined && (typeof value.attemptNumber !== "number"
484
+ || !Number.isSafeInteger(value.attemptNumber) || value.attemptNumber < 1))
485
+ || [value.manifestDigest, value.configurationDigest, value.verifierDigest, value.policyDigest, value.sourceManifestDigest]
486
+ .some((digest) => digest !== undefined && digestValue(digest) === undefined)
487
+ || !Array.isArray(value.issues)
488
+ || value.issues.some((issue) => !isRecord(issue) || typeof issue.field !== "string" || typeof issue.message !== "string")) {
489
+ throw new Error(`Corpus index line ${line} is invalid.`);
490
+ }
491
+ }
492
+ function uniqueSourceIssues(issues) {
493
+ return [...new Map(issues.map((issue) => [`${issue.field}\0${issue.message}`, issue])).values()]
494
+ .sort((left, right) => left.field.localeCompare(right.field) || left.message.localeCompare(right.message));
495
+ }
496
+ function compareValidationIssues(left, right) {
497
+ return left.manifestPath.localeCompare(right.manifestPath)
498
+ || left.kind.localeCompare(right.kind)
499
+ || left.message.localeCompare(right.message);
500
+ }
501
+ function parseJson(bytes, label) {
502
+ const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
503
+ assertNoDuplicateJsonKeys(text);
504
+ try {
505
+ return JSON.parse(text);
506
+ }
507
+ catch {
508
+ throw new Error(`${label} is not valid JSON.`);
509
+ }
510
+ }
511
+ function parseDigest(value) {
512
+ const digest = digestValue(value);
513
+ if (digest === undefined)
514
+ throw new Error("Artifact digest is invalid.");
515
+ return { algorithm: "sha256", value: digest.slice("sha256:".length) };
516
+ }
517
+ function digestString(value) {
518
+ return `sha256:${value.value}`;
519
+ }
520
+ function digestValue(value) {
521
+ return typeof value === "string" && /^sha256:[a-f0-9]{64}$/.test(value) ? value : undefined;
522
+ }
523
+ function numberValue(value) {
524
+ return typeof value === "number" ? value : undefined;
525
+ }
526
+ function assignString(entry, field, value) {
527
+ if (typeof value === "string")
528
+ entry[field] = value;
529
+ }
530
+ function assignDigest(entry, field, value) {
531
+ const digest = digestValue(value);
532
+ if (digest !== undefined)
533
+ entry[field] = digest;
534
+ }
535
+ function errorMessage(error) {
536
+ return error instanceof Error ? error.message : String(error);
537
+ }
538
+ function isRecord(value) {
539
+ return typeof value === "object" && value !== null && !Array.isArray(value);
540
+ }