engineering-behavior-observatory 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +151 -0
  3. package/contracts/codex-app-server-0.150.1/manifest.json +25 -0
  4. package/contracts/codex-app-server-0.150.1/schema/InitializeParams.json +84 -0
  5. package/contracts/codex-app-server-0.150.1/schema/JSONRPCMessage.json +137 -0
  6. package/contracts/codex-app-server-0.150.1/schema/v2/ThreadReadParams.json +17 -0
  7. package/contracts/codex-app-server-0.150.1/schema/v2/ThreadTokenUsageUpdatedNotification.json +82 -0
  8. package/contracts/codex-app-server-0.150.1/schema/v2/TurnInterruptParams.json +17 -0
  9. package/contracts/codex-app-server-0.150.1/types/AskForApproval.ts +5 -0
  10. package/contracts/codex-app-server-0.150.1/types/ClientNotification.ts +5 -0
  11. package/contracts/codex-app-server-0.150.1/types/SandboxMode.ts +5 -0
  12. package/contracts/codex-app-server-0.150.1/types/ThreadReadParams.ts +9 -0
  13. package/contracts/codex-app-server-0.150.1/types/TokenUsageBreakdown.ts +5 -0
  14. package/contracts/codex-app-server-0.150.1/types/TurnInterruptParams.ts +5 -0
  15. package/contracts/codex-app-server-0.153.4/manifest.json +27 -0
  16. package/contracts/codex-app-server-0.153.4/schema/JSONRPCMessage.json +137 -0
  17. package/contracts/codex-app-server-0.153.4/schema/v1/InitializeParams.json +84 -0
  18. package/contracts/codex-app-server-0.153.4/schema/v2/ThreadReadParams.json +17 -0
  19. package/contracts/codex-app-server-0.153.4/schema/v2/ThreadStartParams.json +515 -0
  20. package/contracts/codex-app-server-0.153.4/schema/v2/ThreadTokenUsageUpdatedNotification.json +82 -0
  21. package/contracts/codex-app-server-0.153.4/schema/v2/TurnInterruptParams.json +17 -0
  22. package/contracts/codex-app-server-0.153.4/schema/v2/TurnStartParams.json +921 -0
  23. package/contracts/codex-app-server-0.153.4/types/AskForApproval.ts +5 -0
  24. package/contracts/codex-app-server-0.153.4/types/ClientNotification.ts +5 -0
  25. package/contracts/codex-app-server-0.153.4/types/SandboxMode.ts +5 -0
  26. package/contracts/codex-app-server-0.153.4/types/ThreadReadParams.ts +12 -0
  27. package/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.ts +5 -0
  28. package/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.ts +5 -0
  29. package/contracts/openhands-agent-server-v1.44.1.json +65 -0
  30. package/contracts/openhands-agent-server-v1.46.0.json +46 -0
  31. package/dist/contracts/codex-app-server-0.153.4/types/AskForApproval.d.ts +9 -0
  32. package/dist/contracts/codex-app-server-0.153.4/types/AskForApproval.js +2 -0
  33. package/dist/contracts/codex-app-server-0.153.4/types/ClientNotification.d.ts +3 -0
  34. package/dist/contracts/codex-app-server-0.153.4/types/ClientNotification.js +2 -0
  35. package/dist/contracts/codex-app-server-0.153.4/types/SandboxMode.d.ts +1 -0
  36. package/dist/contracts/codex-app-server-0.153.4/types/SandboxMode.js +2 -0
  37. package/dist/contracts/codex-app-server-0.153.4/types/ThreadReadParams.d.ts +10 -0
  38. package/dist/contracts/codex-app-server-0.153.4/types/ThreadReadParams.js +2 -0
  39. package/dist/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.d.ts +8 -0
  40. package/dist/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.js +2 -0
  41. package/dist/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.d.ts +4 -0
  42. package/dist/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.js +2 -0
  43. package/dist/src/agent-sdk-normalizer.d.ts +22 -0
  44. package/dist/src/agent-sdk-normalizer.js +995 -0
  45. package/dist/src/agent-sdk-run.d.ts +31 -0
  46. package/dist/src/agent-sdk-run.js +242 -0
  47. package/dist/src/agent-sdk-runner.d.ts +96 -0
  48. package/dist/src/agent-sdk-runner.js +407 -0
  49. package/dist/src/agent-sdk.d.ts +294 -0
  50. package/dist/src/agent-sdk.js +644 -0
  51. package/dist/src/aggregation.d.ts +183 -0
  52. package/dist/src/aggregation.js +513 -0
  53. package/dist/src/artifacts.d.ts +36 -0
  54. package/dist/src/artifacts.js +1372 -0
  55. package/dist/src/atlas-grafana.d.ts +377 -0
  56. package/dist/src/atlas-grafana.js +47 -0
  57. package/dist/src/atlas-html.d.ts +2 -0
  58. package/dist/src/atlas-html.js +34 -0
  59. package/dist/src/atlas.d.ts +130 -0
  60. package/dist/src/atlas.js +235 -0
  61. package/dist/src/behavior-assertions.d.ts +95 -0
  62. package/dist/src/behavior-assertions.js +154 -0
  63. package/dist/src/cli.d.ts +5 -0
  64. package/dist/src/cli.js +957 -0
  65. package/dist/src/codex-judge.d.ts +44 -0
  66. package/dist/src/codex-judge.js +214 -0
  67. package/dist/src/codex-run.d.ts +114 -0
  68. package/dist/src/codex-run.js +519 -0
  69. package/dist/src/codex.d.ts +187 -0
  70. package/dist/src/codex.js +1028 -0
  71. package/dist/src/contracts.d.ts +155 -0
  72. package/dist/src/contracts.js +866 -0
  73. package/dist/src/corpus.d.ts +67 -0
  74. package/dist/src/corpus.js +540 -0
  75. package/dist/src/cursor-sdk-runner.d.ts +81 -0
  76. package/dist/src/cursor-sdk-runner.js +362 -0
  77. package/dist/src/cursor-sdk.d.ts +81 -0
  78. package/dist/src/cursor-sdk.js +1078 -0
  79. package/dist/src/deepseek-adapter.d.ts +152 -0
  80. package/dist/src/deepseek-adapter.js +777 -0
  81. package/dist/src/exports.d.ts +69 -0
  82. package/dist/src/exports.js +966 -0
  83. package/dist/src/human-calibration.d.ts +167 -0
  84. package/dist/src/human-calibration.js +618 -0
  85. package/dist/src/index.d.ts +71 -0
  86. package/dist/src/index.js +38 -0
  87. package/dist/src/lifecycle.d.ts +196 -0
  88. package/dist/src/lifecycle.js +1889 -0
  89. package/dist/src/normalization-integrity.d.ts +128 -0
  90. package/dist/src/normalization-integrity.js +429 -0
  91. package/dist/src/openhands-run.d.ts +36 -0
  92. package/dist/src/openhands-run.js +415 -0
  93. package/dist/src/openhands.d.ts +130 -0
  94. package/dist/src/openhands.js +859 -0
  95. package/dist/src/pi.d.ts +203 -0
  96. package/dist/src/pi.js +1345 -0
  97. package/dist/src/process-protocol.d.ts +245 -0
  98. package/dist/src/process-protocol.js +1261 -0
  99. package/dist/src/retained-evidence.d.ts +12 -0
  100. package/dist/src/retained-evidence.js +201 -0
  101. package/dist/src/run-bundles.d.ts +193 -0
  102. package/dist/src/run-bundles.js +993 -0
  103. package/dist/src/scheduler.d.ts +109 -0
  104. package/dist/src/scheduler.js +1080 -0
  105. package/dist/src/semantic-judge.d.ts +194 -0
  106. package/dist/src/semantic-judge.js +875 -0
  107. package/dist/src/structural-observations.d.ts +124 -0
  108. package/dist/src/structural-observations.js +671 -0
  109. package/dist/src/task-packets.d.ts +115 -0
  110. package/dist/src/task-packets.js +683 -0
  111. package/dist/src/uniform-events.d.ts +120 -0
  112. package/dist/src/uniform-events.js +158 -0
  113. package/dist/src/verifiers.d.ts +116 -0
  114. package/dist/src/verifiers.js +819 -0
  115. package/dist/src/workspaces.d.ts +56 -0
  116. package/dist/src/workspaces.js +1314 -0
  117. package/docs/README.md +60 -0
  118. package/docs/development/README.md +71 -0
  119. package/docs/development/documentation-sync.md +58 -0
  120. package/docs/development/extension-contracts.md +307 -0
  121. package/docs/evaluation/README.md +48 -0
  122. package/docs/evaluation/aggregation.md +112 -0
  123. package/docs/evaluation/behavior-assertions.md +45 -0
  124. package/docs/evaluation/human-calibration.md +170 -0
  125. package/docs/evaluation/normalization-integrity.md +80 -0
  126. package/docs/evaluation/semantic-judge.md +188 -0
  127. package/docs/evaluation/structural-observations.md +84 -0
  128. package/docs/evaluation/uniform-events.md +131 -0
  129. package/docs/guides/atlas.md +144 -0
  130. package/docs/guides/evidence-and-sharing.md +100 -0
  131. package/docs/guides/operator-guide.md +424 -0
  132. package/docs/guides/quickstart.md +152 -0
  133. package/docs/guides/telemetry.md +66 -0
  134. package/docs/harnesses/README.md +31 -0
  135. package/docs/harnesses/claude-agent-sdk.md +61 -0
  136. package/docs/harnesses/codex-harness.md +174 -0
  137. package/docs/harnesses/cursor-sdk.md +149 -0
  138. package/docs/harnesses/deepseek-harness.md +134 -0
  139. package/docs/harnesses/openhands-agent-server.md +153 -0
  140. package/docs/harnesses/pi-sdk.md +66 -0
  141. package/docs/reference/README.md +19 -0
  142. package/docs/reference/agent-sdk-operational-runner.md +118 -0
  143. package/docs/reference/cli.md +114 -0
  144. package/docs/reference/contracts.md +222 -0
  145. package/docs/reference/run-bundle-contract.md +354 -0
  146. package/docs/reference/run-lifecycle.md +54 -0
  147. package/examples/cursor-sdk/README.md +11 -0
  148. package/examples/cursor-sdk/capture-profile.json +1 -0
  149. package/examples/cursor-sdk/harness.json +1 -0
  150. package/examples/cursor-sdk/model.json +1 -0
  151. package/examples/cursor-sdk/native-limits.json +1 -0
  152. package/examples/cursor-sdk/native-tool-policy.json +1 -0
  153. package/ontology/behavior-categories.v1.json +46 -0
  154. package/package.json +66 -0
  155. package/release/0.1.0/KNOWN_LIMITATIONS.md +35 -0
  156. package/release/0.1.0/README.md +46 -0
  157. package/release/0.1.0/reproducibility.json +61 -0
  158. package/release/0.2.0/KNOWN_LIMITATIONS.md +40 -0
  159. package/release/0.2.0/README.md +42 -0
  160. package/release/0.2.0/reproducibility.json +73 -0
  161. package/release/0.2.1/KNOWN_LIMITATIONS.md +18 -0
  162. package/release/0.2.1/README.md +63 -0
  163. package/release/0.2.1/reproducibility.json +73 -0
  164. package/release/README.md +13 -0
  165. package/schemas/aggregation.v1.json +293 -0
  166. package/schemas/behavior-assertions.v1.json +220 -0
  167. package/schemas/deepseek-runtime-composition.v1.schema.json +122 -0
  168. package/schemas/experiment.v1.schema.json +188 -0
  169. package/schemas/human-calibration.v1.json +336 -0
  170. package/schemas/normalization-integrity.v1.json +302 -0
  171. package/schemas/run-bundles/v1.json +1040 -0
  172. package/schemas/run-queue.v1.schema.json +206 -0
  173. package/schemas/semantic-judge.v1.json +380 -0
  174. package/schemas/structural-observations.v1.json +131 -0
  175. package/schemas/task-packet-freeze.v1.schema.json +106 -0
  176. package/schemas/task-packet.v1.schema.json +234 -0
  177. package/schemas/uniform-events/v1.json +289 -0
  178. package/scripts/atlas-grafana.sh +18 -0
@@ -0,0 +1,128 @@
1
+ import { type AdapterCapability, type AdapterCapabilityProfile, type NativeEvidenceReference, type NativeEvidenceResolver, type NormalizationInput, type NormalizationResult, type UniformEventFamily } from "./uniform-events.js";
2
+ type DigestString = `sha256:${string}`;
3
+ export type NormalizedNativeRecord = {
4
+ reference: NativeEvidenceReference;
5
+ nativeType: string;
6
+ digest: DigestString;
7
+ };
8
+ export type NormalizedContentReference = {
9
+ reference: NativeEvidenceReference;
10
+ digest: DigestString;
11
+ };
12
+ export type NormalizedDataset = NormalizationResult & {
13
+ schemaVersion: "ebo.normalized-dataset/v1";
14
+ runId: string;
15
+ attemptId: string;
16
+ adapter: {
17
+ id: string;
18
+ version: string;
19
+ harness: string;
20
+ };
21
+ capabilityProfile: AdapterCapabilityProfile;
22
+ nativeRecords: readonly NormalizedNativeRecord[];
23
+ contentReferences: readonly NormalizedContentReference[];
24
+ };
25
+ export type AdapterCoverageReport = {
26
+ schemaVersion: "ebo.adapter-coverage-report/v1";
27
+ runId: string;
28
+ attemptId: string;
29
+ adapter: NormalizedDataset["adapter"];
30
+ records: {
31
+ total: number;
32
+ mapped: number;
33
+ unmapped: number;
34
+ };
35
+ nativeTypes: ReadonlyArray<{
36
+ nativeType: string;
37
+ total: number;
38
+ mapped: number;
39
+ unmapped: number;
40
+ }>;
41
+ families: Record<UniformEventFamily, {
42
+ capability: AdapterCapability;
43
+ observedEvents: number;
44
+ }>;
45
+ evidence: AdapterCapabilityProfile["evidence"];
46
+ };
47
+ export type NormalizedDatasetInput<NativeRecord> = {
48
+ capture: NormalizationInput<NativeRecord>;
49
+ normalization: NormalizationResult;
50
+ capabilityProfile: AdapterCapabilityProfile;
51
+ adapterVersion: string;
52
+ nativeType(record: NativeRecord): string;
53
+ contentDigest?(reference: NativeEvidenceReference): DigestString | undefined;
54
+ };
55
+ export type ComparisonCapability = `family:${UniformEventFamily}` | `evidence:${keyof AdapterCapabilityProfile["evidence"]}`;
56
+ export type ComparisonCandidate = {
57
+ id: string;
58
+ manifestDigest: DigestString;
59
+ adapterVersion: string;
60
+ task: {
61
+ id: string;
62
+ digest: DigestString;
63
+ };
64
+ fixture: {
65
+ id: string;
66
+ digest: DigestString;
67
+ };
68
+ model: {
69
+ id: string;
70
+ configurationDigest: DigestString;
71
+ };
72
+ harness: {
73
+ id: string;
74
+ version: string;
75
+ configurationDigest: DigestString;
76
+ };
77
+ assessmentMode: "observational" | "verified";
78
+ captureProfileDigest: DigestString;
79
+ budgetDigest: DigestString;
80
+ toolPolicyDigest: DigestString;
81
+ capabilityProfile: AdapterCapabilityProfile;
82
+ };
83
+ export type ComparisonRequest = {
84
+ schemaVersion: "ebo.comparison-request/v2";
85
+ measure: string;
86
+ left: ComparisonCandidate;
87
+ right: ComparisonCandidate;
88
+ policy: {
89
+ declaredDifferences: ReadonlyArray<"model" | "harness">;
90
+ requiredCapabilities: readonly ComparisonCapability[];
91
+ };
92
+ };
93
+ export type ComparisonReason = {
94
+ code: "task-mismatch" | "fixture-mismatch" | "material-configuration-mismatch" | "undeclared-model-difference" | "declared-model-difference" | "undeclared-harness-difference" | "declared-harness-difference" | "capability-unsupported" | "capability-partial";
95
+ dimension: string;
96
+ detail: string;
97
+ };
98
+ export type ComparisonReport = {
99
+ schemaVersion: "ebo.comparison-report/v2";
100
+ measure: string;
101
+ requestDigest: DigestString;
102
+ status: "supported" | "qualified-with-caveats" | "unsupported";
103
+ candidates: [string, string];
104
+ policy: ComparisonRequest["policy"];
105
+ reasons: readonly ComparisonReason[];
106
+ };
107
+ export type LegacyComparisonRequest = {
108
+ schemaVersion: "ebo.comparison-request/v1";
109
+ left: Omit<ComparisonCandidate, "manifestDigest">;
110
+ right: Omit<ComparisonCandidate, "manifestDigest">;
111
+ policy: ComparisonRequest["policy"];
112
+ };
113
+ export type LegacyComparisonReport = {
114
+ schemaVersion: "ebo.comparison-report/v1";
115
+ status: ComparisonReport["status"];
116
+ candidates: [string, string];
117
+ reasons: readonly ComparisonReason[];
118
+ };
119
+ export declare function describeNormalizedDataset<NativeRecord>(input: NormalizedDatasetInput<NativeRecord>): NormalizedDataset;
120
+ export declare function createCapturedNativeEvidenceResolver<NativeRecord>(capture: NormalizationInput<NativeRecord>, fallback?: NativeEvidenceResolver): NativeEvidenceResolver;
121
+ export declare function validateNormalizedDataset(dataset: NormalizedDataset, resolver: NativeEvidenceResolver): Promise<AdapterCoverageReport>;
122
+ export declare function validateNormalizedCorpus(entries: ReadonlyArray<{
123
+ dataset: NormalizedDataset;
124
+ resolver: NativeEvidenceResolver;
125
+ }>): Promise<readonly AdapterCoverageReport[]>;
126
+ export declare function assessComparisonEligibility(request: ComparisonRequest): ComparisonReport;
127
+ export declare function assessLegacyComparisonEligibility(request: LegacyComparisonRequest): LegacyComparisonReport;
128
+ export {};
@@ -0,0 +1,429 @@
1
+ import { canonicalizeMetadata, digestMetadata, validateArtifact } from "./artifacts.js";
2
+ import { UNIFORM_EVENT_FAMILIES, validateUniformEvents, } from "./uniform-events.js";
3
+ export function describeNormalizedDataset(input) {
4
+ if (!["qualified", "qualified-with-gaps"].includes(input.capture.qualification)) {
5
+ throw new Error("Normalized datasets require capture-qualified evidence.");
6
+ }
7
+ const nativeRecords = input.capture.records.map(({ reference, record }) => ({
8
+ reference: structuredClone(reference),
9
+ nativeType: required(input.nativeType(record), "Native record type"),
10
+ digest: digestString(digestMetadata(record)),
11
+ }));
12
+ const nativeDigests = new Map(nativeRecords.map((record) => [referenceKey(record.reference), record.digest]));
13
+ const contentReferences = new Map();
14
+ for (const event of input.normalization.events) {
15
+ if (event.content.status !== "known")
16
+ continue;
17
+ for (const { nativeReference: reference } of event.content.value) {
18
+ const key = referenceKey(reference);
19
+ const containing = containingReference(reference);
20
+ const digest = nativeDigests.get(key)
21
+ ?? (containing === undefined ? undefined : nativeDigests.get(referenceKey(containing)))
22
+ ?? input.contentDigest?.(reference);
23
+ if (digest === undefined)
24
+ throw new Error(`Content reference ${formatReference(reference)} requires a trusted digest.`);
25
+ const existing = contentReferences.get(key);
26
+ if (existing !== undefined && existing.digest !== digest) {
27
+ throw new Error(`Content reference ${formatReference(reference)} has conflicting digests.`);
28
+ }
29
+ contentReferences.set(key, { reference: structuredClone(reference), digest });
30
+ }
31
+ }
32
+ return {
33
+ schemaVersion: "ebo.normalized-dataset/v1",
34
+ runId: input.capture.runId,
35
+ attemptId: input.capture.attemptId,
36
+ adapter: {
37
+ id: input.capabilityProfile.adapterId,
38
+ version: required(input.adapterVersion, "Adapter version"),
39
+ harness: input.capabilityProfile.harness,
40
+ },
41
+ capabilityProfile: structuredClone(input.capabilityProfile),
42
+ nativeRecords,
43
+ contentReferences: [...contentReferences.values()],
44
+ events: structuredClone(input.normalization.events),
45
+ unmapped: structuredClone(input.normalization.unmapped),
46
+ };
47
+ }
48
+ export function createCapturedNativeEvidenceResolver(capture, fallback) {
49
+ const records = new Map(capture.records.map(({ reference, record }) => [
50
+ referenceKey(reference),
51
+ {
52
+ record,
53
+ resolution: {
54
+ runId: capture.runId,
55
+ attemptId: capture.attemptId,
56
+ digest: digestString(digestMetadata(record)),
57
+ },
58
+ },
59
+ ]));
60
+ return {
61
+ async resolve(reference) {
62
+ const exact = records.get(referenceKey(reference));
63
+ if (exact !== undefined)
64
+ return exact.resolution;
65
+ const fallbackResolution = await fallback?.resolve(reference);
66
+ if (fallbackResolution) {
67
+ if (typeof fallbackResolution !== "boolean") {
68
+ return fallbackResolution.runId === capture.runId && fallbackResolution.attemptId === capture.attemptId
69
+ ? fallbackResolution
70
+ : false;
71
+ }
72
+ const containing = containingNativeRecord(records, reference);
73
+ return containing !== undefined && resolvesJsonPointer(containing.record, reference.recordLocator)
74
+ ? containing.resolution
75
+ : false;
76
+ }
77
+ const containing = containingNativeRecord(records, reference);
78
+ if (containing !== undefined && resolvesJsonPointer(containing.record, reference.recordLocator)) {
79
+ return containing.resolution;
80
+ }
81
+ return false;
82
+ },
83
+ };
84
+ }
85
+ export async function validateNormalizedDataset(dataset, resolver) {
86
+ assertValidArtifact("normalized dataset", dataset);
87
+ if (dataset.adapter.id !== dataset.capabilityProfile.adapterId
88
+ || dataset.adapter.harness !== dataset.capabilityProfile.harness) {
89
+ throw new Error("Normalized dataset adapter identity does not match its capability profile.");
90
+ }
91
+ const recordByReference = new Map();
92
+ for (const record of dataset.nativeRecords) {
93
+ const key = referenceKey(record.reference);
94
+ if (recordByReference.has(key))
95
+ throw new Error("Normalized dataset contains duplicate native records.");
96
+ recordByReference.set(key, record);
97
+ const resolution = await resolver.resolve(record.reference);
98
+ if (resolution === false)
99
+ throw unresolved(record.reference);
100
+ if (typeof resolution === "boolean") {
101
+ throw new Error(`Native evidence reference ${formatReference(record.reference)} lacks run and digest integrity metadata.`);
102
+ }
103
+ assertResolution(dataset, record, resolution);
104
+ }
105
+ const contentByReference = new Map();
106
+ for (const content of dataset.contentReferences) {
107
+ const key = referenceKey(content.reference);
108
+ if (contentByReference.has(key))
109
+ throw new Error("Normalized dataset contains duplicate content references.");
110
+ contentByReference.set(key, content);
111
+ }
112
+ const usedContent = new Set();
113
+ for (const event of dataset.events) {
114
+ if (event.content.status !== "known")
115
+ continue;
116
+ for (const content of event.content.value) {
117
+ const key = referenceKey(content.nativeReference);
118
+ if (!contentByReference.has(key)) {
119
+ throw new Error(`Content reference ${formatReference(content.nativeReference)} has no expected digest.`);
120
+ }
121
+ usedContent.add(key);
122
+ }
123
+ }
124
+ if ([...contentByReference.keys()].some((key) => !usedContent.has(key))) {
125
+ throw new Error("Normalized dataset contains an unreferenced content digest.");
126
+ }
127
+ const ownedResolver = {
128
+ async resolve(reference) {
129
+ const resolution = await resolver.resolve(reference);
130
+ const expectedContent = contentByReference.get(referenceKey(reference));
131
+ return typeof resolution === "object"
132
+ && resolution.runId === dataset.runId
133
+ && resolution.attemptId === dataset.attemptId
134
+ && (expectedContent === undefined || resolution.digest === expectedContent.digest)
135
+ ? resolution
136
+ : false;
137
+ },
138
+ };
139
+ await validateUniformEvents(dataset.events, ownedResolver);
140
+ const mapped = new Set();
141
+ for (const event of dataset.events) {
142
+ if (event.runId !== dataset.runId || event.attemptId !== dataset.attemptId) {
143
+ throw new Error(`Uniform event "${event.id}" belongs to the wrong run or attempt.`);
144
+ }
145
+ if (event.source.harness !== dataset.adapter.harness) {
146
+ throw new Error(`Uniform event "${event.id}" belongs to the wrong harness.`);
147
+ }
148
+ const key = referenceKey(event.source.nativeReference);
149
+ const record = recordByReference.get(key);
150
+ if (record === undefined)
151
+ throw new Error(`Uniform event "${event.id}" does not identify a captured native record.`);
152
+ if (record.nativeType !== event.source.nativeType) {
153
+ throw new Error(`Uniform event "${event.id}" native type does not match its captured record.`);
154
+ }
155
+ if (!dataset.capabilityProfile.nativeTypes.includes(event.source.nativeType)) {
156
+ throw new Error(`Uniform event "${event.id}" uses undeclared native type "${event.source.nativeType}".`);
157
+ }
158
+ if (dataset.capabilityProfile.families[event.family].status === "unsupported") {
159
+ throw new Error(`Uniform event "${event.id}" uses unsupported event family "${event.family}".`);
160
+ }
161
+ assertEvidenceCapability(dataset.capabilityProfile.evidence.nativeOrder, event.nativeOrder.status, "nativeOrder");
162
+ assertEvidenceCapability(dataset.capabilityProfile.evidence.nativeTime, event.nativeTime.status, "nativeTime");
163
+ assertEvidenceCapability(dataset.capabilityProfile.evidence.parentage, event.relations.parent.status, "parentage");
164
+ assertEvidenceCapability(dataset.capabilityProfile.evidence.content, event.content.status, "content");
165
+ if (dataset.capabilityProfile.evidence.parentage.status === "unsupported" && event.relations.known.length > 0) {
166
+ throw new Error(`Uniform event "${event.id}" contradicts unsupported parentage.`);
167
+ }
168
+ mapped.add(key);
169
+ }
170
+ assertSourceLocalOrder(dataset);
171
+ const unmapped = new Set();
172
+ for (const entry of dataset.unmapped) {
173
+ const key = referenceKey(entry.reference);
174
+ if (unmapped.has(key))
175
+ throw new Error("Normalized dataset lists a native record as unmapped more than once.");
176
+ if (!recordByReference.has(key))
177
+ throw new Error("Normalized dataset lists an uncaptured native record as unmapped.");
178
+ unmapped.add(key);
179
+ }
180
+ for (const key of recordByReference.keys()) {
181
+ if (mapped.has(key) === unmapped.has(key)) {
182
+ throw new Error("Every native record must be mapped or explicitly retained as unmapped.");
183
+ }
184
+ }
185
+ return coverageReport(dataset, mapped, unmapped);
186
+ }
187
+ export async function validateNormalizedCorpus(entries) {
188
+ const attempts = new Set();
189
+ const events = new Set();
190
+ const reports = [];
191
+ for (const { dataset, resolver } of entries) {
192
+ if (attempts.has(dataset.attemptId))
193
+ throw new Error(`Normalized corpus contains duplicate attempt ID "${dataset.attemptId}".`);
194
+ attempts.add(dataset.attemptId);
195
+ for (const event of dataset.events) {
196
+ const key = JSON.stringify([dataset.attemptId, event.id]);
197
+ if (events.has(key))
198
+ throw new Error(`Normalized corpus contains duplicate event ID "${event.id}" in attempt "${dataset.attemptId}".`);
199
+ events.add(key);
200
+ }
201
+ reports.push(await validateNormalizedDataset(dataset, resolver));
202
+ }
203
+ return reports;
204
+ }
205
+ export function assessComparisonEligibility(request) {
206
+ assertValidArtifact("comparison request", request);
207
+ const blockers = [];
208
+ const caveats = [];
209
+ compareExact(request.left.task, request.right.task, "task", "task-mismatch", blockers);
210
+ compareExact(request.left.fixture, request.right.fixture, "fixture", "fixture-mismatch", blockers);
211
+ for (const [dimension, left, right] of [
212
+ ["assessment mode", request.left.assessmentMode, request.right.assessmentMode],
213
+ ["capture profile", request.left.captureProfileDigest, request.right.captureProfileDigest],
214
+ ["budget", request.left.budgetDigest, request.right.budgetDigest],
215
+ ["tool policy", request.left.toolPolicyDigest, request.right.toolPolicyDigest],
216
+ ]) {
217
+ if (left !== right)
218
+ blockers.push({
219
+ code: "material-configuration-mismatch",
220
+ dimension,
221
+ detail: `${dimension} differs between the proposed comparison candidates.`,
222
+ });
223
+ }
224
+ compareDeclaredCondition("model", request.left.model, request.right.model, request.policy.declaredDifferences, blockers, caveats);
225
+ compareDeclaredCondition("harness", request.left.harness, request.right.harness, request.policy.declaredDifferences, blockers, caveats);
226
+ if (request.left.harness.id === request.right.harness.id
227
+ && (request.left.capabilityProfile.adapterId !== request.right.capabilityProfile.adapterId
228
+ || request.left.adapterVersion !== request.right.adapterVersion)) {
229
+ blockers.push({
230
+ code: "material-configuration-mismatch",
231
+ dimension: "normalization adapter",
232
+ detail: "normalization adapter identity differs for the same harness condition.",
233
+ });
234
+ }
235
+ for (const capability of request.policy.requiredCapabilities) {
236
+ for (const candidate of [request.left, request.right]) {
237
+ const value = candidateCapability(candidate, capability);
238
+ if (value.status === "unsupported")
239
+ blockers.push({
240
+ code: "capability-unsupported",
241
+ dimension: capability,
242
+ detail: `${candidate.id} does not support required capability ${capability}.`,
243
+ });
244
+ else if (value.status === "partial")
245
+ caveats.push({
246
+ code: "capability-partial",
247
+ dimension: capability,
248
+ detail: `${candidate.id} only partially supports required capability ${capability}.`,
249
+ });
250
+ }
251
+ }
252
+ const reasons = [...blockers, ...caveats];
253
+ const report = {
254
+ schemaVersion: "ebo.comparison-report/v2",
255
+ measure: request.measure,
256
+ requestDigest: `sha256:${digestMetadata(request).value}`,
257
+ status: blockers.length > 0 ? "unsupported" : caveats.length > 0 ? "qualified-with-caveats" : "supported",
258
+ candidates: [request.left.id, request.right.id],
259
+ policy: structuredClone(request.policy),
260
+ reasons,
261
+ };
262
+ assertValidArtifact("comparison report", report);
263
+ return report;
264
+ }
265
+ export function assessLegacyComparisonEligibility(request) {
266
+ assertValidArtifact("comparison request", request);
267
+ const manifestDigest = `sha256:${"0".repeat(64)}`;
268
+ const report = assessComparisonEligibility({
269
+ ...structuredClone(request),
270
+ schemaVersion: "ebo.comparison-request/v2",
271
+ measure: "legacy-unspecified",
272
+ left: { ...structuredClone(request.left), manifestDigest },
273
+ right: { ...structuredClone(request.right), manifestDigest },
274
+ });
275
+ const legacy = {
276
+ schemaVersion: "ebo.comparison-report/v1",
277
+ status: report.status,
278
+ candidates: report.candidates,
279
+ reasons: report.reasons,
280
+ };
281
+ assertValidArtifact("comparison report", legacy);
282
+ return legacy;
283
+ }
284
+ function coverageReport(dataset, mapped, unmapped) {
285
+ const nativeTypes = new Map();
286
+ for (const record of dataset.nativeRecords) {
287
+ const counts = nativeTypes.get(record.nativeType) ?? { total: 0, mapped: 0, unmapped: 0 };
288
+ counts.total += 1;
289
+ if (mapped.has(referenceKey(record.reference)))
290
+ counts.mapped += 1;
291
+ else
292
+ counts.unmapped += 1;
293
+ nativeTypes.set(record.nativeType, counts);
294
+ }
295
+ const report = {
296
+ schemaVersion: "ebo.adapter-coverage-report/v1",
297
+ runId: dataset.runId,
298
+ attemptId: dataset.attemptId,
299
+ adapter: structuredClone(dataset.adapter),
300
+ records: { total: dataset.nativeRecords.length, mapped: mapped.size, unmapped: unmapped.size },
301
+ nativeTypes: [...nativeTypes].sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0)
302
+ .map(([nativeType, counts]) => ({ nativeType, ...counts })),
303
+ families: Object.fromEntries(UNIFORM_EVENT_FAMILIES.map((family) => [family, {
304
+ capability: structuredClone(dataset.capabilityProfile.families[family]),
305
+ observedEvents: dataset.events.filter((event) => event.family === family).length,
306
+ }])),
307
+ evidence: structuredClone(dataset.capabilityProfile.evidence),
308
+ };
309
+ assertValidArtifact("adapter coverage report", report);
310
+ return report;
311
+ }
312
+ function assertResolution(dataset, record, resolution) {
313
+ if (resolution.runId !== dataset.runId || resolution.attemptId !== dataset.attemptId) {
314
+ throw new Error(`Native evidence reference ${formatReference(record.reference)} resolves to the wrong run or attempt.`);
315
+ }
316
+ if (resolution.digest !== record.digest) {
317
+ throw new Error(`Native evidence reference ${formatReference(record.reference)} has a digest mismatch.`);
318
+ }
319
+ }
320
+ function assertSourceLocalOrder(dataset) {
321
+ const last = new Map();
322
+ for (const event of dataset.events) {
323
+ if (event.nativeOrder.status !== "known")
324
+ continue;
325
+ const previous = last.get(event.nativeOrder.domain);
326
+ if (previous !== undefined && event.nativeOrder.value < previous) {
327
+ throw new Error(`Uniform event "${event.id}" violates source-local order domain "${event.nativeOrder.domain}".`);
328
+ }
329
+ last.set(event.nativeOrder.domain, event.nativeOrder.value);
330
+ }
331
+ }
332
+ function compareExact(left, right, dimension, code, blockers) {
333
+ if (canonicalizeMetadata(left) !== canonicalizeMetadata(right))
334
+ blockers.push({
335
+ code,
336
+ dimension,
337
+ detail: `${dimension} identity or digest differs between the proposed comparison candidates.`,
338
+ });
339
+ }
340
+ function compareDeclaredCondition(dimension, left, right, declared, blockers, caveats) {
341
+ if (canonicalizeMetadata(left) === canonicalizeMetadata(right))
342
+ return;
343
+ if (!declared.includes(dimension)) {
344
+ blockers.push({
345
+ code: dimension === "model" ? "undeclared-model-difference" : "undeclared-harness-difference",
346
+ dimension,
347
+ detail: `${dimension} differs but is not declared by the comparison policy.`,
348
+ });
349
+ return;
350
+ }
351
+ caveats.push({
352
+ code: dimension === "model" ? "declared-model-difference" : "declared-harness-difference",
353
+ dimension,
354
+ detail: `${dimension} differs as declared; results are qualified and do not establish causality.`,
355
+ });
356
+ }
357
+ function candidateCapability(candidate, capability) {
358
+ const [kind, name] = capability.split(":");
359
+ return kind === "family"
360
+ ? candidate.capabilityProfile.families[name]
361
+ : candidate.capabilityProfile.evidence[name];
362
+ }
363
+ function assertEvidenceCapability(capability, evidenceStatus, field) {
364
+ if ((capability.status === "unsupported") !== (evidenceStatus === "unsupported")
365
+ && capability.status !== "partial") {
366
+ throw new Error(`Adapter evidence capability contradicts event ${field}.`);
367
+ }
368
+ }
369
+ function assertValidArtifact(label, value) {
370
+ const errors = validateArtifact(label, value);
371
+ if (errors.length > 0)
372
+ throw new Error(errors.map(({ field, message }) => `${label} ${field}: ${message}`).join("\n"));
373
+ }
374
+ function digestString(digest) {
375
+ return `${digest.algorithm}:${digest.value}`;
376
+ }
377
+ function required(value, label) {
378
+ if (value.trim() === "")
379
+ throw new Error(`${label} is required.`);
380
+ return value;
381
+ }
382
+ function referenceKey(reference) {
383
+ return JSON.stringify([reference.artifactId, reference.recordLocator]);
384
+ }
385
+ function containingNativeRecord(records, reference) {
386
+ const containing = containingReference(reference);
387
+ return containing === undefined ? undefined : records.get(referenceKey(containing));
388
+ }
389
+ function containingReference(reference) {
390
+ const pointer = reference.recordLocator.indexOf("#");
391
+ const base = pointer > 0 ? reference.recordLocator.slice(0, pointer)
392
+ : pointer === 0 && reference.recordLocator.startsWith("#/") ? "#" : undefined;
393
+ return base === undefined ? undefined : { ...reference, recordLocator: base };
394
+ }
395
+ function resolvesJsonPointer(document, locator) {
396
+ const hash = locator.indexOf("#");
397
+ if (hash < 0)
398
+ return false;
399
+ const pointer = locator.slice(hash + 1);
400
+ if (pointer === "")
401
+ return true;
402
+ if (!pointer.startsWith("/"))
403
+ return false;
404
+ let current = document;
405
+ for (const encoded of pointer.slice(1).split("/")) {
406
+ if (/~(?:[^01]|$)/u.test(encoded))
407
+ return false;
408
+ const key = encoded.replaceAll("~1", "/").replaceAll("~0", "~");
409
+ if (Array.isArray(current)) {
410
+ if (!/^(?:0|[1-9][0-9]*)$/u.test(key) || Number(key) >= current.length)
411
+ return false;
412
+ current = current.at(Number(key));
413
+ continue;
414
+ }
415
+ if (typeof current !== "object" || current === null)
416
+ return false;
417
+ const property = Object.getOwnPropertyDescriptor(current, key);
418
+ if (property === undefined || !("value" in property))
419
+ return false;
420
+ current = property.value;
421
+ }
422
+ return true;
423
+ }
424
+ function formatReference(reference) {
425
+ return `${reference.artifactId}:${reference.recordLocator}`;
426
+ }
427
+ function unresolved(reference) {
428
+ return new Error(`Native evidence reference ${formatReference(reference)} is unresolved.`);
429
+ }
@@ -0,0 +1,36 @@
1
+ import { type OpenHandsCapture, type OpenHandsCaptureRequest } from "./openhands.js";
2
+ import { type RunAttemptResult, type VerifierExecutionContext, type WorkspaceCoordinator } from "./lifecycle.js";
3
+ import { type CaptureQualificationReport, type CapturedWorkspaceOutcome, type RunBundleDefinition, type RunManifest } from "./run-bundles.js";
4
+ import type { NormalizationResult } from "./uniform-events.js";
5
+ import type { VerifierResult } from "./verifiers.js";
6
+ export type CaptureOpenHandsAgentServerVerifier = (context: VerifierExecutionContext, workspace: CapturedWorkspaceOutcome, workspacePath: string) => VerifierResult | Promise<VerifierResult>;
7
+ export type OpenHandsAgentServerRunConfiguration = Omit<OpenHandsCaptureRequest, "runId" | "attemptId" | "startConversation" | "signal" | "onRecord"> & {
8
+ model: string;
9
+ startConversation: Record<string, unknown>;
10
+ serverWorkspacePath?: string;
11
+ };
12
+ export type CaptureOpenHandsAgentServerRunOptions = {
13
+ definition: RunBundleDefinition;
14
+ startingWorkspacePath: string;
15
+ workspace: WorkspaceCoordinator;
16
+ configuration: OpenHandsAgentServerRunConfiguration;
17
+ verifier?: CaptureOpenHandsAgentServerVerifier;
18
+ workspaceOutcomeExcludedDirectoryNames?: readonly string[];
19
+ workspaceOutcomeRespectsGitignore?: boolean;
20
+ workspaceOutcomeOmitsEmptyDirectories?: boolean;
21
+ signal?: AbortSignal;
22
+ maxWallClockMs?: number;
23
+ harnessBudgetMs?: number;
24
+ shutdownGraceMs?: number;
25
+ };
26
+ export type CaptureOpenHandsAgentServerRunResult = {
27
+ attempt: RunAttemptResult;
28
+ manifest: RunManifest;
29
+ qualification: CaptureQualificationReport;
30
+ capture?: OpenHandsCapture;
31
+ normalized?: NormalizationResult;
32
+ normalizationError?: string;
33
+ retainedWorkspacePath?: string;
34
+ };
35
+ /** Execute and retain one caller-configured OpenHands Agent Server attempt. */
36
+ export declare function captureOpenHandsAgentServerRun(options: CaptureOpenHandsAgentServerRunOptions): Promise<CaptureOpenHandsAgentServerRunResult>;