engineering-behavior-observatory 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +151 -0
  3. package/contracts/codex-app-server-0.150.1/manifest.json +25 -0
  4. package/contracts/codex-app-server-0.150.1/schema/InitializeParams.json +84 -0
  5. package/contracts/codex-app-server-0.150.1/schema/JSONRPCMessage.json +137 -0
  6. package/contracts/codex-app-server-0.150.1/schema/v2/ThreadReadParams.json +17 -0
  7. package/contracts/codex-app-server-0.150.1/schema/v2/ThreadTokenUsageUpdatedNotification.json +82 -0
  8. package/contracts/codex-app-server-0.150.1/schema/v2/TurnInterruptParams.json +17 -0
  9. package/contracts/codex-app-server-0.150.1/types/AskForApproval.ts +5 -0
  10. package/contracts/codex-app-server-0.150.1/types/ClientNotification.ts +5 -0
  11. package/contracts/codex-app-server-0.150.1/types/SandboxMode.ts +5 -0
  12. package/contracts/codex-app-server-0.150.1/types/ThreadReadParams.ts +9 -0
  13. package/contracts/codex-app-server-0.150.1/types/TokenUsageBreakdown.ts +5 -0
  14. package/contracts/codex-app-server-0.150.1/types/TurnInterruptParams.ts +5 -0
  15. package/contracts/codex-app-server-0.153.4/manifest.json +27 -0
  16. package/contracts/codex-app-server-0.153.4/schema/JSONRPCMessage.json +137 -0
  17. package/contracts/codex-app-server-0.153.4/schema/v1/InitializeParams.json +84 -0
  18. package/contracts/codex-app-server-0.153.4/schema/v2/ThreadReadParams.json +17 -0
  19. package/contracts/codex-app-server-0.153.4/schema/v2/ThreadStartParams.json +515 -0
  20. package/contracts/codex-app-server-0.153.4/schema/v2/ThreadTokenUsageUpdatedNotification.json +82 -0
  21. package/contracts/codex-app-server-0.153.4/schema/v2/TurnInterruptParams.json +17 -0
  22. package/contracts/codex-app-server-0.153.4/schema/v2/TurnStartParams.json +921 -0
  23. package/contracts/codex-app-server-0.153.4/types/AskForApproval.ts +5 -0
  24. package/contracts/codex-app-server-0.153.4/types/ClientNotification.ts +5 -0
  25. package/contracts/codex-app-server-0.153.4/types/SandboxMode.ts +5 -0
  26. package/contracts/codex-app-server-0.153.4/types/ThreadReadParams.ts +12 -0
  27. package/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.ts +5 -0
  28. package/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.ts +5 -0
  29. package/contracts/openhands-agent-server-v1.44.1.json +65 -0
  30. package/contracts/openhands-agent-server-v1.46.0.json +46 -0
  31. package/dist/contracts/codex-app-server-0.153.4/types/AskForApproval.d.ts +9 -0
  32. package/dist/contracts/codex-app-server-0.153.4/types/AskForApproval.js +2 -0
  33. package/dist/contracts/codex-app-server-0.153.4/types/ClientNotification.d.ts +3 -0
  34. package/dist/contracts/codex-app-server-0.153.4/types/ClientNotification.js +2 -0
  35. package/dist/contracts/codex-app-server-0.153.4/types/SandboxMode.d.ts +1 -0
  36. package/dist/contracts/codex-app-server-0.153.4/types/SandboxMode.js +2 -0
  37. package/dist/contracts/codex-app-server-0.153.4/types/ThreadReadParams.d.ts +10 -0
  38. package/dist/contracts/codex-app-server-0.153.4/types/ThreadReadParams.js +2 -0
  39. package/dist/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.d.ts +8 -0
  40. package/dist/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.js +2 -0
  41. package/dist/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.d.ts +4 -0
  42. package/dist/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.js +2 -0
  43. package/dist/src/agent-sdk-normalizer.d.ts +22 -0
  44. package/dist/src/agent-sdk-normalizer.js +995 -0
  45. package/dist/src/agent-sdk-run.d.ts +31 -0
  46. package/dist/src/agent-sdk-run.js +242 -0
  47. package/dist/src/agent-sdk-runner.d.ts +96 -0
  48. package/dist/src/agent-sdk-runner.js +407 -0
  49. package/dist/src/agent-sdk.d.ts +294 -0
  50. package/dist/src/agent-sdk.js +644 -0
  51. package/dist/src/aggregation.d.ts +183 -0
  52. package/dist/src/aggregation.js +513 -0
  53. package/dist/src/artifacts.d.ts +36 -0
  54. package/dist/src/artifacts.js +1372 -0
  55. package/dist/src/atlas-grafana.d.ts +377 -0
  56. package/dist/src/atlas-grafana.js +47 -0
  57. package/dist/src/atlas-html.d.ts +2 -0
  58. package/dist/src/atlas-html.js +34 -0
  59. package/dist/src/atlas.d.ts +130 -0
  60. package/dist/src/atlas.js +235 -0
  61. package/dist/src/behavior-assertions.d.ts +95 -0
  62. package/dist/src/behavior-assertions.js +154 -0
  63. package/dist/src/cli.d.ts +5 -0
  64. package/dist/src/cli.js +957 -0
  65. package/dist/src/codex-judge.d.ts +44 -0
  66. package/dist/src/codex-judge.js +214 -0
  67. package/dist/src/codex-run.d.ts +114 -0
  68. package/dist/src/codex-run.js +519 -0
  69. package/dist/src/codex.d.ts +187 -0
  70. package/dist/src/codex.js +1028 -0
  71. package/dist/src/contracts.d.ts +155 -0
  72. package/dist/src/contracts.js +866 -0
  73. package/dist/src/corpus.d.ts +67 -0
  74. package/dist/src/corpus.js +540 -0
  75. package/dist/src/cursor-sdk-runner.d.ts +81 -0
  76. package/dist/src/cursor-sdk-runner.js +362 -0
  77. package/dist/src/cursor-sdk.d.ts +81 -0
  78. package/dist/src/cursor-sdk.js +1078 -0
  79. package/dist/src/deepseek-adapter.d.ts +152 -0
  80. package/dist/src/deepseek-adapter.js +777 -0
  81. package/dist/src/exports.d.ts +69 -0
  82. package/dist/src/exports.js +966 -0
  83. package/dist/src/human-calibration.d.ts +167 -0
  84. package/dist/src/human-calibration.js +618 -0
  85. package/dist/src/index.d.ts +71 -0
  86. package/dist/src/index.js +38 -0
  87. package/dist/src/lifecycle.d.ts +196 -0
  88. package/dist/src/lifecycle.js +1889 -0
  89. package/dist/src/normalization-integrity.d.ts +128 -0
  90. package/dist/src/normalization-integrity.js +429 -0
  91. package/dist/src/openhands-run.d.ts +36 -0
  92. package/dist/src/openhands-run.js +415 -0
  93. package/dist/src/openhands.d.ts +130 -0
  94. package/dist/src/openhands.js +859 -0
  95. package/dist/src/pi.d.ts +203 -0
  96. package/dist/src/pi.js +1345 -0
  97. package/dist/src/process-protocol.d.ts +245 -0
  98. package/dist/src/process-protocol.js +1261 -0
  99. package/dist/src/retained-evidence.d.ts +12 -0
  100. package/dist/src/retained-evidence.js +201 -0
  101. package/dist/src/run-bundles.d.ts +193 -0
  102. package/dist/src/run-bundles.js +993 -0
  103. package/dist/src/scheduler.d.ts +109 -0
  104. package/dist/src/scheduler.js +1080 -0
  105. package/dist/src/semantic-judge.d.ts +194 -0
  106. package/dist/src/semantic-judge.js +875 -0
  107. package/dist/src/structural-observations.d.ts +124 -0
  108. package/dist/src/structural-observations.js +671 -0
  109. package/dist/src/task-packets.d.ts +115 -0
  110. package/dist/src/task-packets.js +683 -0
  111. package/dist/src/uniform-events.d.ts +120 -0
  112. package/dist/src/uniform-events.js +158 -0
  113. package/dist/src/verifiers.d.ts +116 -0
  114. package/dist/src/verifiers.js +819 -0
  115. package/dist/src/workspaces.d.ts +56 -0
  116. package/dist/src/workspaces.js +1314 -0
  117. package/docs/README.md +60 -0
  118. package/docs/development/README.md +71 -0
  119. package/docs/development/documentation-sync.md +58 -0
  120. package/docs/development/extension-contracts.md +307 -0
  121. package/docs/evaluation/README.md +48 -0
  122. package/docs/evaluation/aggregation.md +112 -0
  123. package/docs/evaluation/behavior-assertions.md +45 -0
  124. package/docs/evaluation/human-calibration.md +170 -0
  125. package/docs/evaluation/normalization-integrity.md +80 -0
  126. package/docs/evaluation/semantic-judge.md +188 -0
  127. package/docs/evaluation/structural-observations.md +84 -0
  128. package/docs/evaluation/uniform-events.md +131 -0
  129. package/docs/guides/atlas.md +144 -0
  130. package/docs/guides/evidence-and-sharing.md +100 -0
  131. package/docs/guides/operator-guide.md +424 -0
  132. package/docs/guides/quickstart.md +152 -0
  133. package/docs/guides/telemetry.md +66 -0
  134. package/docs/harnesses/README.md +31 -0
  135. package/docs/harnesses/claude-agent-sdk.md +61 -0
  136. package/docs/harnesses/codex-harness.md +174 -0
  137. package/docs/harnesses/cursor-sdk.md +149 -0
  138. package/docs/harnesses/deepseek-harness.md +134 -0
  139. package/docs/harnesses/openhands-agent-server.md +153 -0
  140. package/docs/harnesses/pi-sdk.md +66 -0
  141. package/docs/reference/README.md +19 -0
  142. package/docs/reference/agent-sdk-operational-runner.md +118 -0
  143. package/docs/reference/cli.md +114 -0
  144. package/docs/reference/contracts.md +222 -0
  145. package/docs/reference/run-bundle-contract.md +354 -0
  146. package/docs/reference/run-lifecycle.md +54 -0
  147. package/examples/cursor-sdk/README.md +11 -0
  148. package/examples/cursor-sdk/capture-profile.json +1 -0
  149. package/examples/cursor-sdk/harness.json +1 -0
  150. package/examples/cursor-sdk/model.json +1 -0
  151. package/examples/cursor-sdk/native-limits.json +1 -0
  152. package/examples/cursor-sdk/native-tool-policy.json +1 -0
  153. package/ontology/behavior-categories.v1.json +46 -0
  154. package/package.json +66 -0
  155. package/release/0.1.0/KNOWN_LIMITATIONS.md +35 -0
  156. package/release/0.1.0/README.md +46 -0
  157. package/release/0.1.0/reproducibility.json +61 -0
  158. package/release/0.2.0/KNOWN_LIMITATIONS.md +40 -0
  159. package/release/0.2.0/README.md +42 -0
  160. package/release/0.2.0/reproducibility.json +73 -0
  161. package/release/0.2.1/KNOWN_LIMITATIONS.md +18 -0
  162. package/release/0.2.1/README.md +63 -0
  163. package/release/0.2.1/reproducibility.json +73 -0
  164. package/release/README.md +13 -0
  165. package/schemas/aggregation.v1.json +293 -0
  166. package/schemas/behavior-assertions.v1.json +220 -0
  167. package/schemas/deepseek-runtime-composition.v1.schema.json +122 -0
  168. package/schemas/experiment.v1.schema.json +188 -0
  169. package/schemas/human-calibration.v1.json +336 -0
  170. package/schemas/normalization-integrity.v1.json +302 -0
  171. package/schemas/run-bundles/v1.json +1040 -0
  172. package/schemas/run-queue.v1.schema.json +206 -0
  173. package/schemas/semantic-judge.v1.json +380 -0
  174. package/schemas/structural-observations.v1.json +131 -0
  175. package/schemas/task-packet-freeze.v1.schema.json +106 -0
  176. package/schemas/task-packet.v1.schema.json +234 -0
  177. package/schemas/uniform-events/v1.json +289 -0
  178. package/scripts/atlas-grafana.sh +18 -0
@@ -0,0 +1,45 @@
1
+ # Behavior assertions and evidence resolution
2
+
3
+ `ebo.behavior-assertion/v1` records one semantic assessment or abstention for
4
+ one declared behavior dimension. The initial `ebo.behavior-vocabulary/v1` is a
5
+ small data file at
6
+ [`ontology/behavior-categories.v1.json`](../../ontology/behavior-categories.v1.json).
7
+ Its eight entries are the categories in the canonical specification; unmapped
8
+ observations do not force a new category.
9
+
10
+ Each assertion binds the normalized-dataset schema version and canonical
11
+ digest, run and attempt IDs, vocabulary version, one category/dimension pair,
12
+ and evaluator/rubric identities. An assessment is constructive, adverse,
13
+ mixed, or context-dependent and retains confidence on the explicit
14
+ `evaluator-reported-0-to-1` scale, rationale, an alternative explanation, and
15
+ at least one citation. Confidence is not a calibrated probability. An abstention instead retains its reason and
16
+ may name the missing event or evidence capability; citations are optional.
17
+
18
+ A citation names both a normalized event ID and that event's exact native
19
+ source reference. `validateBehaviorAssertion` first applies the existing
20
+ normalized-dataset integrity validator, then checks the dataset digest and
21
+ assertion identity, declared dimension, event ownership, native reference, and
22
+ resolver-provided run/attempt/digest. Changed datasets, foreign attempts,
23
+ dangling events or native records, and stale digests fail closed.
24
+
25
+ `ebo.behavior-review/v1` is a separate record bound to the assertion digest.
26
+ Its states are `proposed`, `confirmed`, `disputed`, `rejected`, and
27
+ `insufficient-evidence`. Proposed records cannot carry a reviewer; all other
28
+ states require an explicitly identified human reviewer and rationale. The
29
+ library derives confirmed aggregation eligibility only after revalidating the
30
+ assertion against its dataset and native resolver, and then finding a valid
31
+ `confirmed` review of a non-abstaining assertion. A judge assertion cannot set
32
+ human confirmation itself.
33
+
34
+ The callable validator accepts any `ebo.normalized-dataset/v1` and its
35
+ source-specific native resolver. The current CLI rebuilds and validates the
36
+ normalized dataset from a retained Claude Agent SDK, Codex, OpenHands, Pi, or
37
+ DeepSeek run bundle through `validateRetainedBehaviorAssertion`:
38
+
39
+ ```sh
40
+ ebo assertions validate <run-bundle-root> <assertion.json> [review.json]
41
+ ```
42
+
43
+ This command validates contracts and evidence only. It does not run a judge,
44
+ change review state, adjudicate a dispute, aggregate results, or build an
45
+ Atlas.
@@ -0,0 +1,170 @@
1
+ # Human calibration and adjudication
2
+
3
+ EBO supplies a local, static workflow for people to review evidence-grounded
4
+ behavior assertions. It does not assign reviewers, authenticate users, host a
5
+ review service, or treat synthetic fixture decisions as research labels.
6
+
7
+ ## Select a reproducible sample
8
+
9
+ Create a source file that points to capture-qualified Claude Agent SDK, Codex,
10
+ OpenHands, DeepSeek, Pi, or Cursor bundles and
11
+ their judge-produced assertions. Paths are local operator inputs; native
12
+ evidence is revalidated through the existing assertion resolver before a
13
+ candidate can enter a sample.
14
+
15
+ ```json
16
+ {
17
+ "schemaVersion": "ebo.review-source-set/v1",
18
+ "sources": [
19
+ {
20
+ "bundleRoot": "/local/restricted/runs/run-1",
21
+ "assertionPath": "/local/restricted/judgments/run-1/assertion.json",
22
+ "taskContext": "Reviewable task prompt or bounded operator-authored task synopsis."
23
+ }
24
+ ]
25
+ }
26
+ ```
27
+
28
+ Sampling criteria use explicit, non-overlapping strata. Each stratum has its
29
+ own requested size and may filter by task, model, harness, verifier outcome,
30
+ terminal state, behavior category, confidence, or judge abstention. A single
31
+ stratum with empty filters is an unstratified sample.
32
+
33
+ ```json
34
+ {
35
+ "schemaVersion": "ebo.review-sample-criteria/v1",
36
+ "seed": "study-2026-09-wave-1",
37
+ "strata": [
38
+ {
39
+ "id": "high-confidence-verification",
40
+ "sampleSize": 20,
41
+ "filters": {
42
+ "categoryIds": ["verification-completion"],
43
+ "confidence": { "min": 0.75 },
44
+ "abstentions": [false]
45
+ }
46
+ },
47
+ {
48
+ "id": "judge-abstentions",
49
+ "sampleSize": 10,
50
+ "filters": { "abstentions": [true] }
51
+ }
52
+ ]
53
+ }
54
+ ```
55
+
56
+ ```sh
57
+ ebo calibration sample sources.json criteria.json selection.json
58
+ ```
59
+
60
+ The saved selection retains the criteria and seed, every source root, source and
61
+ eligible population sizes, eligible and selected assertion IDs, per-stratum requested,
62
+ eligible, and selected counts, and any zero-candidate strata. Ordering is a
63
+ SHA-256 function of the seed, stratum, assertion ID, and assertion digest.
64
+ Repeated request-derived assertion IDs across runs remain distinct through
65
+ their digest-bound identity. Where a CLI selector is ambiguous, append the
66
+ digest as `<assertion-id>@sha256:<digest>`.
67
+ Every command that reuses a saved selection deterministically revalidates its
68
+ criteria, eligible population, strata, seeded ordering, and candidate bindings
69
+ against the retained source evidence.
70
+ Observational runs always record verifier outcome as `unavailable`; normal
71
+ completion is never substituted for verifier success.
72
+
73
+ ## Generate and inspect a static packet
74
+
75
+ ```sh
76
+ ebo calibration packet selection.json review-packet
77
+ open review-packet/index.html
78
+ ebo calibration inspect \
79
+ review-packet/packet.json <assertion-id> [event-id]
80
+ ```
81
+
82
+ `index.html` contains the supplied task context, condition context, assertion,
83
+ rationale, alternative explanation, and links to local pages that render the
84
+ exact cited native records. Each evidence page also links to its source artifact
85
+ and names the native locator. All task, assertion, and evidence content is
86
+ HTML-escaped. Links remain relative to the packet and its declared local bundle
87
+ root. The packet copies only cited records, keeps them `restricted-local-only`
88
+ with mode `0600`, and keeps those heavyweight records out of `packet.json`.
89
+ It is not a partner or public export. Moving the packet
90
+ separately from its source bundles breaks the original-artifact links by design.
91
+ Packet files are staged together and the completed directory is published in
92
+ one rename, so `packet.json` never names partially rendered evidence pages.
93
+ All selection, packet, history, and summary destinations are rejected when they
94
+ would be written inside a source run bundle.
95
+
96
+ ## Import human decisions
97
+
98
+ Get the exact assertion and current history bindings without manufacturing a
99
+ decision:
100
+
101
+ ```sh
102
+ ebo calibration binding selection.json <assertion-id>
103
+ ebo calibration binding selection.json <assertion-id> history.json
104
+ ```
105
+
106
+ A person supplies the reviewer identity, timestamp, decision, and rationale.
107
+ `insufficient-evidence` is the human abstention state. The first decision uses
108
+ `previousHistory: null`; later decisions use the binding printed for the
109
+ current history.
110
+
111
+ ```json
112
+ {
113
+ "schemaVersion": "ebo.human-review-decision/v1",
114
+ "id": "review-001",
115
+ "kind": "review",
116
+ "assertion": {
117
+ "id": "<assertion-id>",
118
+ "schemaVersion": "ebo.behavior-assertion/v1",
119
+ "digest": "sha256:<assertion-digest>"
120
+ },
121
+ "reviewer": { "kind": "human", "id": "<reviewer-id>" },
122
+ "decidedAt": "2026-09-07T12:00:00Z",
123
+ "state": "confirmed",
124
+ "rationale": "<human-authored rationale>",
125
+ "previousHistory": null
126
+ }
127
+ ```
128
+
129
+ ```sh
130
+ ebo calibration import selection.json history.json decision.json
131
+ ```
132
+
133
+ Import appends to one history and never rewrites an earlier decision. It rejects
134
+ unknown or changed assertions, stale history bindings, invalid adjudication
135
+ targets, and reuse of a decision ID with different content. Reimporting the
136
+ exact same decision is idempotent.
137
+ Imports serialize through a per-history portable filesystem lock with a stale
138
+ timeout and heartbeat. Interrupted imports are recovered automatically; an
139
+ active import makes a concurrent command fail closed for an idempotent retry.
140
+
141
+ An adjudication is another explicit human decision. It names at least two prior
142
+ review decisions for the same assertion:
143
+
144
+ ```sh
145
+ ebo calibration adjudicate \
146
+ selection.json history.json adjudication.json
147
+ ```
148
+
149
+ Its JSON uses `kind: "adjudication"` and an `adjudicates` array of prior review
150
+ decision IDs. This records lineage; it does not infer or automate adjudication.
151
+
152
+ ## Summarize calibration
153
+
154
+ ```sh
155
+ ebo calibration summarize \
156
+ selection.json history.json calibration-summary.json
157
+ ```
158
+
159
+ The summary reports totals and category-level counts. Judge-human agreement is
160
+ over latest judge-human decision pairs on non-abstaining assertions.
161
+ Human-human agreement is over distinct-reviewer pairs on the same assertion,
162
+ excluding human abstentions. Adjudication counts use human adjudication
163
+ decisions as their separate denominator. Every agreement result names its
164
+ population and denominator; with no comparable reviews it is `unavailable`
165
+ with denominator zero, never perfect agreement. Disputed, inconsistent,
166
+ unreviewed, insufficient-evidence, and judge-abstained assertions are excluded
167
+ from confirmed eligibility. A review appended after adjudication makes that
168
+ adjudication non-current until the updated review population is adjudicated.
169
+ These are calibration summaries, not comparison
170
+ or Behavior Atlas aggregates.
@@ -0,0 +1,80 @@
1
+ # Normalization integrity and comparison gates
2
+
3
+ Normalization is accepted only when every projected event remains bound to one
4
+ capture-qualified run and attempt. The native record stays authoritative; the
5
+ normalized dataset retains only its source reference, native type, and digest.
6
+
7
+ ## Dataset validation and coverage
8
+
9
+ `describeNormalizedDataset` combines a qualified capture, its normalization
10
+ result, the adapter capability profile, and the pinned adapter version into an
11
+ `ebo.normalized-dataset/v1` record. Source and content references retain
12
+ separate expected digests without copying native records. It rejects input that
13
+ is not explicitly `qualified` or `qualified-with-gaps`.
14
+ `validateNormalizedDataset` then checks:
15
+
16
+ - the dataset, event, and capability-profile schemas;
17
+ - stable run, attempt, adapter, harness, and native-type identity;
18
+ - resolver-provided run/attempt ownership and every source/content digest;
19
+ - source and content references, event relations, and acyclic parentage;
20
+ - nondecreasing order within each native order domain;
21
+ - that every native record is mapped or explicitly retained as unmapped; and
22
+ - that emitted families and evidence do not contradict adapter capabilities.
23
+
24
+ The resolver must return `{ runId, attemptId, digest }` for every source and
25
+ content reference. A boolean resolver remains supported by the lower-level
26
+ uniform event validator, but it is insufficient for dataset integrity
27
+ validation. The dataset validator rejects resolution metadata owned by another
28
+ run or attempt.
29
+ `createCapturedNativeEvidenceResolver` supplies the stronger metadata for an
30
+ in-memory qualified capture and can delegate other content references to an
31
+ adapter-specific resolver. Derived JSON Pointer locators are resolved against
32
+ the source record; the helper never treats a containing record as proof that an
33
+ arbitrary child locator exists.
34
+
35
+ Successful validation returns an `ebo.adapter-coverage-report/v1` report. It
36
+ counts mapped and unmapped native records by adapter version and native type.
37
+ For each uniform family it reports both the declared capability status and the
38
+ observed event count. Consequently, an unsupported family with zero events is
39
+ not confused with an available family that happened to have zero observations.
40
+ Unknown source records remain present in the native-type table and in the
41
+ dataset's `unmapped` list.
42
+
43
+ DeepSeek JSON-RPC methods and OpenHands REST/WebSocket record kinds remain
44
+ native types in these reports. They do not create event families or imply a
45
+ shared control protocol.
46
+
47
+ ## Comparison eligibility
48
+
49
+ The comparison gate consumes one explicit `ebo.comparison-request/v1` or
50
+ `ebo.comparison-request/v2` artifact. V2 additionally retains the exact measure
51
+ and candidate manifest digests required by aggregation:
52
+
53
+ ```sh
54
+ ebo comparison check <request.json>
55
+ ```
56
+
57
+ The request names both candidates, their task and fixture digests, model and
58
+ harness configuration identities, material capture/budget/tool-policy digests,
59
+ normalization-adapter identities and pinned versions, capability profiles, required capabilities,
60
+ and any declared model or harness difference. The command prints an
61
+ matching versioned `ebo.comparison-report/v1` or
62
+ `ebo.comparison-report/v2` JSON object and
63
+ returns nonzero when its status is `unsupported`.
64
+
65
+ Statuses are:
66
+
67
+ - `supported`: conditions match and required capabilities are available;
68
+ - `qualified-with-caveats`: a model/harness difference was declared or a
69
+ required capability is partial; and
70
+ - `unsupported`: task, fixture, material configuration, or undeclared
71
+ model/harness conditions differ, or a required capability is unsupported.
72
+
73
+ Declaring a model or harness difference makes that difference inspectable; it
74
+ does not claim identical conditions or establish that the harness caused an
75
+ observed behavioral difference.
76
+
77
+ The schema is
78
+ [`schemas/normalization-integrity.v1.json`](../../schemas/normalization-integrity.v1.json).
79
+ Exact-match and incompatible-fixture examples live under
80
+ `test/fixtures/comparison` in the source repository.
@@ -0,0 +1,188 @@
1
+ # Evidence-grounded semantic judge
2
+
3
+ `ebo judge run` evaluates one declared behavior dimension against a bounded,
4
+ caller-selected projection of a qualified Claude Agent SDK, Codex, OpenHands, Pi,
5
+ Cursor, or DeepSeek retained run bundle and an
6
+ `ebo.structural-observation-set/v1` file:
7
+
8
+ ```sh
9
+ ebo judge run \
10
+ <run-bundle-root> <observations.json> <request.json> <output-root>
11
+ ```
12
+
13
+ The output root must not exist and must be outside the immutable run bundle.
14
+ Every rerun therefore creates a new record. Runs with the same rubric,
15
+ selection, limits, and evidence share an input digest even when their output
16
+ roots or judge identities differ.
17
+
18
+ ## Request
19
+
20
+ The request is caller-owned configuration; EBO does not select a study model
21
+ or retry a judgment:
22
+
23
+ ```json
24
+ {
25
+ "schemaVersion": "ebo.semantic-judge-request/v1",
26
+ "id": "verification-run-001",
27
+ "behavior": {
28
+ "vocabularyVersion": "1.0.0",
29
+ "categoryId": "verification-completion",
30
+ "dimensionId": "verification-completion"
31
+ },
32
+ "rubric": {
33
+ "id": "verification-rubric",
34
+ "version": "1.0.0",
35
+ "instructions": "Assess whether retained evidence shows validation before completion."
36
+ },
37
+ "evaluator": {
38
+ "provider": "anthropic",
39
+ "model": "<existing authenticated Claude Agent SDK route>",
40
+ "effort": "low"
41
+ },
42
+ "selection": {
43
+ "eventIds": ["<normalized-event-id>"],
44
+ "structuralObservationIds": ["<structural-observation-id>"],
45
+ "includeOutcomeObservations": true
46
+ },
47
+ "limits": {
48
+ "maxEvidenceItems": 16,
49
+ "maxRecordChars": 4096,
50
+ "maxInputChars": 64000,
51
+ "maxOutputChars": 8192,
52
+ "maxCitations": 8,
53
+ "maxWallClockMs": 30000,
54
+ "maxTurns": 1,
55
+ "maxBudgetUsd": 0.1
56
+ },
57
+ "blinding": { "evaluatedModelIdentity": "redact" }
58
+ }
59
+ ```
60
+
61
+ Selection is exact. Unknown IDs fail before execution. Optional outcome
62
+ observations are added only when requested, and the total must remain within
63
+ `maxEvidenceItems`. Per-record truncation and whole-input omission are recorded
64
+ in `input.json`; omitted event IDs cannot be cited. Exact evaluated-model
65
+ strings are redacted from evidence values and keys when requested. Citation
66
+ IDs and native references are never rewritten, and the input records the
67
+ remaining harness, native-type, behavioral, and citation clues that may still
68
+ reveal origin.
69
+
70
+ Selected structural observations automatically include their normalized source
71
+ events so their claims remain citable. An explicitly selected observation with
72
+ no normalized source event, including an exact zero fact, is rejected before
73
+ execution; the same kind of automatically included outcome is recorded as
74
+ omitted instead of being shown to the judge as uncitable support.
75
+
76
+ ## Backend and trust boundary
77
+
78
+ The default backend is the installed TypeScript Claude Agent SDK. The caller
79
+ supplies model, effort, wall-clock, turn, output, citation, and optional cost
80
+ limits. The runner passes no tools, settings sources, skills, plugins, MCP
81
+ servers, additional directories, or persistent session. It uses an empty
82
+ temporary working directory, strict empty MCP configuration, `dontAsk`
83
+ permissions, and a custom system prompt that treats the delimited trajectory
84
+ payload as untrusted data rather than instructions.
85
+
86
+ The SDK subprocess inherits the parent environment for authentication and
87
+ routing, but EBO removes ambient model and effort override variables before
88
+ launch so the caller request remains authoritative. The retained evaluator
89
+ metadata records that policy without retaining secret values.
90
+ Ambient OpenTelemetry and Claude telemetry/tracing controls are also removed;
91
+ restricted rubric and trajectory content is never sent to a parent-configured
92
+ collector, and a console exporter cannot corrupt the SDK protocol channel.
93
+
94
+ For the native Codex backend, set `evaluator.backend` to `codex-app-server`,
95
+ `provider` to `openai`, and supply `model` and `effort`. The optional
96
+ `executable` selects the installed `codex` executable (default: PATH). Relative
97
+ paths resolve against the caller's working directory before isolation; its version
98
+ must be `0.153.4`. Set `maxTurns` to `1` and omit `maxBudgetUsd`: this backend
99
+ does not support USD budget enforcement. No automatic provider fallback occurs.
100
+ Backend selection is independent of the evaluated harness.
101
+ Both request and retained judgment validation enforce backend-specific provider,
102
+ effort, limits and environment-policy combinations. A qualified partial Codex
103
+ capture can be evaluated without a turn identity when no turn was accepted;
104
+ missing turn events stay unavailable and the judge can abstain.
105
+ The selected model must exist in the pinned executable's bundled catalog.
106
+ EBO copies that exact entry into a temporary catalog with apply-patch and
107
+ experimental tool declarations removed; its digest is retained with the raw
108
+ runtime response. Unknown catalog models fail before a turn starts.
109
+
110
+ Codex owns a fresh stdio app-server child with an empty working directory and
111
+ temporary HOME/CODEX_HOME. Only an existing `auth.json` login is copied into it;
112
+ personal settings, plugins, MCP, hooks, memories, shell, browser, image,
113
+ delegation, plan, sleep, and interactive tools are disabled. Thread and turn
114
+ environment selections are explicitly empty, and host skill discovery is
115
+ disabled. The child environment
116
+ allows only PATH, locale, and temporary-directory variables. Analytics and
117
+ telemetry exporters are disabled. The thread is ephemeral with no instruction
118
+ sources, read-only sandbox and no sandbox network access; supplied evidence
119
+ enters only through the prompt. Unexpected tool requests fail the judgment.
120
+ Timeouts interrupt the owned turn and reap the process group before deleting
121
+ the temporary home. Missing cost/API timing remains unavailable.
122
+ `runRetainedSemanticJudge` also accepts an optional `signal`; CLI SIGINT and
123
+ SIGTERM propagate through it to either backend. Interrupted calls retain a
124
+ failed record and bounded received output. Timeout and interruption take
125
+ precedence over a late successful terminal message; native startup probes
126
+ consume the same wall-clock budget as the turn.
127
+
128
+ The native structured response uses `turn/start.outputSchema` and completion
129
+ must match both owned thread and turn IDs. See the
130
+ [official app-server contract](https://developers.openai.com/codex/app-server/).
131
+ New assertions carry optional `evaluator.configurationDigest`, binding the
132
+ prompt version, rubric instructions, evaluator parameters, limits and blinding.
133
+ Evaluator defaults are made explicit before hashing: omitted and explicit
134
+ Claude backend defaults match, as do omitted/explicit native `codex`
135
+ executables and equivalent relative/absolute executable paths. Model or effort
136
+ changes still produce distinct configuration digests.
137
+ Existing v1 requests and assertions remain readable without rewriting them.
138
+
139
+ The response can contain only an assessed proposal or an abstention. Assessed
140
+ responses require confidence, rationale, an alternative explanation, and at
141
+ least one packaged citation. Abstentions require a reason, rationale, and
142
+ alternative explanation and may cite packaged events. Extra fields such as a
143
+ claimed `confirmed` review state are rejected. The runner constructs trusted
144
+ run, attempt, dataset, rubric, behavior, and evaluator bindings itself, then
145
+ calls `validateBehaviorAssertion`; fabricated, stale, foreign, digest-mismatched,
146
+ or ownership-invalid citations cannot produce a proposed assertion.
147
+
148
+ ## Retention
149
+
150
+ Raw input and output files are mode `0600` and marked `restricted`. A valid
151
+ response writes `input.json`, `raw-response.json`, `assertion.json`, and
152
+ `judgment.json`. Malformed responses, invalid citations, provider errors, and
153
+ timeouts instead write `failure.json` beside any bounded raw output. No failed
154
+ record enters review, no retry occurs, and native evidence is never changed.
155
+ Timing, cost, and usage are recorded only when the backend reports them;
156
+ otherwise the record says they are unavailable.
157
+ Native Codex records name the replacement environment policy and its exact
158
+ allowlist. Model output, including partial deltas and malformed final text,
159
+ is retained separately in restricted `raw-model-response.json` and referenced
160
+ by optional `rawModelResponse`. It and control-frame `raw-response.json` each
161
+ have their own `maxOutputChars` bound; startup or user-echo frames cannot
162
+ consume the model-output retention budget.
163
+
164
+ `ebo.semantic-judge-request/v1`, `ebo.semantic-judge-input/v1`, and
165
+ `ebo.semantic-judgment/v1` are registered artifacts and can be independently
166
+ checked with `ebo validate`.
167
+
168
+ This runner does not confirm assertions, create human reviews, adjudicate,
169
+ aggregate rates, build an Atlas, or publish evidence.
170
+
171
+ The opt-in live wiring test requires an already authenticated route discovered
172
+ by the operator; it never invents a model ID:
173
+
174
+ ```sh
175
+ EBO_LIVE_SEMANTIC_JUDGE_SMOKE=1 \
176
+ EBO_LIVE_SEMANTIC_JUDGE_MODEL='<existing-route>' \
177
+ node --test --test-name-pattern='approved live semantic judge smoke' \
178
+ dist/test/semantic-judge.test.js
179
+ ```
180
+
181
+ The optional installed-runtime contract test intercepts a synthetic local
182
+ model request, uses a dummy credential, and verifies an empty tool list:
183
+
184
+ ```sh
185
+ EBO_NATIVE_CODEX_CONTRACT=1 node --test dist/test/codex-judge.test.js
186
+ EBO_LIVE_CODEX_JUDGE_SMOKE=1 EBO_LIVE_CODEX_JUDGE_MODEL='<existing-route>' \
187
+ node --test --test-name-pattern='approved existing-auth' dist/test/codex-judge.test.js
188
+ ```
@@ -0,0 +1,84 @@
1
+ # Outcome ingestion and structural observations
2
+
3
+ Retained native loading accepts the implemented source versions: OpenHands
4
+ 1.44.1/1.46.0, DeepSeek SDK 0.1.1-rc.2, and Codex 0.150.1/0.153.4. Unsupported versions
5
+ fail explicitly rather than receiving a different adapter's provenance.
6
+ Completed Codex bundles require exactly one matching owned native terminal to
7
+ report completion; duplicate owned terminals reject. Qualified failed/partial
8
+ evidence remains separate.
9
+ Native envelopes and physical JSONL sequences are checked before dispatch;
10
+ Codex notifications from a foreign or client-only source cannot become events.
11
+ Codex start ownership requires unique ordered client-request/server-response
12
+ pairs with the same JSON-RPC ID; turn requests must name the owned thread and
13
+ completion must follow acceptance. OpenHands
14
+ capture requires exactly one server-info record, and its version and
15
+ conversation records must agree with the manifest; completed
16
+ runs require one owned final conversation with `execution_status: finished`.
17
+ DeepSeek requires the root prompt receipt and native parent/child links for
18
+ related sessions, and runtime reap must follow root idle. A coarse related-session list alone cannot authorize foreign
19
+ records, and the retained composition must match the pinned client version.
20
+ Verifier task failures also require normal native completion; infrastructure
21
+ failures and interruptions can retain qualified partial evidence instead.
22
+
23
+ `ebo observations` derives versioned, deterministic facts from a
24
+ capture-qualified retained Claude Agent SDK, Codex, OpenHands, DeepSeek, Pi, or Cursor run bundle. It runs qualification,
25
+ normalization, native-reference integrity validation, and the registered
26
+ extractors in that order. Source bundles are read-only; the command rejects an
27
+ output path inside the source bundle or corpus.
28
+
29
+ ```sh
30
+ ebo observations create <run-bundle-root> <output.json>
31
+ ebo observations corpus <corpus-root> <index.jsonl> <output-root> \
32
+ [--run <id>] [--attempt <id>] [--task <id>] [--model <id>] [--harness <id>] \
33
+ [--assessment-mode <observational|verified>]
34
+ ```
35
+
36
+ The corpus command first validates the supplied deterministic index, selects
37
+ run manifests with the same exact-match filters as `ebo corpus query`, and
38
+ writes one bounded `sha256-<run-attempt-tuple>.json` file per selection. It fails rather than skipping
39
+ an invalid or unsupported selected bundle. `createRetainedBehaviorEvidence`
40
+ dispatches verified native session records to each source-specific normalizer and
41
+ resolver. `createRetainedStructuralObservationSet` is the corresponding public
42
+ library call; the older Agent SDK-specific calls remain available. The same
43
+ loader supplies judge input, assertion validation, calibration, and aggregation.
44
+ Unsupported source fields remain unavailable; native schemas and identities
45
+ are preserved. Supplemental bundle metadata is retained separately as
46
+ `outcomeCapture`, so it cannot replace source-native session records.
47
+ OpenHands datasets retain their captured runtime version (`1.44.1` or `1.46.0`).
48
+ DeepSeek reapplies its native composition, capability, initialization, prompt,
49
+ and completed receipt-to-idle/runtime-reap gate before normalization; qualified
50
+ partial captures remain qualified-with-gaps. Physical JSONL locators stay
51
+ unchanged, and normalization qualification cannot exceed either the structural
52
+ bundle gate or the source-specific gate.
53
+
54
+ Each `ebo.structural-observation/v1` states its extractor/version, exact
55
+ definition, one-attempt denominator, unit, uniform event IDs, native-record
56
+ count, and native citations. `ebo.structural-observation-set/v1` also binds the observations to
57
+ the normalized-dataset digest and adapter coverage report.
58
+
59
+ Definitions are deliberately mechanical:
60
+
61
+ - logical tool operations are grouped only by explicit source-native operation
62
+ IDs; lifecycle records without one are counted separately;
63
+ - repetition counts distinct operations beyond the first that share the same
64
+ explicit tool identity and native-input digest;
65
+ - failure-followed-by-operation compares only explicit failures and later
66
+ operation starts in the same native-order domain, split into same-tool and
67
+ alternate-tool counts; it does not claim recovery;
68
+ - validation-after-mutation requires explicit mutation records and validation
69
+ records in one native-order domain; tool names are never used to infer either;
70
+ - model requests require model-request events and native request identities;
71
+ assistant messages and model reroutes are not requests;
72
+ - cumulative resource snapshots select only the latest snapshot in one known
73
+ native-order domain, increments are summed only when every record declares
74
+ increment semantics, and a cumulative-final record is used directly;
75
+ - native token categories stay separate, native total tokens are never rebuilt
76
+ from components, and cost does not imply subscription utilization;
77
+ - compaction counts include only native records that explicitly identify a
78
+ compaction boundary.
79
+
80
+ An observed zero is emitted only with available family coverage. Partial or
81
+ unsupported capability, ambiguous identity, unknown order, overlapping usage,
82
+ or missing timing remains `unavailable` with a reason. Observational runs have
83
+ no verifier assertion records and make no task-pass claim. Verified runs retain
84
+ each assertion outcome with its native verifier citation.