engineering-behavior-observatory 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +151 -0
- package/contracts/codex-app-server-0.150.1/manifest.json +25 -0
- package/contracts/codex-app-server-0.150.1/schema/InitializeParams.json +84 -0
- package/contracts/codex-app-server-0.150.1/schema/JSONRPCMessage.json +137 -0
- package/contracts/codex-app-server-0.150.1/schema/v2/ThreadReadParams.json +17 -0
- package/contracts/codex-app-server-0.150.1/schema/v2/ThreadTokenUsageUpdatedNotification.json +82 -0
- package/contracts/codex-app-server-0.150.1/schema/v2/TurnInterruptParams.json +17 -0
- package/contracts/codex-app-server-0.150.1/types/AskForApproval.ts +5 -0
- package/contracts/codex-app-server-0.150.1/types/ClientNotification.ts +5 -0
- package/contracts/codex-app-server-0.150.1/types/SandboxMode.ts +5 -0
- package/contracts/codex-app-server-0.150.1/types/ThreadReadParams.ts +9 -0
- package/contracts/codex-app-server-0.150.1/types/TokenUsageBreakdown.ts +5 -0
- package/contracts/codex-app-server-0.150.1/types/TurnInterruptParams.ts +5 -0
- package/contracts/codex-app-server-0.153.4/manifest.json +27 -0
- package/contracts/codex-app-server-0.153.4/schema/JSONRPCMessage.json +137 -0
- package/contracts/codex-app-server-0.153.4/schema/v1/InitializeParams.json +84 -0
- package/contracts/codex-app-server-0.153.4/schema/v2/ThreadReadParams.json +17 -0
- package/contracts/codex-app-server-0.153.4/schema/v2/ThreadStartParams.json +515 -0
- package/contracts/codex-app-server-0.153.4/schema/v2/ThreadTokenUsageUpdatedNotification.json +82 -0
- package/contracts/codex-app-server-0.153.4/schema/v2/TurnInterruptParams.json +17 -0
- package/contracts/codex-app-server-0.153.4/schema/v2/TurnStartParams.json +921 -0
- package/contracts/codex-app-server-0.153.4/types/AskForApproval.ts +5 -0
- package/contracts/codex-app-server-0.153.4/types/ClientNotification.ts +5 -0
- package/contracts/codex-app-server-0.153.4/types/SandboxMode.ts +5 -0
- package/contracts/codex-app-server-0.153.4/types/ThreadReadParams.ts +12 -0
- package/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.ts +5 -0
- package/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.ts +5 -0
- package/contracts/openhands-agent-server-v1.44.1.json +65 -0
- package/contracts/openhands-agent-server-v1.46.0.json +46 -0
- package/dist/contracts/codex-app-server-0.153.4/types/AskForApproval.d.ts +9 -0
- package/dist/contracts/codex-app-server-0.153.4/types/AskForApproval.js +2 -0
- package/dist/contracts/codex-app-server-0.153.4/types/ClientNotification.d.ts +3 -0
- package/dist/contracts/codex-app-server-0.153.4/types/ClientNotification.js +2 -0
- package/dist/contracts/codex-app-server-0.153.4/types/SandboxMode.d.ts +1 -0
- package/dist/contracts/codex-app-server-0.153.4/types/SandboxMode.js +2 -0
- package/dist/contracts/codex-app-server-0.153.4/types/ThreadReadParams.d.ts +10 -0
- package/dist/contracts/codex-app-server-0.153.4/types/ThreadReadParams.js +2 -0
- package/dist/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.d.ts +8 -0
- package/dist/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.js +2 -0
- package/dist/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.d.ts +4 -0
- package/dist/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.js +2 -0
- package/dist/src/agent-sdk-normalizer.d.ts +22 -0
- package/dist/src/agent-sdk-normalizer.js +995 -0
- package/dist/src/agent-sdk-run.d.ts +31 -0
- package/dist/src/agent-sdk-run.js +242 -0
- package/dist/src/agent-sdk-runner.d.ts +96 -0
- package/dist/src/agent-sdk-runner.js +407 -0
- package/dist/src/agent-sdk.d.ts +294 -0
- package/dist/src/agent-sdk.js +644 -0
- package/dist/src/aggregation.d.ts +183 -0
- package/dist/src/aggregation.js +513 -0
- package/dist/src/artifacts.d.ts +36 -0
- package/dist/src/artifacts.js +1372 -0
- package/dist/src/atlas-grafana.d.ts +377 -0
- package/dist/src/atlas-grafana.js +47 -0
- package/dist/src/atlas-html.d.ts +2 -0
- package/dist/src/atlas-html.js +34 -0
- package/dist/src/atlas.d.ts +130 -0
- package/dist/src/atlas.js +235 -0
- package/dist/src/behavior-assertions.d.ts +95 -0
- package/dist/src/behavior-assertions.js +154 -0
- package/dist/src/cli.d.ts +5 -0
- package/dist/src/cli.js +957 -0
- package/dist/src/codex-judge.d.ts +44 -0
- package/dist/src/codex-judge.js +214 -0
- package/dist/src/codex-run.d.ts +114 -0
- package/dist/src/codex-run.js +519 -0
- package/dist/src/codex.d.ts +187 -0
- package/dist/src/codex.js +1028 -0
- package/dist/src/contracts.d.ts +155 -0
- package/dist/src/contracts.js +866 -0
- package/dist/src/corpus.d.ts +67 -0
- package/dist/src/corpus.js +540 -0
- package/dist/src/cursor-sdk-runner.d.ts +81 -0
- package/dist/src/cursor-sdk-runner.js +362 -0
- package/dist/src/cursor-sdk.d.ts +81 -0
- package/dist/src/cursor-sdk.js +1078 -0
- package/dist/src/deepseek-adapter.d.ts +152 -0
- package/dist/src/deepseek-adapter.js +777 -0
- package/dist/src/exports.d.ts +69 -0
- package/dist/src/exports.js +966 -0
- package/dist/src/human-calibration.d.ts +167 -0
- package/dist/src/human-calibration.js +618 -0
- package/dist/src/index.d.ts +71 -0
- package/dist/src/index.js +38 -0
- package/dist/src/lifecycle.d.ts +196 -0
- package/dist/src/lifecycle.js +1889 -0
- package/dist/src/normalization-integrity.d.ts +128 -0
- package/dist/src/normalization-integrity.js +429 -0
- package/dist/src/openhands-run.d.ts +36 -0
- package/dist/src/openhands-run.js +415 -0
- package/dist/src/openhands.d.ts +130 -0
- package/dist/src/openhands.js +859 -0
- package/dist/src/pi.d.ts +203 -0
- package/dist/src/pi.js +1345 -0
- package/dist/src/process-protocol.d.ts +245 -0
- package/dist/src/process-protocol.js +1261 -0
- package/dist/src/retained-evidence.d.ts +12 -0
- package/dist/src/retained-evidence.js +201 -0
- package/dist/src/run-bundles.d.ts +193 -0
- package/dist/src/run-bundles.js +993 -0
- package/dist/src/scheduler.d.ts +109 -0
- package/dist/src/scheduler.js +1080 -0
- package/dist/src/semantic-judge.d.ts +194 -0
- package/dist/src/semantic-judge.js +875 -0
- package/dist/src/structural-observations.d.ts +124 -0
- package/dist/src/structural-observations.js +671 -0
- package/dist/src/task-packets.d.ts +115 -0
- package/dist/src/task-packets.js +683 -0
- package/dist/src/uniform-events.d.ts +120 -0
- package/dist/src/uniform-events.js +158 -0
- package/dist/src/verifiers.d.ts +116 -0
- package/dist/src/verifiers.js +819 -0
- package/dist/src/workspaces.d.ts +56 -0
- package/dist/src/workspaces.js +1314 -0
- package/docs/README.md +60 -0
- package/docs/development/README.md +71 -0
- package/docs/development/documentation-sync.md +58 -0
- package/docs/development/extension-contracts.md +307 -0
- package/docs/evaluation/README.md +48 -0
- package/docs/evaluation/aggregation.md +112 -0
- package/docs/evaluation/behavior-assertions.md +45 -0
- package/docs/evaluation/human-calibration.md +170 -0
- package/docs/evaluation/normalization-integrity.md +80 -0
- package/docs/evaluation/semantic-judge.md +188 -0
- package/docs/evaluation/structural-observations.md +84 -0
- package/docs/evaluation/uniform-events.md +131 -0
- package/docs/guides/atlas.md +144 -0
- package/docs/guides/evidence-and-sharing.md +100 -0
- package/docs/guides/operator-guide.md +424 -0
- package/docs/guides/quickstart.md +152 -0
- package/docs/guides/telemetry.md +66 -0
- package/docs/harnesses/README.md +31 -0
- package/docs/harnesses/claude-agent-sdk.md +61 -0
- package/docs/harnesses/codex-harness.md +174 -0
- package/docs/harnesses/cursor-sdk.md +149 -0
- package/docs/harnesses/deepseek-harness.md +134 -0
- package/docs/harnesses/openhands-agent-server.md +153 -0
- package/docs/harnesses/pi-sdk.md +66 -0
- package/docs/reference/README.md +19 -0
- package/docs/reference/agent-sdk-operational-runner.md +118 -0
- package/docs/reference/cli.md +114 -0
- package/docs/reference/contracts.md +222 -0
- package/docs/reference/run-bundle-contract.md +354 -0
- package/docs/reference/run-lifecycle.md +54 -0
- package/examples/cursor-sdk/README.md +11 -0
- package/examples/cursor-sdk/capture-profile.json +1 -0
- package/examples/cursor-sdk/harness.json +1 -0
- package/examples/cursor-sdk/model.json +1 -0
- package/examples/cursor-sdk/native-limits.json +1 -0
- package/examples/cursor-sdk/native-tool-policy.json +1 -0
- package/ontology/behavior-categories.v1.json +46 -0
- package/package.json +66 -0
- package/release/0.1.0/KNOWN_LIMITATIONS.md +35 -0
- package/release/0.1.0/README.md +46 -0
- package/release/0.1.0/reproducibility.json +61 -0
- package/release/0.2.0/KNOWN_LIMITATIONS.md +40 -0
- package/release/0.2.0/README.md +42 -0
- package/release/0.2.0/reproducibility.json +73 -0
- package/release/0.2.1/KNOWN_LIMITATIONS.md +18 -0
- package/release/0.2.1/README.md +63 -0
- package/release/0.2.1/reproducibility.json +73 -0
- package/release/README.md +13 -0
- package/schemas/aggregation.v1.json +293 -0
- package/schemas/behavior-assertions.v1.json +220 -0
- package/schemas/deepseek-runtime-composition.v1.schema.json +122 -0
- package/schemas/experiment.v1.schema.json +188 -0
- package/schemas/human-calibration.v1.json +336 -0
- package/schemas/normalization-integrity.v1.json +302 -0
- package/schemas/run-bundles/v1.json +1040 -0
- package/schemas/run-queue.v1.schema.json +206 -0
- package/schemas/semantic-judge.v1.json +380 -0
- package/schemas/structural-observations.v1.json +131 -0
- package/schemas/task-packet-freeze.v1.schema.json +106 -0
- package/schemas/task-packet.v1.schema.json +234 -0
- package/schemas/uniform-events/v1.json +289 -0
- package/scripts/atlas-grafana.sh +18 -0
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# Behavior assertions and evidence resolution
|
|
2
|
+
|
|
3
|
+
`ebo.behavior-assertion/v1` records one semantic assessment or abstention for
|
|
4
|
+
one declared behavior dimension. The initial `ebo.behavior-vocabulary/v1` is a
|
|
5
|
+
small data file at
|
|
6
|
+
[`ontology/behavior-categories.v1.json`](../../ontology/behavior-categories.v1.json).
|
|
7
|
+
Its eight entries are the categories in the canonical specification; unmapped
|
|
8
|
+
observations do not force a new category.
|
|
9
|
+
|
|
10
|
+
Each assertion binds the normalized-dataset schema version and canonical
|
|
11
|
+
digest, run and attempt IDs, vocabulary version, one category/dimension pair,
|
|
12
|
+
and evaluator/rubric identities. An assessment is constructive, adverse,
|
|
13
|
+
mixed, or context-dependent and retains confidence on the explicit
|
|
14
|
+
`evaluator-reported-0-to-1` scale, rationale, an alternative explanation, and
|
|
15
|
+
at least one citation. Confidence is not a calibrated probability. An abstention instead retains its reason and
|
|
16
|
+
may name the missing event or evidence capability; citations are optional.
|
|
17
|
+
|
|
18
|
+
A citation names both a normalized event ID and that event's exact native
|
|
19
|
+
source reference. `validateBehaviorAssertion` first applies the existing
|
|
20
|
+
normalized-dataset integrity validator, then checks the dataset digest and
|
|
21
|
+
assertion identity, declared dimension, event ownership, native reference, and
|
|
22
|
+
resolver-provided run/attempt/digest. Changed datasets, foreign attempts,
|
|
23
|
+
dangling events or native records, and stale digests fail closed.
|
|
24
|
+
|
|
25
|
+
`ebo.behavior-review/v1` is a separate record bound to the assertion digest.
|
|
26
|
+
Its states are `proposed`, `confirmed`, `disputed`, `rejected`, and
|
|
27
|
+
`insufficient-evidence`. Proposed records cannot carry a reviewer; all other
|
|
28
|
+
states require an explicitly identified human reviewer and rationale. The
|
|
29
|
+
library derives confirmed aggregation eligibility only after revalidating the
|
|
30
|
+
assertion against its dataset and native resolver, and then finding a valid
|
|
31
|
+
`confirmed` review of a non-abstaining assertion. A judge assertion cannot set
|
|
32
|
+
human confirmation itself.
|
|
33
|
+
|
|
34
|
+
The callable validator accepts any `ebo.normalized-dataset/v1` and its
|
|
35
|
+
source-specific native resolver. The current CLI rebuilds and validates the
|
|
36
|
+
normalized dataset from a retained Claude Agent SDK, Codex, OpenHands, Pi, or
|
|
37
|
+
DeepSeek run bundle through `validateRetainedBehaviorAssertion`:
|
|
38
|
+
|
|
39
|
+
```sh
|
|
40
|
+
ebo assertions validate <run-bundle-root> <assertion.json> [review.json]
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
This command validates contracts and evidence only. It does not run a judge,
|
|
44
|
+
change review state, adjudicate a dispute, aggregate results, or build an
|
|
45
|
+
Atlas.
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
# Human calibration and adjudication
|
|
2
|
+
|
|
3
|
+
EBO supplies a local, static workflow for people to review evidence-grounded
|
|
4
|
+
behavior assertions. It does not assign reviewers, authenticate users, host a
|
|
5
|
+
review service, or treat synthetic fixture decisions as research labels.
|
|
6
|
+
|
|
7
|
+
## Select a reproducible sample
|
|
8
|
+
|
|
9
|
+
Create a source file that points to capture-qualified Claude Agent SDK, Codex,
|
|
10
|
+
OpenHands, DeepSeek, Pi, or Cursor bundles and
|
|
11
|
+
their judge-produced assertions. Paths are local operator inputs; native
|
|
12
|
+
evidence is revalidated through the existing assertion resolver before a
|
|
13
|
+
candidate can enter a sample.
|
|
14
|
+
|
|
15
|
+
```json
|
|
16
|
+
{
|
|
17
|
+
"schemaVersion": "ebo.review-source-set/v1",
|
|
18
|
+
"sources": [
|
|
19
|
+
{
|
|
20
|
+
"bundleRoot": "/local/restricted/runs/run-1",
|
|
21
|
+
"assertionPath": "/local/restricted/judgments/run-1/assertion.json",
|
|
22
|
+
"taskContext": "Reviewable task prompt or bounded operator-authored task synopsis."
|
|
23
|
+
}
|
|
24
|
+
]
|
|
25
|
+
}
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Sampling criteria use explicit, non-overlapping strata. Each stratum has its
|
|
29
|
+
own requested size and may filter by task, model, harness, verifier outcome,
|
|
30
|
+
terminal state, behavior category, confidence, or judge abstention. A single
|
|
31
|
+
stratum with empty filters is an unstratified sample.
|
|
32
|
+
|
|
33
|
+
```json
|
|
34
|
+
{
|
|
35
|
+
"schemaVersion": "ebo.review-sample-criteria/v1",
|
|
36
|
+
"seed": "study-2026-09-wave-1",
|
|
37
|
+
"strata": [
|
|
38
|
+
{
|
|
39
|
+
"id": "high-confidence-verification",
|
|
40
|
+
"sampleSize": 20,
|
|
41
|
+
"filters": {
|
|
42
|
+
"categoryIds": ["verification-completion"],
|
|
43
|
+
"confidence": { "min": 0.75 },
|
|
44
|
+
"abstentions": [false]
|
|
45
|
+
}
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
"id": "judge-abstentions",
|
|
49
|
+
"sampleSize": 10,
|
|
50
|
+
"filters": { "abstentions": [true] }
|
|
51
|
+
}
|
|
52
|
+
]
|
|
53
|
+
}
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
```sh
|
|
57
|
+
ebo calibration sample sources.json criteria.json selection.json
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
The saved selection retains the criteria and seed, every source root, source and
|
|
61
|
+
eligible population sizes, eligible and selected assertion IDs, per-stratum requested,
|
|
62
|
+
eligible, and selected counts, and any zero-candidate strata. Ordering is a
|
|
63
|
+
SHA-256 function of the seed, stratum, assertion ID, and assertion digest.
|
|
64
|
+
Repeated request-derived assertion IDs across runs remain distinct through
|
|
65
|
+
their digest-bound identity. Where a CLI selector is ambiguous, append the
|
|
66
|
+
digest as `<assertion-id>@sha256:<digest>`.
|
|
67
|
+
Every command that reuses a saved selection deterministically revalidates its
|
|
68
|
+
criteria, eligible population, strata, seeded ordering, and candidate bindings
|
|
69
|
+
against the retained source evidence.
|
|
70
|
+
Observational runs always record verifier outcome as `unavailable`; normal
|
|
71
|
+
completion is never substituted for verifier success.
|
|
72
|
+
|
|
73
|
+
## Generate and inspect a static packet
|
|
74
|
+
|
|
75
|
+
```sh
|
|
76
|
+
ebo calibration packet selection.json review-packet
|
|
77
|
+
open review-packet/index.html
|
|
78
|
+
ebo calibration inspect \
|
|
79
|
+
review-packet/packet.json <assertion-id> [event-id]
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
`index.html` contains the supplied task context, condition context, assertion,
|
|
83
|
+
rationale, alternative explanation, and links to local pages that render the
|
|
84
|
+
exact cited native records. Each evidence page also links to its source artifact
|
|
85
|
+
and names the native locator. All task, assertion, and evidence content is
|
|
86
|
+
HTML-escaped. Links remain relative to the packet and its declared local bundle
|
|
87
|
+
root. The packet copies only cited records, keeps them `restricted-local-only`
|
|
88
|
+
with mode `0600`, and keeps those heavyweight records out of `packet.json`.
|
|
89
|
+
It is not a partner or public export. Moving the packet
|
|
90
|
+
separately from its source bundles breaks the original-artifact links by design.
|
|
91
|
+
Packet files are staged together and the completed directory is published in
|
|
92
|
+
one rename, so `packet.json` never names partially rendered evidence pages.
|
|
93
|
+
All selection, packet, history, and summary destinations are rejected when they
|
|
94
|
+
would be written inside a source run bundle.
|
|
95
|
+
|
|
96
|
+
## Import human decisions
|
|
97
|
+
|
|
98
|
+
Get the exact assertion and current history bindings without manufacturing a
|
|
99
|
+
decision:
|
|
100
|
+
|
|
101
|
+
```sh
|
|
102
|
+
ebo calibration binding selection.json <assertion-id>
|
|
103
|
+
ebo calibration binding selection.json <assertion-id> history.json
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
A person supplies the reviewer identity, timestamp, decision, and rationale.
|
|
107
|
+
`insufficient-evidence` is the human abstention state. The first decision uses
|
|
108
|
+
`previousHistory: null`; later decisions use the binding printed for the
|
|
109
|
+
current history.
|
|
110
|
+
|
|
111
|
+
```json
|
|
112
|
+
{
|
|
113
|
+
"schemaVersion": "ebo.human-review-decision/v1",
|
|
114
|
+
"id": "review-001",
|
|
115
|
+
"kind": "review",
|
|
116
|
+
"assertion": {
|
|
117
|
+
"id": "<assertion-id>",
|
|
118
|
+
"schemaVersion": "ebo.behavior-assertion/v1",
|
|
119
|
+
"digest": "sha256:<assertion-digest>"
|
|
120
|
+
},
|
|
121
|
+
"reviewer": { "kind": "human", "id": "<reviewer-id>" },
|
|
122
|
+
"decidedAt": "2026-09-07T12:00:00Z",
|
|
123
|
+
"state": "confirmed",
|
|
124
|
+
"rationale": "<human-authored rationale>",
|
|
125
|
+
"previousHistory": null
|
|
126
|
+
}
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
```sh
|
|
130
|
+
ebo calibration import selection.json history.json decision.json
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Import appends to one history and never rewrites an earlier decision. It rejects
|
|
134
|
+
unknown or changed assertions, stale history bindings, invalid adjudication
|
|
135
|
+
targets, and reuse of a decision ID with different content. Reimporting the
|
|
136
|
+
exact same decision is idempotent.
|
|
137
|
+
Imports serialize through a per-history portable filesystem lock with a stale
|
|
138
|
+
timeout and heartbeat. Interrupted imports are recovered automatically; an
|
|
139
|
+
active import makes a concurrent command fail closed for an idempotent retry.
|
|
140
|
+
|
|
141
|
+
An adjudication is another explicit human decision. It names at least two prior
|
|
142
|
+
review decisions for the same assertion:
|
|
143
|
+
|
|
144
|
+
```sh
|
|
145
|
+
ebo calibration adjudicate \
|
|
146
|
+
selection.json history.json adjudication.json
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Its JSON uses `kind: "adjudication"` and an `adjudicates` array of prior review
|
|
150
|
+
decision IDs. This records lineage; it does not infer or automate adjudication.
|
|
151
|
+
|
|
152
|
+
## Summarize calibration
|
|
153
|
+
|
|
154
|
+
```sh
|
|
155
|
+
ebo calibration summarize \
|
|
156
|
+
selection.json history.json calibration-summary.json
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
The summary reports totals and category-level counts. Judge-human agreement is
|
|
160
|
+
over latest judge-human decision pairs on non-abstaining assertions.
|
|
161
|
+
Human-human agreement is over distinct-reviewer pairs on the same assertion,
|
|
162
|
+
excluding human abstentions. Adjudication counts use human adjudication
|
|
163
|
+
decisions as their separate denominator. Every agreement result names its
|
|
164
|
+
population and denominator; with no comparable reviews it is `unavailable`
|
|
165
|
+
with denominator zero, never perfect agreement. Disputed, inconsistent,
|
|
166
|
+
unreviewed, insufficient-evidence, and judge-abstained assertions are excluded
|
|
167
|
+
from confirmed eligibility. A review appended after adjudication makes that
|
|
168
|
+
adjudication non-current until the updated review population is adjudicated.
|
|
169
|
+
These are calibration summaries, not comparison
|
|
170
|
+
or Behavior Atlas aggregates.
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# Normalization integrity and comparison gates
|
|
2
|
+
|
|
3
|
+
Normalization is accepted only when every projected event remains bound to one
|
|
4
|
+
capture-qualified run and attempt. The native record stays authoritative; the
|
|
5
|
+
normalized dataset retains only its source reference, native type, and digest.
|
|
6
|
+
|
|
7
|
+
## Dataset validation and coverage
|
|
8
|
+
|
|
9
|
+
`describeNormalizedDataset` combines a qualified capture, its normalization
|
|
10
|
+
result, the adapter capability profile, and the pinned adapter version into an
|
|
11
|
+
`ebo.normalized-dataset/v1` record. Source and content references retain
|
|
12
|
+
separate expected digests without copying native records. It rejects input that
|
|
13
|
+
is not explicitly `qualified` or `qualified-with-gaps`.
|
|
14
|
+
`validateNormalizedDataset` then checks:
|
|
15
|
+
|
|
16
|
+
- the dataset, event, and capability-profile schemas;
|
|
17
|
+
- stable run, attempt, adapter, harness, and native-type identity;
|
|
18
|
+
- resolver-provided run/attempt ownership and every source/content digest;
|
|
19
|
+
- source and content references, event relations, and acyclic parentage;
|
|
20
|
+
- nondecreasing order within each native order domain;
|
|
21
|
+
- that every native record is mapped or explicitly retained as unmapped; and
|
|
22
|
+
- that emitted families and evidence do not contradict adapter capabilities.
|
|
23
|
+
|
|
24
|
+
The resolver must return `{ runId, attemptId, digest }` for every source and
|
|
25
|
+
content reference. A boolean resolver remains supported by the lower-level
|
|
26
|
+
uniform event validator, but it is insufficient for dataset integrity
|
|
27
|
+
validation. The dataset validator rejects resolution metadata owned by another
|
|
28
|
+
run or attempt.
|
|
29
|
+
`createCapturedNativeEvidenceResolver` supplies the stronger metadata for an
|
|
30
|
+
in-memory qualified capture and can delegate other content references to an
|
|
31
|
+
adapter-specific resolver. Derived JSON Pointer locators are resolved against
|
|
32
|
+
the source record; the helper never treats a containing record as proof that an
|
|
33
|
+
arbitrary child locator exists.
|
|
34
|
+
|
|
35
|
+
Successful validation returns an `ebo.adapter-coverage-report/v1` report. It
|
|
36
|
+
counts mapped and unmapped native records by adapter version and native type.
|
|
37
|
+
For each uniform family it reports both the declared capability status and the
|
|
38
|
+
observed event count. Consequently, an unsupported family with zero events is
|
|
39
|
+
not confused with an available family that happened to have zero observations.
|
|
40
|
+
Unknown source records remain present in the native-type table and in the
|
|
41
|
+
dataset's `unmapped` list.
|
|
42
|
+
|
|
43
|
+
DeepSeek JSON-RPC methods and OpenHands REST/WebSocket record kinds remain
|
|
44
|
+
native types in these reports. They do not create event families or imply a
|
|
45
|
+
shared control protocol.
|
|
46
|
+
|
|
47
|
+
## Comparison eligibility
|
|
48
|
+
|
|
49
|
+
The comparison gate consumes one explicit `ebo.comparison-request/v1` or
|
|
50
|
+
`ebo.comparison-request/v2` artifact. V2 additionally retains the exact measure
|
|
51
|
+
and candidate manifest digests required by aggregation:
|
|
52
|
+
|
|
53
|
+
```sh
|
|
54
|
+
ebo comparison check <request.json>
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
The request names both candidates, their task and fixture digests, model and
|
|
58
|
+
harness configuration identities, material capture/budget/tool-policy digests,
|
|
59
|
+
normalization-adapter identities and pinned versions, capability profiles, required capabilities,
|
|
60
|
+
and any declared model or harness difference. The command prints an
|
|
61
|
+
matching versioned `ebo.comparison-report/v1` or
|
|
62
|
+
`ebo.comparison-report/v2` JSON object and
|
|
63
|
+
returns nonzero when its status is `unsupported`.
|
|
64
|
+
|
|
65
|
+
Statuses are:
|
|
66
|
+
|
|
67
|
+
- `supported`: conditions match and required capabilities are available;
|
|
68
|
+
- `qualified-with-caveats`: a model/harness difference was declared or a
|
|
69
|
+
required capability is partial; and
|
|
70
|
+
- `unsupported`: task, fixture, material configuration, or undeclared
|
|
71
|
+
model/harness conditions differ, or a required capability is unsupported.
|
|
72
|
+
|
|
73
|
+
Declaring a model or harness difference makes that difference inspectable; it
|
|
74
|
+
does not claim identical conditions or establish that the harness caused an
|
|
75
|
+
observed behavioral difference.
|
|
76
|
+
|
|
77
|
+
The schema is
|
|
78
|
+
[`schemas/normalization-integrity.v1.json`](../../schemas/normalization-integrity.v1.json).
|
|
79
|
+
Exact-match and incompatible-fixture examples live under
|
|
80
|
+
`test/fixtures/comparison` in the source repository.
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
# Evidence-grounded semantic judge
|
|
2
|
+
|
|
3
|
+
`ebo judge run` evaluates one declared behavior dimension against a bounded,
|
|
4
|
+
caller-selected projection of a qualified Claude Agent SDK, Codex, OpenHands, Pi,
|
|
5
|
+
Cursor, or DeepSeek retained run bundle and an
|
|
6
|
+
`ebo.structural-observation-set/v1` file:
|
|
7
|
+
|
|
8
|
+
```sh
|
|
9
|
+
ebo judge run \
|
|
10
|
+
<run-bundle-root> <observations.json> <request.json> <output-root>
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
The output root must not exist and must be outside the immutable run bundle.
|
|
14
|
+
Every rerun therefore creates a new record. Runs with the same rubric,
|
|
15
|
+
selection, limits, and evidence share an input digest even when their output
|
|
16
|
+
roots or judge identities differ.
|
|
17
|
+
|
|
18
|
+
## Request
|
|
19
|
+
|
|
20
|
+
The request is caller-owned configuration; EBO does not select a study model
|
|
21
|
+
or retry a judgment:
|
|
22
|
+
|
|
23
|
+
```json
|
|
24
|
+
{
|
|
25
|
+
"schemaVersion": "ebo.semantic-judge-request/v1",
|
|
26
|
+
"id": "verification-run-001",
|
|
27
|
+
"behavior": {
|
|
28
|
+
"vocabularyVersion": "1.0.0",
|
|
29
|
+
"categoryId": "verification-completion",
|
|
30
|
+
"dimensionId": "verification-completion"
|
|
31
|
+
},
|
|
32
|
+
"rubric": {
|
|
33
|
+
"id": "verification-rubric",
|
|
34
|
+
"version": "1.0.0",
|
|
35
|
+
"instructions": "Assess whether retained evidence shows validation before completion."
|
|
36
|
+
},
|
|
37
|
+
"evaluator": {
|
|
38
|
+
"provider": "anthropic",
|
|
39
|
+
"model": "<existing authenticated Claude Agent SDK route>",
|
|
40
|
+
"effort": "low"
|
|
41
|
+
},
|
|
42
|
+
"selection": {
|
|
43
|
+
"eventIds": ["<normalized-event-id>"],
|
|
44
|
+
"structuralObservationIds": ["<structural-observation-id>"],
|
|
45
|
+
"includeOutcomeObservations": true
|
|
46
|
+
},
|
|
47
|
+
"limits": {
|
|
48
|
+
"maxEvidenceItems": 16,
|
|
49
|
+
"maxRecordChars": 4096,
|
|
50
|
+
"maxInputChars": 64000,
|
|
51
|
+
"maxOutputChars": 8192,
|
|
52
|
+
"maxCitations": 8,
|
|
53
|
+
"maxWallClockMs": 30000,
|
|
54
|
+
"maxTurns": 1,
|
|
55
|
+
"maxBudgetUsd": 0.1
|
|
56
|
+
},
|
|
57
|
+
"blinding": { "evaluatedModelIdentity": "redact" }
|
|
58
|
+
}
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Selection is exact. Unknown IDs fail before execution. Optional outcome
|
|
62
|
+
observations are added only when requested, and the total must remain within
|
|
63
|
+
`maxEvidenceItems`. Per-record truncation and whole-input omission are recorded
|
|
64
|
+
in `input.json`; omitted event IDs cannot be cited. Exact evaluated-model
|
|
65
|
+
strings are redacted from evidence values and keys when requested. Citation
|
|
66
|
+
IDs and native references are never rewritten, and the input records the
|
|
67
|
+
remaining harness, native-type, behavioral, and citation clues that may still
|
|
68
|
+
reveal origin.
|
|
69
|
+
|
|
70
|
+
Selected structural observations automatically include their normalized source
|
|
71
|
+
events so their claims remain citable. An explicitly selected observation with
|
|
72
|
+
no normalized source event, including an exact zero fact, is rejected before
|
|
73
|
+
execution; the same kind of automatically included outcome is recorded as
|
|
74
|
+
omitted instead of being shown to the judge as uncitable support.
|
|
75
|
+
|
|
76
|
+
## Backend and trust boundary
|
|
77
|
+
|
|
78
|
+
The default backend is the installed TypeScript Claude Agent SDK. The caller
|
|
79
|
+
supplies model, effort, wall-clock, turn, output, citation, and optional cost
|
|
80
|
+
limits. The runner passes no tools, settings sources, skills, plugins, MCP
|
|
81
|
+
servers, additional directories, or persistent session. It uses an empty
|
|
82
|
+
temporary working directory, strict empty MCP configuration, `dontAsk`
|
|
83
|
+
permissions, and a custom system prompt that treats the delimited trajectory
|
|
84
|
+
payload as untrusted data rather than instructions.
|
|
85
|
+
|
|
86
|
+
The SDK subprocess inherits the parent environment for authentication and
|
|
87
|
+
routing, but EBO removes ambient model and effort override variables before
|
|
88
|
+
launch so the caller request remains authoritative. The retained evaluator
|
|
89
|
+
metadata records that policy without retaining secret values.
|
|
90
|
+
Ambient OpenTelemetry and Claude telemetry/tracing controls are also removed;
|
|
91
|
+
restricted rubric and trajectory content is never sent to a parent-configured
|
|
92
|
+
collector, and a console exporter cannot corrupt the SDK protocol channel.
|
|
93
|
+
|
|
94
|
+
For the native Codex backend, set `evaluator.backend` to `codex-app-server`,
|
|
95
|
+
`provider` to `openai`, and supply `model` and `effort`. The optional
|
|
96
|
+
`executable` selects the installed `codex` executable (default: PATH). Relative
|
|
97
|
+
paths resolve against the caller's working directory before isolation; its version
|
|
98
|
+
must be `0.153.4`. Set `maxTurns` to `1` and omit `maxBudgetUsd`: this backend
|
|
99
|
+
does not support USD budget enforcement. No automatic provider fallback occurs.
|
|
100
|
+
Backend selection is independent of the evaluated harness.
|
|
101
|
+
Both request and retained judgment validation enforce backend-specific provider,
|
|
102
|
+
effort, limits and environment-policy combinations. A qualified partial Codex
|
|
103
|
+
capture can be evaluated without a turn identity when no turn was accepted;
|
|
104
|
+
missing turn events stay unavailable and the judge can abstain.
|
|
105
|
+
The selected model must exist in the pinned executable's bundled catalog.
|
|
106
|
+
EBO copies that exact entry into a temporary catalog with apply-patch and
|
|
107
|
+
experimental tool declarations removed; its digest is retained with the raw
|
|
108
|
+
runtime response. Unknown catalog models fail before a turn starts.
|
|
109
|
+
|
|
110
|
+
Codex owns a fresh stdio app-server child with an empty working directory and
|
|
111
|
+
temporary HOME/CODEX_HOME. Only an existing `auth.json` login is copied into it;
|
|
112
|
+
personal settings, plugins, MCP, hooks, memories, shell, browser, image,
|
|
113
|
+
delegation, plan, sleep, and interactive tools are disabled. Thread and turn
|
|
114
|
+
environment selections are explicitly empty, and host skill discovery is
|
|
115
|
+
disabled. The child environment
|
|
116
|
+
allows only PATH, locale, and temporary-directory variables. Analytics and
|
|
117
|
+
telemetry exporters are disabled. The thread is ephemeral with no instruction
|
|
118
|
+
sources, read-only sandbox and no sandbox network access; supplied evidence
|
|
119
|
+
enters only through the prompt. Unexpected tool requests fail the judgment.
|
|
120
|
+
Timeouts interrupt the owned turn and reap the process group before deleting
|
|
121
|
+
the temporary home. Missing cost/API timing remains unavailable.
|
|
122
|
+
`runRetainedSemanticJudge` also accepts an optional `signal`; CLI SIGINT and
|
|
123
|
+
SIGTERM propagate through it to either backend. Interrupted calls retain a
|
|
124
|
+
failed record and bounded received output. Timeout and interruption take
|
|
125
|
+
precedence over a late successful terminal message; native startup probes
|
|
126
|
+
consume the same wall-clock budget as the turn.
|
|
127
|
+
|
|
128
|
+
The native structured response uses `turn/start.outputSchema` and completion
|
|
129
|
+
must match both owned thread and turn IDs. See the
|
|
130
|
+
[official app-server contract](https://developers.openai.com/codex/app-server/).
|
|
131
|
+
New assertions carry optional `evaluator.configurationDigest`, binding the
|
|
132
|
+
prompt version, rubric instructions, evaluator parameters, limits and blinding.
|
|
133
|
+
Evaluator defaults are made explicit before hashing: omitted and explicit
|
|
134
|
+
Claude backend defaults match, as do omitted/explicit native `codex`
|
|
135
|
+
executables and equivalent relative/absolute executable paths. Model or effort
|
|
136
|
+
changes still produce distinct configuration digests.
|
|
137
|
+
Existing v1 requests and assertions remain readable without rewriting them.
|
|
138
|
+
|
|
139
|
+
The response can contain only an assessed proposal or an abstention. Assessed
|
|
140
|
+
responses require confidence, rationale, an alternative explanation, and at
|
|
141
|
+
least one packaged citation. Abstentions require a reason, rationale, and
|
|
142
|
+
alternative explanation and may cite packaged events. Extra fields such as a
|
|
143
|
+
claimed `confirmed` review state are rejected. The runner constructs trusted
|
|
144
|
+
run, attempt, dataset, rubric, behavior, and evaluator bindings itself, then
|
|
145
|
+
calls `validateBehaviorAssertion`; fabricated, stale, foreign, digest-mismatched,
|
|
146
|
+
or ownership-invalid citations cannot produce a proposed assertion.
|
|
147
|
+
|
|
148
|
+
## Retention
|
|
149
|
+
|
|
150
|
+
Raw input and output files are mode `0600` and marked `restricted`. A valid
|
|
151
|
+
response writes `input.json`, `raw-response.json`, `assertion.json`, and
|
|
152
|
+
`judgment.json`. Malformed responses, invalid citations, provider errors, and
|
|
153
|
+
timeouts instead write `failure.json` beside any bounded raw output. No failed
|
|
154
|
+
record enters review, no retry occurs, and native evidence is never changed.
|
|
155
|
+
Timing, cost, and usage are recorded only when the backend reports them;
|
|
156
|
+
otherwise the record says they are unavailable.
|
|
157
|
+
Native Codex records name the replacement environment policy and its exact
|
|
158
|
+
allowlist. Model output, including partial deltas and malformed final text,
|
|
159
|
+
is retained separately in restricted `raw-model-response.json` and referenced
|
|
160
|
+
by optional `rawModelResponse`. It and control-frame `raw-response.json` each
|
|
161
|
+
have their own `maxOutputChars` bound; startup or user-echo frames cannot
|
|
162
|
+
consume the model-output retention budget.
|
|
163
|
+
|
|
164
|
+
`ebo.semantic-judge-request/v1`, `ebo.semantic-judge-input/v1`, and
|
|
165
|
+
`ebo.semantic-judgment/v1` are registered artifacts and can be independently
|
|
166
|
+
checked with `ebo validate`.
|
|
167
|
+
|
|
168
|
+
This runner does not confirm assertions, create human reviews, adjudicate,
|
|
169
|
+
aggregate rates, build an Atlas, or publish evidence.
|
|
170
|
+
|
|
171
|
+
The opt-in live wiring test requires an already authenticated route discovered
|
|
172
|
+
by the operator; it never invents a model ID:
|
|
173
|
+
|
|
174
|
+
```sh
|
|
175
|
+
EBO_LIVE_SEMANTIC_JUDGE_SMOKE=1 \
|
|
176
|
+
EBO_LIVE_SEMANTIC_JUDGE_MODEL='<existing-route>' \
|
|
177
|
+
node --test --test-name-pattern='approved live semantic judge smoke' \
|
|
178
|
+
dist/test/semantic-judge.test.js
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
The optional installed-runtime contract test intercepts a synthetic local
|
|
182
|
+
model request, uses a dummy credential, and verifies an empty tool list:
|
|
183
|
+
|
|
184
|
+
```sh
|
|
185
|
+
EBO_NATIVE_CODEX_CONTRACT=1 node --test dist/test/codex-judge.test.js
|
|
186
|
+
EBO_LIVE_CODEX_JUDGE_SMOKE=1 EBO_LIVE_CODEX_JUDGE_MODEL='<existing-route>' \
|
|
187
|
+
node --test --test-name-pattern='approved existing-auth' dist/test/codex-judge.test.js
|
|
188
|
+
```
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# Outcome ingestion and structural observations
|
|
2
|
+
|
|
3
|
+
Retained native loading accepts the implemented source versions: OpenHands
|
|
4
|
+
1.44.1/1.46.0, DeepSeek SDK 0.1.1-rc.2, and Codex 0.150.1/0.153.4. Unsupported versions
|
|
5
|
+
fail explicitly rather than receiving a different adapter's provenance.
|
|
6
|
+
Completed Codex bundles require exactly one matching owned native terminal to
|
|
7
|
+
report completion; duplicate owned terminals reject. Qualified failed/partial
|
|
8
|
+
evidence remains separate.
|
|
9
|
+
Native envelopes and physical JSONL sequences are checked before dispatch;
|
|
10
|
+
Codex notifications from a foreign or client-only source cannot become events.
|
|
11
|
+
Codex start ownership requires unique ordered client-request/server-response
|
|
12
|
+
pairs with the same JSON-RPC ID; turn requests must name the owned thread and
|
|
13
|
+
completion must follow acceptance. OpenHands
|
|
14
|
+
capture requires exactly one server-info record, and its version and
|
|
15
|
+
conversation records must agree with the manifest; completed
|
|
16
|
+
runs require one owned final conversation with `execution_status: finished`.
|
|
17
|
+
DeepSeek requires the root prompt receipt and native parent/child links for
|
|
18
|
+
related sessions, and runtime reap must follow root idle. A coarse related-session list alone cannot authorize foreign
|
|
19
|
+
records, and the retained composition must match the pinned client version.
|
|
20
|
+
Verifier task failures also require normal native completion; infrastructure
|
|
21
|
+
failures and interruptions can retain qualified partial evidence instead.
|
|
22
|
+
|
|
23
|
+
`ebo observations` derives versioned, deterministic facts from a
|
|
24
|
+
capture-qualified retained Claude Agent SDK, Codex, OpenHands, DeepSeek, Pi, or Cursor run bundle. It runs qualification,
|
|
25
|
+
normalization, native-reference integrity validation, and the registered
|
|
26
|
+
extractors in that order. Source bundles are read-only; the command rejects an
|
|
27
|
+
output path inside the source bundle or corpus.
|
|
28
|
+
|
|
29
|
+
```sh
|
|
30
|
+
ebo observations create <run-bundle-root> <output.json>
|
|
31
|
+
ebo observations corpus <corpus-root> <index.jsonl> <output-root> \
|
|
32
|
+
[--run <id>] [--attempt <id>] [--task <id>] [--model <id>] [--harness <id>] \
|
|
33
|
+
[--assessment-mode <observational|verified>]
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
The corpus command first validates the supplied deterministic index, selects
|
|
37
|
+
run manifests with the same exact-match filters as `ebo corpus query`, and
|
|
38
|
+
writes one bounded `sha256-<run-attempt-tuple>.json` file per selection. It fails rather than skipping
|
|
39
|
+
an invalid or unsupported selected bundle. `createRetainedBehaviorEvidence`
|
|
40
|
+
dispatches verified native session records to each source-specific normalizer and
|
|
41
|
+
resolver. `createRetainedStructuralObservationSet` is the corresponding public
|
|
42
|
+
library call; the older Agent SDK-specific calls remain available. The same
|
|
43
|
+
loader supplies judge input, assertion validation, calibration, and aggregation.
|
|
44
|
+
Unsupported source fields remain unavailable; native schemas and identities
|
|
45
|
+
are preserved. Supplemental bundle metadata is retained separately as
|
|
46
|
+
`outcomeCapture`, so it cannot replace source-native session records.
|
|
47
|
+
OpenHands datasets retain their captured runtime version (`1.44.1` or `1.46.0`).
|
|
48
|
+
DeepSeek reapplies its native composition, capability, initialization, prompt,
|
|
49
|
+
and completed receipt-to-idle/runtime-reap gate before normalization; qualified
|
|
50
|
+
partial captures remain qualified-with-gaps. Physical JSONL locators stay
|
|
51
|
+
unchanged, and normalization qualification cannot exceed either the structural
|
|
52
|
+
bundle gate or the source-specific gate.
|
|
53
|
+
|
|
54
|
+
Each `ebo.structural-observation/v1` states its extractor/version, exact
|
|
55
|
+
definition, one-attempt denominator, unit, uniform event IDs, native-record
|
|
56
|
+
count, and native citations. `ebo.structural-observation-set/v1` also binds the observations to
|
|
57
|
+
the normalized-dataset digest and adapter coverage report.
|
|
58
|
+
|
|
59
|
+
Definitions are deliberately mechanical:
|
|
60
|
+
|
|
61
|
+
- logical tool operations are grouped only by explicit source-native operation
|
|
62
|
+
IDs; lifecycle records without one are counted separately;
|
|
63
|
+
- repetition counts distinct operations beyond the first that share the same
|
|
64
|
+
explicit tool identity and native-input digest;
|
|
65
|
+
- failure-followed-by-operation compares only explicit failures and later
|
|
66
|
+
operation starts in the same native-order domain, split into same-tool and
|
|
67
|
+
alternate-tool counts; it does not claim recovery;
|
|
68
|
+
- validation-after-mutation requires explicit mutation records and validation
|
|
69
|
+
records in one native-order domain; tool names are never used to infer either;
|
|
70
|
+
- model requests require model-request events and native request identities;
|
|
71
|
+
assistant messages and model reroutes are not requests;
|
|
72
|
+
- cumulative resource snapshots select only the latest snapshot in one known
|
|
73
|
+
native-order domain, increments are summed only when every record declares
|
|
74
|
+
increment semantics, and a cumulative-final record is used directly;
|
|
75
|
+
- native token categories stay separate, native total tokens are never rebuilt
|
|
76
|
+
from components, and cost does not imply subscription utilization;
|
|
77
|
+
- compaction counts include only native records that explicitly identify a
|
|
78
|
+
compaction boundary.
|
|
79
|
+
|
|
80
|
+
An observed zero is emitted only with available family coverage. Partial or
|
|
81
|
+
unsupported capability, ambiguous identity, unknown order, overlapping usage,
|
|
82
|
+
or missing timing remains `unavailable` with a reason. Observational runs have
|
|
83
|
+
no verifier assertion records and make no task-pass claim. Verified runs retain
|
|
84
|
+
each assertion outcome with its native verifier citation.
|