engineering-behavior-observatory 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +151 -0
- package/contracts/codex-app-server-0.150.1/manifest.json +25 -0
- package/contracts/codex-app-server-0.150.1/schema/InitializeParams.json +84 -0
- package/contracts/codex-app-server-0.150.1/schema/JSONRPCMessage.json +137 -0
- package/contracts/codex-app-server-0.150.1/schema/v2/ThreadReadParams.json +17 -0
- package/contracts/codex-app-server-0.150.1/schema/v2/ThreadTokenUsageUpdatedNotification.json +82 -0
- package/contracts/codex-app-server-0.150.1/schema/v2/TurnInterruptParams.json +17 -0
- package/contracts/codex-app-server-0.150.1/types/AskForApproval.ts +5 -0
- package/contracts/codex-app-server-0.150.1/types/ClientNotification.ts +5 -0
- package/contracts/codex-app-server-0.150.1/types/SandboxMode.ts +5 -0
- package/contracts/codex-app-server-0.150.1/types/ThreadReadParams.ts +9 -0
- package/contracts/codex-app-server-0.150.1/types/TokenUsageBreakdown.ts +5 -0
- package/contracts/codex-app-server-0.150.1/types/TurnInterruptParams.ts +5 -0
- package/contracts/codex-app-server-0.153.4/manifest.json +27 -0
- package/contracts/codex-app-server-0.153.4/schema/JSONRPCMessage.json +137 -0
- package/contracts/codex-app-server-0.153.4/schema/v1/InitializeParams.json +84 -0
- package/contracts/codex-app-server-0.153.4/schema/v2/ThreadReadParams.json +17 -0
- package/contracts/codex-app-server-0.153.4/schema/v2/ThreadStartParams.json +515 -0
- package/contracts/codex-app-server-0.153.4/schema/v2/ThreadTokenUsageUpdatedNotification.json +82 -0
- package/contracts/codex-app-server-0.153.4/schema/v2/TurnInterruptParams.json +17 -0
- package/contracts/codex-app-server-0.153.4/schema/v2/TurnStartParams.json +921 -0
- package/contracts/codex-app-server-0.153.4/types/AskForApproval.ts +5 -0
- package/contracts/codex-app-server-0.153.4/types/ClientNotification.ts +5 -0
- package/contracts/codex-app-server-0.153.4/types/SandboxMode.ts +5 -0
- package/contracts/codex-app-server-0.153.4/types/ThreadReadParams.ts +12 -0
- package/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.ts +5 -0
- package/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.ts +5 -0
- package/contracts/openhands-agent-server-v1.44.1.json +65 -0
- package/contracts/openhands-agent-server-v1.46.0.json +46 -0
- package/dist/contracts/codex-app-server-0.153.4/types/AskForApproval.d.ts +9 -0
- package/dist/contracts/codex-app-server-0.153.4/types/AskForApproval.js +2 -0
- package/dist/contracts/codex-app-server-0.153.4/types/ClientNotification.d.ts +3 -0
- package/dist/contracts/codex-app-server-0.153.4/types/ClientNotification.js +2 -0
- package/dist/contracts/codex-app-server-0.153.4/types/SandboxMode.d.ts +1 -0
- package/dist/contracts/codex-app-server-0.153.4/types/SandboxMode.js +2 -0
- package/dist/contracts/codex-app-server-0.153.4/types/ThreadReadParams.d.ts +10 -0
- package/dist/contracts/codex-app-server-0.153.4/types/ThreadReadParams.js +2 -0
- package/dist/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.d.ts +8 -0
- package/dist/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.js +2 -0
- package/dist/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.d.ts +4 -0
- package/dist/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.js +2 -0
- package/dist/src/agent-sdk-normalizer.d.ts +22 -0
- package/dist/src/agent-sdk-normalizer.js +995 -0
- package/dist/src/agent-sdk-run.d.ts +31 -0
- package/dist/src/agent-sdk-run.js +242 -0
- package/dist/src/agent-sdk-runner.d.ts +96 -0
- package/dist/src/agent-sdk-runner.js +407 -0
- package/dist/src/agent-sdk.d.ts +294 -0
- package/dist/src/agent-sdk.js +644 -0
- package/dist/src/aggregation.d.ts +183 -0
- package/dist/src/aggregation.js +513 -0
- package/dist/src/artifacts.d.ts +36 -0
- package/dist/src/artifacts.js +1372 -0
- package/dist/src/atlas-grafana.d.ts +377 -0
- package/dist/src/atlas-grafana.js +47 -0
- package/dist/src/atlas-html.d.ts +2 -0
- package/dist/src/atlas-html.js +34 -0
- package/dist/src/atlas.d.ts +130 -0
- package/dist/src/atlas.js +235 -0
- package/dist/src/behavior-assertions.d.ts +95 -0
- package/dist/src/behavior-assertions.js +154 -0
- package/dist/src/cli.d.ts +5 -0
- package/dist/src/cli.js +957 -0
- package/dist/src/codex-judge.d.ts +44 -0
- package/dist/src/codex-judge.js +214 -0
- package/dist/src/codex-run.d.ts +114 -0
- package/dist/src/codex-run.js +519 -0
- package/dist/src/codex.d.ts +187 -0
- package/dist/src/codex.js +1028 -0
- package/dist/src/contracts.d.ts +155 -0
- package/dist/src/contracts.js +866 -0
- package/dist/src/corpus.d.ts +67 -0
- package/dist/src/corpus.js +540 -0
- package/dist/src/cursor-sdk-runner.d.ts +81 -0
- package/dist/src/cursor-sdk-runner.js +362 -0
- package/dist/src/cursor-sdk.d.ts +81 -0
- package/dist/src/cursor-sdk.js +1078 -0
- package/dist/src/deepseek-adapter.d.ts +152 -0
- package/dist/src/deepseek-adapter.js +777 -0
- package/dist/src/exports.d.ts +69 -0
- package/dist/src/exports.js +966 -0
- package/dist/src/human-calibration.d.ts +167 -0
- package/dist/src/human-calibration.js +618 -0
- package/dist/src/index.d.ts +71 -0
- package/dist/src/index.js +38 -0
- package/dist/src/lifecycle.d.ts +196 -0
- package/dist/src/lifecycle.js +1889 -0
- package/dist/src/normalization-integrity.d.ts +128 -0
- package/dist/src/normalization-integrity.js +429 -0
- package/dist/src/openhands-run.d.ts +36 -0
- package/dist/src/openhands-run.js +415 -0
- package/dist/src/openhands.d.ts +130 -0
- package/dist/src/openhands.js +859 -0
- package/dist/src/pi.d.ts +203 -0
- package/dist/src/pi.js +1345 -0
- package/dist/src/process-protocol.d.ts +245 -0
- package/dist/src/process-protocol.js +1261 -0
- package/dist/src/retained-evidence.d.ts +12 -0
- package/dist/src/retained-evidence.js +201 -0
- package/dist/src/run-bundles.d.ts +193 -0
- package/dist/src/run-bundles.js +993 -0
- package/dist/src/scheduler.d.ts +109 -0
- package/dist/src/scheduler.js +1080 -0
- package/dist/src/semantic-judge.d.ts +194 -0
- package/dist/src/semantic-judge.js +875 -0
- package/dist/src/structural-observations.d.ts +124 -0
- package/dist/src/structural-observations.js +671 -0
- package/dist/src/task-packets.d.ts +115 -0
- package/dist/src/task-packets.js +683 -0
- package/dist/src/uniform-events.d.ts +120 -0
- package/dist/src/uniform-events.js +158 -0
- package/dist/src/verifiers.d.ts +116 -0
- package/dist/src/verifiers.js +819 -0
- package/dist/src/workspaces.d.ts +56 -0
- package/dist/src/workspaces.js +1314 -0
- package/docs/README.md +60 -0
- package/docs/development/README.md +71 -0
- package/docs/development/documentation-sync.md +58 -0
- package/docs/development/extension-contracts.md +307 -0
- package/docs/evaluation/README.md +48 -0
- package/docs/evaluation/aggregation.md +112 -0
- package/docs/evaluation/behavior-assertions.md +45 -0
- package/docs/evaluation/human-calibration.md +170 -0
- package/docs/evaluation/normalization-integrity.md +80 -0
- package/docs/evaluation/semantic-judge.md +188 -0
- package/docs/evaluation/structural-observations.md +84 -0
- package/docs/evaluation/uniform-events.md +131 -0
- package/docs/guides/atlas.md +144 -0
- package/docs/guides/evidence-and-sharing.md +100 -0
- package/docs/guides/operator-guide.md +424 -0
- package/docs/guides/quickstart.md +152 -0
- package/docs/guides/telemetry.md +66 -0
- package/docs/harnesses/README.md +31 -0
- package/docs/harnesses/claude-agent-sdk.md +61 -0
- package/docs/harnesses/codex-harness.md +174 -0
- package/docs/harnesses/cursor-sdk.md +149 -0
- package/docs/harnesses/deepseek-harness.md +134 -0
- package/docs/harnesses/openhands-agent-server.md +153 -0
- package/docs/harnesses/pi-sdk.md +66 -0
- package/docs/reference/README.md +19 -0
- package/docs/reference/agent-sdk-operational-runner.md +118 -0
- package/docs/reference/cli.md +114 -0
- package/docs/reference/contracts.md +222 -0
- package/docs/reference/run-bundle-contract.md +354 -0
- package/docs/reference/run-lifecycle.md +54 -0
- package/examples/cursor-sdk/README.md +11 -0
- package/examples/cursor-sdk/capture-profile.json +1 -0
- package/examples/cursor-sdk/harness.json +1 -0
- package/examples/cursor-sdk/model.json +1 -0
- package/examples/cursor-sdk/native-limits.json +1 -0
- package/examples/cursor-sdk/native-tool-policy.json +1 -0
- package/ontology/behavior-categories.v1.json +46 -0
- package/package.json +66 -0
- package/release/0.1.0/KNOWN_LIMITATIONS.md +35 -0
- package/release/0.1.0/README.md +46 -0
- package/release/0.1.0/reproducibility.json +61 -0
- package/release/0.2.0/KNOWN_LIMITATIONS.md +40 -0
- package/release/0.2.0/README.md +42 -0
- package/release/0.2.0/reproducibility.json +73 -0
- package/release/0.2.1/KNOWN_LIMITATIONS.md +18 -0
- package/release/0.2.1/README.md +63 -0
- package/release/0.2.1/reproducibility.json +73 -0
- package/release/README.md +13 -0
- package/schemas/aggregation.v1.json +293 -0
- package/schemas/behavior-assertions.v1.json +220 -0
- package/schemas/deepseek-runtime-composition.v1.schema.json +122 -0
- package/schemas/experiment.v1.schema.json +188 -0
- package/schemas/human-calibration.v1.json +336 -0
- package/schemas/normalization-integrity.v1.json +302 -0
- package/schemas/run-bundles/v1.json +1040 -0
- package/schemas/run-queue.v1.schema.json +206 -0
- package/schemas/semantic-judge.v1.json +380 -0
- package/schemas/structural-observations.v1.json +131 -0
- package/schemas/task-packet-freeze.v1.schema.json +106 -0
- package/schemas/task-packet.v1.schema.json +234 -0
- package/schemas/uniform-events/v1.json +289 -0
- package/scripts/atlas-grafana.sh +18 -0
package/docs/README.md
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# EBO documentation
|
|
2
|
+
|
|
3
|
+
Start with the question you need to answer. Operational guides explain what to
|
|
4
|
+
do; reference pages describe the exact contracts. All shell examples assume
|
|
5
|
+
the `ebo` command is installed as described in the quickstart.
|
|
6
|
+
|
|
7
|
+
## Start here
|
|
8
|
+
|
|
9
|
+
1. [Quickstart](guides/quickstart.md): install EBO and open a synthetic
|
|
10
|
+
report without credentials; then prepare for a real capture.
|
|
11
|
+
2. [Choose a harness](harnesses/README.md): compare execution boundaries,
|
|
12
|
+
evidence coverage, and prerequisites.
|
|
13
|
+
3. [Operator guide](guides/operator-guide.md): prepare, freeze, execute,
|
|
14
|
+
inspect, export, and evaluate a task.
|
|
15
|
+
|
|
16
|
+
## Operate and inspect
|
|
17
|
+
|
|
18
|
+
- [Evidence and sharing](guides/evidence-and-sharing.md): which files to read,
|
|
19
|
+
how to interpret a partial run, and how to create a portable archive.
|
|
20
|
+
- [Telemetry](guides/telemetry.md): native events versus OTLP, collector receipt,
|
|
21
|
+
Grafana, and resource-accounting caveats.
|
|
22
|
+
- [Behavior Atlas](guides/atlas.md): cohort filters, cited evidence, Grafana
|
|
23
|
+
provisioning, and static reports.
|
|
24
|
+
|
|
25
|
+
## Harness guides
|
|
26
|
+
|
|
27
|
+
[Claude Agent SDK](harnesses/claude-agent-sdk.md) ·
|
|
28
|
+
[Codex](harnesses/codex-harness.md) · [Cursor](harnesses/cursor-sdk.md) ·
|
|
29
|
+
[Pi](harnesses/pi-sdk.md) · [OpenHands](harnesses/openhands-agent-server.md) ·
|
|
30
|
+
[DeepSeek Harness](harnesses/deepseek-harness.md)
|
|
31
|
+
|
|
32
|
+
These guides own runtime pins, configuration, native evidence, and known gaps.
|
|
33
|
+
|
|
34
|
+
## Evaluate behavior
|
|
35
|
+
|
|
36
|
+
The [evaluation reading path](evaluation/README.md) explains how the parts fit:
|
|
37
|
+
|
|
38
|
+
- [Uniform events](evaluation/uniform-events.md) and
|
|
39
|
+
[normalization integrity](evaluation/normalization-integrity.md)
|
|
40
|
+
- [Structural observations](evaluation/structural-observations.md)
|
|
41
|
+
- [Behavior assertions](evaluation/behavior-assertions.md) and
|
|
42
|
+
[semantic judging](evaluation/semantic-judge.md)
|
|
43
|
+
- [Human calibration](evaluation/human-calibration.md)
|
|
44
|
+
- [Comparison and aggregation](evaluation/aggregation.md)
|
|
45
|
+
|
|
46
|
+
## Build and look things up
|
|
47
|
+
|
|
48
|
+
- [Reference index](reference/README.md): grouped CLI syntax, packet/experiment
|
|
49
|
+
contracts, run bundles, lifecycle, and Agent SDK configuration.
|
|
50
|
+
- [Contributor guide](development/README.md): checks, documentation ownership,
|
|
51
|
+
and release preparation.
|
|
52
|
+
- [Extension contracts](development/extension-contracts.md): adapter, extractor,
|
|
53
|
+
rubric, verifier, and export-policy development.
|
|
54
|
+
- [Documentation sync findings](development/documentation-sync.md): why generated
|
|
55
|
+
change provenance is kept separate from public documentation.
|
|
56
|
+
- [Release records](../release/README.md): versioned verification and limitations.
|
|
57
|
+
|
|
58
|
+
For agents: read the repository's `AGENTS.md`, then the relevant guide and
|
|
59
|
+
contract. CLI syntax is in [one grouped reference](reference/cli.md), not
|
|
60
|
+
scattered through release history.
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# Contribute to EBO
|
|
2
|
+
|
|
3
|
+
Start with the repository's `AGENTS.md`, then
|
|
4
|
+
[the documentation index](../README.md). The TypeScript coordinator owns
|
|
5
|
+
evidence capture and evaluation; each harness retains its native protocol and
|
|
6
|
+
semantics.
|
|
7
|
+
|
|
8
|
+
## Set up and check
|
|
9
|
+
|
|
10
|
+
Use the pinned Node version and build before invoking the checkout's CLI:
|
|
11
|
+
|
|
12
|
+
```sh
|
|
13
|
+
nvm use
|
|
14
|
+
npm ci
|
|
15
|
+
npm run build
|
|
16
|
+
npm link
|
|
17
|
+
ebo --help
|
|
18
|
+
npm run typecheck
|
|
19
|
+
npm test
|
|
20
|
+
git diff --check
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
The test suite is deterministic by default. Live provider smokes are opt-in,
|
|
24
|
+
consume external capacity, and require approved credentials. Their instructions
|
|
25
|
+
belong in the relevant harness guide. Do not treat a skipped live test as proof
|
|
26
|
+
of provider access.
|
|
27
|
+
|
|
28
|
+
Use [extension contracts](extension-contracts.md) when adding an adapter,
|
|
29
|
+
structural extractor, rubric, verifier, or export policy. Preserve native
|
|
30
|
+
records and qualified partial attempts before adding derived views.
|
|
31
|
+
|
|
32
|
+
## Documentation ownership
|
|
33
|
+
|
|
34
|
+
- **README:** motivation, harness overview, installation, quickstart, navigation.
|
|
35
|
+
- **Guides:** executable workflows and recovery.
|
|
36
|
+
- **Harnesses:** source-specific setup, pins, policies, evidence, and limitations.
|
|
37
|
+
- **Evaluation:** methods, semantics, interpretation boundaries.
|
|
38
|
+
- **Reference:** exact syntax and artifact contracts.
|
|
39
|
+
- **Release records:** version-specific changes, verification, and limitations.
|
|
40
|
+
|
|
41
|
+
Before adding a page, check whether an existing page already owns the topic.
|
|
42
|
+
Link every new page from a relevant index. Update relative links when moving
|
|
43
|
+
documents, including links from release records and packaged examples.
|
|
44
|
+
Use `ebo` in operator commands; keep Node invocations for tests and scripts
|
|
45
|
+
that are not CLI commands. Do not duplicate the complete CLI in the README.
|
|
46
|
+
|
|
47
|
+
Run a cold-reader check: can a newcomer follow the commands without knowing
|
|
48
|
+
which placeholders, working directory, credentials, or prior artifacts they
|
|
49
|
+
need? Keep runnable examples separate from syntax templates.
|
|
50
|
+
|
|
51
|
+
## Memory and documentation sync
|
|
52
|
+
|
|
53
|
+
OpenSymphony memory remains available for implementation provenance. In this
|
|
54
|
+
repository its generated topic notes are private and excluded from the package.
|
|
55
|
+
Public docs are authored from current implementation and verified behavior,
|
|
56
|
+
not generated issue lists.
|
|
57
|
+
|
|
58
|
+
`opensymphony memory sync-docs` updates private notes; it no longer writes
|
|
59
|
+
the public guide tree. After a feature change, use those notes to find evidence
|
|
60
|
+
and deliberately update the relevant authored guide. See
|
|
61
|
+
[the sync failure analysis](documentation-sync.md).
|
|
62
|
+
|
|
63
|
+
## Release gate
|
|
64
|
+
|
|
65
|
+
From a **clean** checkout, `npm run acceptance` runs the deterministic suite,
|
|
66
|
+
checks fixture digests and documentation links, scans the package, and verifies
|
|
67
|
+
two byte-identical package builds. It writes under
|
|
68
|
+
`.ebo/releases/<package-version>/`; it does not publish or tag.
|
|
69
|
+
|
|
70
|
+
See [release records](../../release/README.md). A documentation edit after a
|
|
71
|
+
tag belongs to a later change; do not move an existing release tag to include it.
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# Documentation sync: failure signals and local policy
|
|
2
|
+
|
|
3
|
+
## What failed
|
|
4
|
+
|
|
5
|
+
The public docs tree had 58 Markdown files. After removing managed sync blocks
|
|
6
|
+
and frontmatter, **39 contained only a title**. They were provenance summaries,
|
|
7
|
+
not documentation a reader could use.
|
|
8
|
+
|
|
9
|
+
Examples included `adapters.md` and `documentation.md`. Their managed
|
|
10
|
+
sections listed issue contributions, generic invariants, and statements that
|
|
11
|
+
no gotchas or diagrams had been inferred. They provided no operational model,
|
|
12
|
+
configuration, example, or recovery instructions.
|
|
13
|
+
|
|
14
|
+
Area associations also spread across unrelated topics: foundation and SDK
|
|
15
|
+
integration issues appeared in many area lists. A high mapping confidence and
|
|
16
|
+
a clean public-link lint did not establish topical relevance or usefulness.
|
|
17
|
+
|
|
18
|
+
## Why it matters
|
|
19
|
+
|
|
20
|
+
- Alphabetical navigation gave empty pages the same prominence as real guides.
|
|
21
|
+
- Repeated contribution lists obscured substantive content.
|
|
22
|
+
- `Docs pending: 0` meant the sync ran, not that user-facing behavior was
|
|
23
|
+
documented.
|
|
24
|
+
- Incremental syncing encouraged append-only maintenance rather than designing
|
|
25
|
+
an entry point around reader tasks.
|
|
26
|
+
|
|
27
|
+
## EBO's containment change
|
|
28
|
+
|
|
29
|
+
The repository memory configuration routes generated topic notes to the
|
|
30
|
+
ignored private memory tree and sets their visibility to private. Public
|
|
31
|
+
managed blocks and title-only pages are removed. Authored guides are grouped
|
|
32
|
+
by purpose and linked from a documentation index.
|
|
33
|
+
|
|
34
|
+
Memory capture and provenance are preserved. The broad learned area mappings
|
|
35
|
+
are not repaired by this change; their output is simply kept out of public
|
|
36
|
+
docs. No OpenSymphony engine behavior is changed.
|
|
37
|
+
|
|
38
|
+
Future captures may learn mappings again. Inspect sync's dry-run target paths
|
|
39
|
+
before applying it, and keep public authored guides out of generated targets.
|
|
40
|
+
Do not rerun initialization with force merely to regenerate public pages.
|
|
41
|
+
|
|
42
|
+
## Candidate acceptance checks for a future OpenSymphony fix
|
|
43
|
+
|
|
44
|
+
These are proposed product checks, not implemented EBO features:
|
|
45
|
+
|
|
46
|
+
1. A page consisting only of a title and generated boilerplate should remain
|
|
47
|
+
private provenance, not become public documentation.
|
|
48
|
+
2. Topic relevance should come from narrow source evidence; shared milestone
|
|
49
|
+
membership alone should not spread every issue across every area.
|
|
50
|
+
3. A sync report should distinguish provenance updated, authored documentation
|
|
51
|
+
reviewed, and actual user-facing coverage.
|
|
52
|
+
4. Generated output should identify concrete behaviors and limitations, or
|
|
53
|
+
state that human authoring is needed without publishing an empty page.
|
|
54
|
+
5. Validation should test whether an intended reader can perform a documented
|
|
55
|
+
action, in addition to checking links and privacy.
|
|
56
|
+
|
|
57
|
+
Use this case as a reproducible failure example when revisiting sync-docs.
|
|
58
|
+
Do not replace the issue lists with invented implementation prose.
|
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
# Extension contracts
|
|
2
|
+
|
|
3
|
+
EBO extensions are ordinary, explicit TypeScript registrations backed by
|
|
4
|
+
contract tests. There is no dynamic package discovery, plugin registry, or
|
|
5
|
+
universal harness protocol. Add a loader only if separately installed
|
|
6
|
+
third-party extensions become a demonstrated requirement.
|
|
7
|
+
|
|
8
|
+
Start with `npm ci && npm run build`. Every command below runs a checked-in
|
|
9
|
+
contract fixture with Node's test runner. The snippets are minimal shapes;
|
|
10
|
+
copy the nearby production adapter/test that matches the source boundary and
|
|
11
|
+
replace only source-owned fields.
|
|
12
|
+
|
|
13
|
+
## Harness capture and normalization adapter
|
|
14
|
+
|
|
15
|
+
Implement a `HarnessAdapter<Request, NativeRecord>` with separate capture and
|
|
16
|
+
normalization members, then register it explicitly:
|
|
17
|
+
|
|
18
|
+
```ts
|
|
19
|
+
import {
|
|
20
|
+
AdapterRegistry,
|
|
21
|
+
assertAdapterContract,
|
|
22
|
+
type HarnessAdapter,
|
|
23
|
+
} from "engineering-behavior-observatory";
|
|
24
|
+
|
|
25
|
+
type NativeRecord = { kind: string };
|
|
26
|
+
|
|
27
|
+
const adapter: HarnessAdapter<Request, NativeRecord> = {
|
|
28
|
+
capture: {
|
|
29
|
+
id: "example-adapter",
|
|
30
|
+
harness: "example-harness",
|
|
31
|
+
capture: async (request) => captureNativeRecords(request),
|
|
32
|
+
},
|
|
33
|
+
normalization: {
|
|
34
|
+
id: "example-adapter",
|
|
35
|
+
harness: "example-harness",
|
|
36
|
+
capabilityProfile,
|
|
37
|
+
normalize: async (capture) => projectQualifiedCapture(capture),
|
|
38
|
+
},
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
await assertAdapterContract(adapter, request, nativeEvidenceResolver);
|
|
42
|
+
const adapters = new AdapterRegistry([adapter]);
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
Evidence obligations:
|
|
46
|
+
|
|
47
|
+
- persist native records before projection and give every mapped event a
|
|
48
|
+
resolvable `artifactId` plus `recordLocator`;
|
|
49
|
+
- normalize only qualified or qualified-with-gaps capture;
|
|
50
|
+
- declare every native type and available/partial/unsupported capability;
|
|
51
|
+
- keep unknown, unsupported, and missing values explicit;
|
|
52
|
+
- retain every native record as mapped or explicitly unmapped; and
|
|
53
|
+
- preserve source run/attempt identity, ordering domains, timestamps,
|
|
54
|
+
parentage, content references, and terminal semantics without inference.
|
|
55
|
+
|
|
56
|
+
Runnable contract fixture:
|
|
57
|
+
|
|
58
|
+
```sh
|
|
59
|
+
node --test --test-name-pattern='minimal capture and normalization adapter contract' \
|
|
60
|
+
dist/test/uniform-events.test.js
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Current source-specific examples are the Agent SDK normalizer,
|
|
64
|
+
`createOpenHandsHarnessAdapter`, `createDeepSeekHarnessAdapter`, and
|
|
65
|
+
`createCodexHarnessAdapter`; `createCursorSdkBehaviorEvidence` applies the same
|
|
66
|
+
native-first contract to the direct SDK boundary. A new adapter must not weaken
|
|
67
|
+
their rules to fit a common convenience model.
|
|
68
|
+
|
|
69
|
+
The Pi integration uses direct public SDK session/history/subscription APIs in
|
|
70
|
+
`src/pi.ts`, without forcing Pi into a process protocol.
|
|
71
|
+
|
|
72
|
+
## Source-specific process or API boundary
|
|
73
|
+
|
|
74
|
+
Use `runProtocolProcess` only for a source whose owned child actually speaks
|
|
75
|
+
newline-delimited JSON on stdout:
|
|
76
|
+
|
|
77
|
+
```ts
|
|
78
|
+
import { runProtocolProcess } from "engineering-behavior-observatory";
|
|
79
|
+
|
|
80
|
+
const result = await runProtocolProcess({
|
|
81
|
+
command: runtime,
|
|
82
|
+
args: ["serve"],
|
|
83
|
+
source: "example-harness",
|
|
84
|
+
evidencePath: "evidence/protocol.jsonl",
|
|
85
|
+
stderrPath: "evidence/stderr.log",
|
|
86
|
+
onFrame: async (frame, recorder) => {
|
|
87
|
+
await recorder.recordNotification({
|
|
88
|
+
source: "example-harness",
|
|
89
|
+
method: nativeMethod(frame),
|
|
90
|
+
payload: frame,
|
|
91
|
+
});
|
|
92
|
+
},
|
|
93
|
+
});
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
This reusable layer owns bounded frame parsing, append-only JSONL recording,
|
|
97
|
+
stderr capture, interruption, teardown, and partial evidence. The adapter still
|
|
98
|
+
owns source method schemas, request/response correlation, session identities,
|
|
99
|
+
capabilities, and completion evidence. A clean process exit is not a source
|
|
100
|
+
completion event.
|
|
101
|
+
|
|
102
|
+
DeepSeek integrations must use the official
|
|
103
|
+
`@deepseek-ai/dsh-sdk-client` and `@deepseek-ai/dsh-sdk-protocol` packages. The
|
|
104
|
+
official client owns JSON-RPC framing, transport, and teardown; EBO records its
|
|
105
|
+
typed calls/notifications rather than recreating the protocol client with
|
|
106
|
+
`runProtocolProcess`. OpenHands uses its pinned REST/WebSocket contract, and
|
|
107
|
+
Codex uses its pinned app-server contract; neither is forced into a generic
|
|
108
|
+
JSONL method schema.
|
|
109
|
+
|
|
110
|
+
Cursor uses direct `Agent.create({ local: ... })`, `send`, detailed callbacks,
|
|
111
|
+
stream/wait/conversation, and the exported `JsonlLocalAgentStore`. Its durable
|
|
112
|
+
store and callback channels overlap with the stream; retain them separately and
|
|
113
|
+
select one authoritative normalized source instead of counting them twice.
|
|
114
|
+
|
|
115
|
+
Evidence obligations:
|
|
116
|
+
|
|
117
|
+
- append source frames/observations in observed order before deriving state;
|
|
118
|
+
- bound stdout frames and diagnostics before parsing/retention;
|
|
119
|
+
- distinguish malformed output, recorder failure, interruption, shutdown, and
|
|
120
|
+
natural exit; and
|
|
121
|
+
- retain a readable partial record when the source terminates unexpectedly.
|
|
122
|
+
|
|
123
|
+
Runnable contract fixture:
|
|
124
|
+
|
|
125
|
+
```sh
|
|
126
|
+
node --test --test-name-pattern='records JSONL frames' \
|
|
127
|
+
dist/test/process-protocol.test.js
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
## Structural extractor
|
|
131
|
+
|
|
132
|
+
Structural extractors are deterministic entries in
|
|
133
|
+
`STRUCTURAL_EXTRACTOR_REGISTRY`, not runtime-loaded classes:
|
|
134
|
+
|
|
135
|
+
```ts
|
|
136
|
+
const registration = {
|
|
137
|
+
id: "example-explicit-tool-failure-count",
|
|
138
|
+
requiredCapabilities: ["family:tool"],
|
|
139
|
+
definition: "Distinct native tool operations with an explicit failure.",
|
|
140
|
+
} as const;
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
Add the registration and its extraction function in
|
|
144
|
+
`src/structural-observations.ts`, bump `STRUCTURAL_EXTRACTOR_VERSION`, update
|
|
145
|
+
both extractor-version constraints in
|
|
146
|
+
`schemas/structural-observations.v1.json`, then add one small golden fixture
|
|
147
|
+
case. Schema/readback changes must continue accepting already retained
|
|
148
|
+
extractor versions. Every registry or extraction-behavior change requires this
|
|
149
|
+
coordinated version update so new observations validate and retained provenance
|
|
150
|
+
stays unambiguous. Do not use a semantic heuristic for a structural fact.
|
|
151
|
+
|
|
152
|
+
Evidence obligations:
|
|
153
|
+
|
|
154
|
+
- state the denominator, unit, and exact native condition being counted;
|
|
155
|
+
- cite every contributing native record and normalized source event;
|
|
156
|
+
- deduplicate only on a source-owned stable identity;
|
|
157
|
+
- emit `unavailable` with a reason when required capability/order/identity is
|
|
158
|
+
absent; and
|
|
159
|
+
- never turn missing evidence into zero or combine cumulative usage snapshots.
|
|
160
|
+
|
|
161
|
+
Runnable contract fixtures:
|
|
162
|
+
|
|
163
|
+
```sh
|
|
164
|
+
node --test --test-name-pattern='golden structural facts|available zero' \
|
|
165
|
+
dist/test/structural-observations.test.js
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
## Rubric and semantic judge backend
|
|
169
|
+
|
|
170
|
+
A rubric is caller-owned data inside a versioned
|
|
171
|
+
`SemanticJudgeRequest`; it is not executable plugin code:
|
|
172
|
+
|
|
173
|
+
```ts
|
|
174
|
+
import type { SemanticJudgeRequest } from "engineering-behavior-observatory";
|
|
175
|
+
|
|
176
|
+
const rubric: SemanticJudgeRequest["rubric"] = {
|
|
177
|
+
id: "example-verification-rubric",
|
|
178
|
+
version: "1.0.0",
|
|
179
|
+
instructions: "Assess only whether cited evidence shows validation.",
|
|
180
|
+
};
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
The caller also selects the evaluator. Omit `backend` (or use
|
|
184
|
+
`claude-agent-sdk`) with provider `anthropic`, or use `codex-app-server` with
|
|
185
|
+
provider `openai`. Model, effort, limits, and optional executable remain
|
|
186
|
+
configuration; no automatic fallback occurs. Extend backend code only when an
|
|
187
|
+
issue explicitly requires another trusted execution boundary.
|
|
188
|
+
|
|
189
|
+
Evidence obligations:
|
|
190
|
+
|
|
191
|
+
- select bounded event and structural-observation IDs explicitly;
|
|
192
|
+
- treat packaged trajectory text as untrusted data;
|
|
193
|
+
- require citations for assessed claims plus rationale and an alternative
|
|
194
|
+
explanation;
|
|
195
|
+
- allow abstention when evidence is insufficient; and
|
|
196
|
+
- retain evaluator configuration identity without exposing secrets or
|
|
197
|
+
silently changing model/effort.
|
|
198
|
+
|
|
199
|
+
Runnable contract fixtures:
|
|
200
|
+
|
|
201
|
+
```sh
|
|
202
|
+
node --test --test-name-pattern='request schema admits exactly|packages bounded blinded' \
|
|
203
|
+
dist/test/semantic-judge.test.js
|
|
204
|
+
node --test dist/test/codex-judge.test.js
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
## Verified-task verifier
|
|
208
|
+
|
|
209
|
+
A verifier is an admitted, digest-pinned CommonJS or ESM file, not an arbitrary
|
|
210
|
+
command. It reads the private workspace snapshot path from `process.argv[2]`
|
|
211
|
+
and writes one JSON object to stdout. Compile this TypeScript example to
|
|
212
|
+
CommonJS or ESM before digesting and admitting it. Admit compiled ESM with an
|
|
213
|
+
`.mjs` locator; the operational runners treat every other locator as CommonJS:
|
|
214
|
+
|
|
215
|
+
```ts
|
|
216
|
+
import { readFileSync } from "node:fs";
|
|
217
|
+
import { join } from "node:path";
|
|
218
|
+
|
|
219
|
+
const workspace = process.argv[2];
|
|
220
|
+
const passed = readFileSync(join(workspace, "result.txt"), "utf8") === "done\n";
|
|
221
|
+
if (!passed) process.exitCode = 1;
|
|
222
|
+
process.stdout.write(JSON.stringify({
|
|
223
|
+
assertions: [{ id: "expected-result", status: passed ? "passed" : "failed" }],
|
|
224
|
+
}));
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
Evidence obligations:
|
|
228
|
+
|
|
229
|
+
- use only the admitted digest-pinned verifier and evaluated workspace
|
|
230
|
+
snapshot;
|
|
231
|
+
- emit bounded assertion IDs with `passed` or `failed` status;
|
|
232
|
+
- write diagnostics to stderr, not additional stdout records;
|
|
233
|
+
- preserve timeout/crash/malformed output as verifier error evidence; and
|
|
234
|
+
- never run a verifier for an observational packet or treat not-run as passed.
|
|
235
|
+
|
|
236
|
+
Runnable contract fixtures:
|
|
237
|
+
|
|
238
|
+
```sh
|
|
239
|
+
node --test --test-name-pattern='executes a verifier outside|preserves ESM' \
|
|
240
|
+
dist/test/verifiers.test.js
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
## Export policy
|
|
244
|
+
|
|
245
|
+
An export policy is explicit caller data passed to the existing fail-closed
|
|
246
|
+
export implementation:
|
|
247
|
+
|
|
248
|
+
```ts
|
|
249
|
+
import {
|
|
250
|
+
createPortableRunBundleExport,
|
|
251
|
+
type PortableExportPolicy,
|
|
252
|
+
} from "engineering-behavior-observatory";
|
|
253
|
+
|
|
254
|
+
const policy: PortableExportPolicy = {
|
|
255
|
+
sharingClass: "partner",
|
|
256
|
+
maxArtifactBytes: 16 * 1024 * 1024,
|
|
257
|
+
maxStringBytes: 8192,
|
|
258
|
+
sensitiveValues: [callerKnownConfidentialValue],
|
|
259
|
+
};
|
|
260
|
+
|
|
261
|
+
await createPortableRunBundleExport({
|
|
262
|
+
sourceRoot: restrictedRunBundle,
|
|
263
|
+
destinationRoot: newExportRoot,
|
|
264
|
+
policy,
|
|
265
|
+
});
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
Keep caller-known `sensitiveValues` with restricted study inputs and never
|
|
269
|
+
commit real values. See [the operator guide](../guides/operator-guide.md#5-export-an-approved-derivative)
|
|
270
|
+
for handling details.
|
|
271
|
+
|
|
272
|
+
Extend the existing sanitizer/readback only when a new artifact kind has an
|
|
273
|
+
explicit sharing classification. Do not add a permissive fallback.
|
|
274
|
+
|
|
275
|
+
Evidence obligations:
|
|
276
|
+
|
|
277
|
+
- leave the restricted source bundle unchanged;
|
|
278
|
+
- allow only known artifact kinds and classifications;
|
|
279
|
+
- remove hidden reasoning, secrets, environment values, and local identifiers;
|
|
280
|
+
- rewrite correlation IDs and verify every output digest; and
|
|
281
|
+
- rerun policy-bound readback and secret scanning before use or packing.
|
|
282
|
+
|
|
283
|
+
Runnable contract fixtures:
|
|
284
|
+
|
|
285
|
+
```sh
|
|
286
|
+
node --test --test-name-pattern='exports a sanitized|fails closed on unknown' \
|
|
287
|
+
dist/test/exports.test.js
|
|
288
|
+
```
|
|
289
|
+
|
|
290
|
+
## Contract checklist
|
|
291
|
+
|
|
292
|
+
For every extension:
|
|
293
|
+
|
|
294
|
+
1. Pin the source runtime/API/schema version and record configuration digests.
|
|
295
|
+
2. Name native evidence, completion semantics, known gaps, and unsupported
|
|
296
|
+
capabilities before mapping anything.
|
|
297
|
+
3. Preserve partial/failed attempts and immutable native evidence.
|
|
298
|
+
4. Add one focused `node:test` contract fixture beside the closest existing
|
|
299
|
+
test; use synthetic or permissively licensed data only.
|
|
300
|
+
5. Run the targeted fixture, `npm run build`, `npm run typecheck`, `npm test`,
|
|
301
|
+
and `git diff --check`.
|
|
302
|
+
6. Update [the operator guide](../guides/operator-guide.md) only if the public command or
|
|
303
|
+
operational recovery path changed.
|
|
304
|
+
|
|
305
|
+
These are implementation workflows. Task authorship, model selection, study
|
|
306
|
+
execution, human corpus review, trial counts, and partner delivery remain
|
|
307
|
+
caller-owned operations outside the extension contract.
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# Evaluate behavior
|
|
2
|
+
|
|
3
|
+
Start with a captured run and a question: did the agent validate its changes?
|
|
4
|
+
How did it respond to a tool failure? Was repeated investigation useful or
|
|
5
|
+
redundant? A tool count alone cannot answer those questions.
|
|
6
|
+
|
|
7
|
+
```mermaid
|
|
8
|
+
flowchart TD
|
|
9
|
+
A["Qualified native bundle"] --> B["Uniform events with native citations"]
|
|
10
|
+
B --> C["Structural observations"]
|
|
11
|
+
B --> D["Bounded evidence + rubric"]
|
|
12
|
+
C --> D
|
|
13
|
+
D --> E["Judge proposal or abstention"]
|
|
14
|
+
E --> F["Human review / adjudication"]
|
|
15
|
+
C --> G["Comparison and aggregation"]
|
|
16
|
+
F --> G
|
|
17
|
+
G --> H["Atlas / report"]
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Reading path
|
|
21
|
+
|
|
22
|
+
| Step | Read | Result |
|
|
23
|
+
| :--- | :--- | :--- |
|
|
24
|
+
| 1. Establish what is observable | [Uniform events](uniform-events.md), [integrity gates](normalization-integrity.md) | Source-bound events, unmapped records, capability coverage |
|
|
25
|
+
| 2. Extract exact facts | [Structural observations](structural-observations.md) | Counts, resource observations, explicit denominators and missing values |
|
|
26
|
+
| 3. Ask a behavioral question | [Assertions](behavior-assertions.md), [semantic judge](semantic-judge.md) | Cited proposal or abstention for one rubric dimension |
|
|
27
|
+
| 4. Review the interpretation | [Human calibration](human-calibration.md) | Human-authored decisions and agreement populations |
|
|
28
|
+
| 5. Compare declared conditions | [Aggregation](aggregation.md) | Descriptive distributions, matched differences, exclusions and caveats |
|
|
29
|
+
| 6. Explore or report | [Behavior Atlas](../guides/atlas.md) | Local drilldown and reproducible report |
|
|
30
|
+
|
|
31
|
+
The judge backend is independent of the evaluated harness: choose the Claude
|
|
32
|
+
Agent SDK or native Codex backend explicitly. EBO retains the evaluator, rubric,
|
|
33
|
+
evidence selection, limits, and configuration identity.
|
|
34
|
+
|
|
35
|
+
## Keep three questions separate
|
|
36
|
+
|
|
37
|
+
- **Did execution end?** The terminal record answers this.
|
|
38
|
+
- **Did we capture enough evidence?** Qualification and capability coverage
|
|
39
|
+
answer this, separately for each relevant signal.
|
|
40
|
+
- **Was the behavior useful?** A cited interpretation and review can address
|
|
41
|
+
this; execution status and counts cannot substitute for it.
|
|
42
|
+
|
|
43
|
+
A failure followed by another operation is a structural observation, not proof
|
|
44
|
+
of recovery. An absent tool family is not necessarily zero activity. Repeated
|
|
45
|
+
judge calls are not additional independent engineering trials.
|
|
46
|
+
|
|
47
|
+
Use the [operator workflow](../guides/operator-guide.md#6-normalize-and-extract-structural-observations)
|
|
48
|
+
for commands, and [CLI reference](../reference/cli.md) for complete syntax.
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
# Comparison and aggregation
|
|
2
|
+
|
|
3
|
+
`ebo aggregate build` creates a deterministic local read model from a current
|
|
4
|
+
corpus index, structural observations, proposed behavior assertions, human
|
|
5
|
+
calibration lineage, and per-measure comparison eligibility reports. It reads
|
|
6
|
+
JSON/JSONL directly and writes one derived JSON document outside the source
|
|
7
|
+
corpus; it does not alter retained bundles.
|
|
8
|
+
|
|
9
|
+
```sh
|
|
10
|
+
ebo aggregate build request.json aggregate.json
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
The request uses paths relative to `request.json`:
|
|
14
|
+
|
|
15
|
+
```json
|
|
16
|
+
{
|
|
17
|
+
"schemaVersion": "ebo.aggregation-request/v1",
|
|
18
|
+
"sources": {
|
|
19
|
+
"corpusRoot": "corpus",
|
|
20
|
+
"corpusIndex": "corpus-index.jsonl",
|
|
21
|
+
"observationSets": [{ "bundleRoot": "corpus/run-a", "path": "derived/observations/run-a.json" }],
|
|
22
|
+
"assertions": [{ "bundleRoot": "corpus/run-a", "path": "derived/assertions/assertion-a.json" }],
|
|
23
|
+
"calibrations": [{
|
|
24
|
+
"selection": "derived/review/selection.json",
|
|
25
|
+
"history": "derived/review/history.json"
|
|
26
|
+
}]
|
|
27
|
+
},
|
|
28
|
+
"groupBy": ["task", "model", "harness"],
|
|
29
|
+
"selectedAttemptPolicy": "all-attempts",
|
|
30
|
+
"recurrence": { "minimumOccurrences": 2 },
|
|
31
|
+
"comparisons": [{
|
|
32
|
+
"id": "matched-verifier-outcome",
|
|
33
|
+
"measure": "verified:verifier-passed",
|
|
34
|
+
"left": { "model": "model-a" },
|
|
35
|
+
"right": { "model": "model-b" },
|
|
36
|
+
"matchBy": ["task", "harness", "trial"],
|
|
37
|
+
"eligibilityGates": [{
|
|
38
|
+
"request": "derived/comparisons/model-a-model-b.request.json",
|
|
39
|
+
"report": "derived/comparisons/model-a-model-b.report.json"
|
|
40
|
+
}]
|
|
41
|
+
}]
|
|
42
|
+
}
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
Grouping dimensions are `task`, `model`, `harness`, `trial`, and
|
|
46
|
+
`capture-qualification`. `all-attempts` is the evidence-preserving default.
|
|
47
|
+
`trial` is the queue's retained numeric trial index; legacy bundles without it
|
|
48
|
+
remain unavailable and are excluded from trial-matched comparisons.
|
|
49
|
+
`latest-attempt-per-run` selects the highest declared attempt number regardless
|
|
50
|
+
of terminal state and reports every excluded earlier attempt; it never searches
|
|
51
|
+
for or substitutes a completed retry.
|
|
52
|
+
|
|
53
|
+
Each metric names its run, attempt, operation, assertion, or reviewed-assertion
|
|
54
|
+
population and carries numerator, denominator, units, exclusions, and a claim
|
|
55
|
+
status. Empty denominators are `unavailable`. Observational completion is only
|
|
56
|
+
a terminal-state measure; task-success rates come only from available verified
|
|
57
|
+
attempt outcomes. Duplicate identical inputs are counted once, while conflicting
|
|
58
|
+
records with the same identity fail the build.
|
|
59
|
+
|
|
60
|
+
Each group's additive `behaviors` array retains constructive, adverse, mixed,
|
|
61
|
+
and context-dependent distributions separately from review agreement. A
|
|
62
|
+
partition names the exact vocabulary/category/dimension, rubric id/version,
|
|
63
|
+
and evaluator identity/version/configuration digest. Missing legacy evaluator
|
|
64
|
+
configuration stays a separate partition. Its denominator is distinct
|
|
65
|
+
confirmed attempt-dimensions, not judge calls: agreeing confirmed reruns count
|
|
66
|
+
once and conflicting confirmed reruns exclude that attempt. Disputed,
|
|
67
|
+
unreviewed, rejected, and abstained assertions cannot contribute; assertion
|
|
68
|
+
references expose review outcome and inclusion alongside run/attempt/digest.
|
|
69
|
+
An empty denominator is unavailable. These distributions remain descriptive.
|
|
70
|
+
|
|
71
|
+
Corpus indexing prefers `structuralQualification.status` over legacy
|
|
72
|
+
`qualification`. Legacy-only known states remain readable; missing or invalid
|
|
73
|
+
structural status is unavailable. Qualified-with-gaps, incomplete, unqualified,
|
|
74
|
+
and unavailable remain distinct. Rebuild older corpus indexes before using
|
|
75
|
+
them; source manifests and native artifacts are never rewritten.
|
|
76
|
+
|
|
77
|
+
Every report retains the logical request and corpus-index digests plus the
|
|
78
|
+
exact admitted manifest, observation-set, assertion, calibration, and
|
|
79
|
+
comparison-gate digests. Moving the report therefore does not detach its
|
|
80
|
+
metrics from their rebuild inputs.
|
|
81
|
+
|
|
82
|
+
Observation and assertion sources name their retained bundle so the build can
|
|
83
|
+
recompute structural observations, resolve assertion citations, and reject
|
|
84
|
+
stale derived evidence. Calibration selections are likewise reloaded from their
|
|
85
|
+
digest-bound sources before their review history contributes to a metric.
|
|
86
|
+
Legacy v1 structural sets without the optional declared capability profile stay
|
|
87
|
+
readable; aggregation and semantic judging rebuild that profile from the native
|
|
88
|
+
bundle before use.
|
|
89
|
+
|
|
90
|
+
Matched comparisons require one candidate on each side for every caller-chosen
|
|
91
|
+
match key and one eligibility report whose candidate IDs are those two run IDs.
|
|
92
|
+
Missing, ambiguous, unavailable, or ungated pairs are exclusions. Supported
|
|
93
|
+
measures are `attempt:infrastructure-failure`,
|
|
94
|
+
`attempt:terminal-completed`, `verified:verifier-passed`, and any numeric
|
|
95
|
+
`structural:<extractor-id>` observation. The supplied comparison gate applies
|
|
96
|
+
only to that candidate pair, must name the exact measure, and must include the
|
|
97
|
+
measure's required capability;
|
|
98
|
+
an unsupported gate makes the comparison
|
|
99
|
+
unavailable, and partial-capability or declared-condition caveats remain in the
|
|
100
|
+
output.
|
|
101
|
+
|
|
102
|
+
Each eligibility report carries the digest of its source comparison request.
|
|
103
|
+
The aggregate build reloads that request and recomputes the report before using
|
|
104
|
+
the gate, so an edited or stale report cannot authorize a matched difference.
|
|
105
|
+
These bindings use `ebo.comparison-request/v2` and
|
|
106
|
+
`ebo.comparison-report/v2`; the original v1 request/report remain supported by
|
|
107
|
+
`ebo comparison check` but do not contain enough lineage for aggregation.
|
|
108
|
+
|
|
109
|
+
The caller supplies the recurrence threshold. A divergent matched unit below
|
|
110
|
+
that threshold is a `case-study`; reaching it yields only a
|
|
111
|
+
`recurring-description`. Neither state establishes causality or statistical
|
|
112
|
+
significance, and the report never emits a composite model ranking.
|