engineering-behavior-observatory 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +151 -0
  3. package/contracts/codex-app-server-0.150.1/manifest.json +25 -0
  4. package/contracts/codex-app-server-0.150.1/schema/InitializeParams.json +84 -0
  5. package/contracts/codex-app-server-0.150.1/schema/JSONRPCMessage.json +137 -0
  6. package/contracts/codex-app-server-0.150.1/schema/v2/ThreadReadParams.json +17 -0
  7. package/contracts/codex-app-server-0.150.1/schema/v2/ThreadTokenUsageUpdatedNotification.json +82 -0
  8. package/contracts/codex-app-server-0.150.1/schema/v2/TurnInterruptParams.json +17 -0
  9. package/contracts/codex-app-server-0.150.1/types/AskForApproval.ts +5 -0
  10. package/contracts/codex-app-server-0.150.1/types/ClientNotification.ts +5 -0
  11. package/contracts/codex-app-server-0.150.1/types/SandboxMode.ts +5 -0
  12. package/contracts/codex-app-server-0.150.1/types/ThreadReadParams.ts +9 -0
  13. package/contracts/codex-app-server-0.150.1/types/TokenUsageBreakdown.ts +5 -0
  14. package/contracts/codex-app-server-0.150.1/types/TurnInterruptParams.ts +5 -0
  15. package/contracts/codex-app-server-0.153.4/manifest.json +27 -0
  16. package/contracts/codex-app-server-0.153.4/schema/JSONRPCMessage.json +137 -0
  17. package/contracts/codex-app-server-0.153.4/schema/v1/InitializeParams.json +84 -0
  18. package/contracts/codex-app-server-0.153.4/schema/v2/ThreadReadParams.json +17 -0
  19. package/contracts/codex-app-server-0.153.4/schema/v2/ThreadStartParams.json +515 -0
  20. package/contracts/codex-app-server-0.153.4/schema/v2/ThreadTokenUsageUpdatedNotification.json +82 -0
  21. package/contracts/codex-app-server-0.153.4/schema/v2/TurnInterruptParams.json +17 -0
  22. package/contracts/codex-app-server-0.153.4/schema/v2/TurnStartParams.json +921 -0
  23. package/contracts/codex-app-server-0.153.4/types/AskForApproval.ts +5 -0
  24. package/contracts/codex-app-server-0.153.4/types/ClientNotification.ts +5 -0
  25. package/contracts/codex-app-server-0.153.4/types/SandboxMode.ts +5 -0
  26. package/contracts/codex-app-server-0.153.4/types/ThreadReadParams.ts +12 -0
  27. package/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.ts +5 -0
  28. package/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.ts +5 -0
  29. package/contracts/openhands-agent-server-v1.44.1.json +65 -0
  30. package/contracts/openhands-agent-server-v1.46.0.json +46 -0
  31. package/dist/contracts/codex-app-server-0.153.4/types/AskForApproval.d.ts +9 -0
  32. package/dist/contracts/codex-app-server-0.153.4/types/AskForApproval.js +2 -0
  33. package/dist/contracts/codex-app-server-0.153.4/types/ClientNotification.d.ts +3 -0
  34. package/dist/contracts/codex-app-server-0.153.4/types/ClientNotification.js +2 -0
  35. package/dist/contracts/codex-app-server-0.153.4/types/SandboxMode.d.ts +1 -0
  36. package/dist/contracts/codex-app-server-0.153.4/types/SandboxMode.js +2 -0
  37. package/dist/contracts/codex-app-server-0.153.4/types/ThreadReadParams.d.ts +10 -0
  38. package/dist/contracts/codex-app-server-0.153.4/types/ThreadReadParams.js +2 -0
  39. package/dist/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.d.ts +8 -0
  40. package/dist/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.js +2 -0
  41. package/dist/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.d.ts +4 -0
  42. package/dist/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.js +2 -0
  43. package/dist/src/agent-sdk-normalizer.d.ts +22 -0
  44. package/dist/src/agent-sdk-normalizer.js +995 -0
  45. package/dist/src/agent-sdk-run.d.ts +31 -0
  46. package/dist/src/agent-sdk-run.js +242 -0
  47. package/dist/src/agent-sdk-runner.d.ts +96 -0
  48. package/dist/src/agent-sdk-runner.js +407 -0
  49. package/dist/src/agent-sdk.d.ts +294 -0
  50. package/dist/src/agent-sdk.js +644 -0
  51. package/dist/src/aggregation.d.ts +183 -0
  52. package/dist/src/aggregation.js +513 -0
  53. package/dist/src/artifacts.d.ts +36 -0
  54. package/dist/src/artifacts.js +1372 -0
  55. package/dist/src/atlas-grafana.d.ts +377 -0
  56. package/dist/src/atlas-grafana.js +47 -0
  57. package/dist/src/atlas-html.d.ts +2 -0
  58. package/dist/src/atlas-html.js +34 -0
  59. package/dist/src/atlas.d.ts +130 -0
  60. package/dist/src/atlas.js +235 -0
  61. package/dist/src/behavior-assertions.d.ts +95 -0
  62. package/dist/src/behavior-assertions.js +154 -0
  63. package/dist/src/cli.d.ts +5 -0
  64. package/dist/src/cli.js +957 -0
  65. package/dist/src/codex-judge.d.ts +44 -0
  66. package/dist/src/codex-judge.js +214 -0
  67. package/dist/src/codex-run.d.ts +114 -0
  68. package/dist/src/codex-run.js +519 -0
  69. package/dist/src/codex.d.ts +187 -0
  70. package/dist/src/codex.js +1028 -0
  71. package/dist/src/contracts.d.ts +155 -0
  72. package/dist/src/contracts.js +866 -0
  73. package/dist/src/corpus.d.ts +67 -0
  74. package/dist/src/corpus.js +540 -0
  75. package/dist/src/cursor-sdk-runner.d.ts +81 -0
  76. package/dist/src/cursor-sdk-runner.js +362 -0
  77. package/dist/src/cursor-sdk.d.ts +81 -0
  78. package/dist/src/cursor-sdk.js +1078 -0
  79. package/dist/src/deepseek-adapter.d.ts +152 -0
  80. package/dist/src/deepseek-adapter.js +777 -0
  81. package/dist/src/exports.d.ts +69 -0
  82. package/dist/src/exports.js +966 -0
  83. package/dist/src/human-calibration.d.ts +167 -0
  84. package/dist/src/human-calibration.js +618 -0
  85. package/dist/src/index.d.ts +71 -0
  86. package/dist/src/index.js +38 -0
  87. package/dist/src/lifecycle.d.ts +196 -0
  88. package/dist/src/lifecycle.js +1889 -0
  89. package/dist/src/normalization-integrity.d.ts +128 -0
  90. package/dist/src/normalization-integrity.js +429 -0
  91. package/dist/src/openhands-run.d.ts +36 -0
  92. package/dist/src/openhands-run.js +415 -0
  93. package/dist/src/openhands.d.ts +130 -0
  94. package/dist/src/openhands.js +859 -0
  95. package/dist/src/pi.d.ts +203 -0
  96. package/dist/src/pi.js +1345 -0
  97. package/dist/src/process-protocol.d.ts +245 -0
  98. package/dist/src/process-protocol.js +1261 -0
  99. package/dist/src/retained-evidence.d.ts +12 -0
  100. package/dist/src/retained-evidence.js +201 -0
  101. package/dist/src/run-bundles.d.ts +193 -0
  102. package/dist/src/run-bundles.js +993 -0
  103. package/dist/src/scheduler.d.ts +109 -0
  104. package/dist/src/scheduler.js +1080 -0
  105. package/dist/src/semantic-judge.d.ts +194 -0
  106. package/dist/src/semantic-judge.js +875 -0
  107. package/dist/src/structural-observations.d.ts +124 -0
  108. package/dist/src/structural-observations.js +671 -0
  109. package/dist/src/task-packets.d.ts +115 -0
  110. package/dist/src/task-packets.js +683 -0
  111. package/dist/src/uniform-events.d.ts +120 -0
  112. package/dist/src/uniform-events.js +158 -0
  113. package/dist/src/verifiers.d.ts +116 -0
  114. package/dist/src/verifiers.js +819 -0
  115. package/dist/src/workspaces.d.ts +56 -0
  116. package/dist/src/workspaces.js +1314 -0
  117. package/docs/README.md +60 -0
  118. package/docs/development/README.md +71 -0
  119. package/docs/development/documentation-sync.md +58 -0
  120. package/docs/development/extension-contracts.md +307 -0
  121. package/docs/evaluation/README.md +48 -0
  122. package/docs/evaluation/aggregation.md +112 -0
  123. package/docs/evaluation/behavior-assertions.md +45 -0
  124. package/docs/evaluation/human-calibration.md +170 -0
  125. package/docs/evaluation/normalization-integrity.md +80 -0
  126. package/docs/evaluation/semantic-judge.md +188 -0
  127. package/docs/evaluation/structural-observations.md +84 -0
  128. package/docs/evaluation/uniform-events.md +131 -0
  129. package/docs/guides/atlas.md +144 -0
  130. package/docs/guides/evidence-and-sharing.md +100 -0
  131. package/docs/guides/operator-guide.md +424 -0
  132. package/docs/guides/quickstart.md +152 -0
  133. package/docs/guides/telemetry.md +66 -0
  134. package/docs/harnesses/README.md +31 -0
  135. package/docs/harnesses/claude-agent-sdk.md +61 -0
  136. package/docs/harnesses/codex-harness.md +174 -0
  137. package/docs/harnesses/cursor-sdk.md +149 -0
  138. package/docs/harnesses/deepseek-harness.md +134 -0
  139. package/docs/harnesses/openhands-agent-server.md +153 -0
  140. package/docs/harnesses/pi-sdk.md +66 -0
  141. package/docs/reference/README.md +19 -0
  142. package/docs/reference/agent-sdk-operational-runner.md +118 -0
  143. package/docs/reference/cli.md +114 -0
  144. package/docs/reference/contracts.md +222 -0
  145. package/docs/reference/run-bundle-contract.md +354 -0
  146. package/docs/reference/run-lifecycle.md +54 -0
  147. package/examples/cursor-sdk/README.md +11 -0
  148. package/examples/cursor-sdk/capture-profile.json +1 -0
  149. package/examples/cursor-sdk/harness.json +1 -0
  150. package/examples/cursor-sdk/model.json +1 -0
  151. package/examples/cursor-sdk/native-limits.json +1 -0
  152. package/examples/cursor-sdk/native-tool-policy.json +1 -0
  153. package/ontology/behavior-categories.v1.json +46 -0
  154. package/package.json +66 -0
  155. package/release/0.1.0/KNOWN_LIMITATIONS.md +35 -0
  156. package/release/0.1.0/README.md +46 -0
  157. package/release/0.1.0/reproducibility.json +61 -0
  158. package/release/0.2.0/KNOWN_LIMITATIONS.md +40 -0
  159. package/release/0.2.0/README.md +42 -0
  160. package/release/0.2.0/reproducibility.json +73 -0
  161. package/release/0.2.1/KNOWN_LIMITATIONS.md +18 -0
  162. package/release/0.2.1/README.md +63 -0
  163. package/release/0.2.1/reproducibility.json +73 -0
  164. package/release/README.md +13 -0
  165. package/schemas/aggregation.v1.json +293 -0
  166. package/schemas/behavior-assertions.v1.json +220 -0
  167. package/schemas/deepseek-runtime-composition.v1.schema.json +122 -0
  168. package/schemas/experiment.v1.schema.json +188 -0
  169. package/schemas/human-calibration.v1.json +336 -0
  170. package/schemas/normalization-integrity.v1.json +302 -0
  171. package/schemas/run-bundles/v1.json +1040 -0
  172. package/schemas/run-queue.v1.schema.json +206 -0
  173. package/schemas/semantic-judge.v1.json +380 -0
  174. package/schemas/structural-observations.v1.json +131 -0
  175. package/schemas/task-packet-freeze.v1.schema.json +106 -0
  176. package/schemas/task-packet.v1.schema.json +234 -0
  177. package/schemas/uniform-events/v1.json +289 -0
  178. package/scripts/atlas-grafana.sh +18 -0
package/docs/README.md ADDED
@@ -0,0 +1,60 @@
1
+ # EBO documentation
2
+
3
+ Start with the question you need to answer. Operational guides explain what to
4
+ do; reference pages describe the exact contracts. All shell examples assume
5
+ the `ebo` command is installed as described in the quickstart.
6
+
7
+ ## Start here
8
+
9
+ 1. [Quickstart](guides/quickstart.md): install EBO and open a synthetic
10
+ report without credentials; then prepare for a real capture.
11
+ 2. [Choose a harness](harnesses/README.md): compare execution boundaries,
12
+ evidence coverage, and prerequisites.
13
+ 3. [Operator guide](guides/operator-guide.md): prepare, freeze, execute,
14
+ inspect, export, and evaluate a task.
15
+
16
+ ## Operate and inspect
17
+
18
+ - [Evidence and sharing](guides/evidence-and-sharing.md): which files to read,
19
+ how to interpret a partial run, and how to create a portable archive.
20
+ - [Telemetry](guides/telemetry.md): native events versus OTLP, collector receipt,
21
+ Grafana, and resource-accounting caveats.
22
+ - [Behavior Atlas](guides/atlas.md): cohort filters, cited evidence, Grafana
23
+ provisioning, and static reports.
24
+
25
+ ## Harness guides
26
+
27
+ [Claude Agent SDK](harnesses/claude-agent-sdk.md) ·
28
+ [Codex](harnesses/codex-harness.md) · [Cursor](harnesses/cursor-sdk.md) ·
29
+ [Pi](harnesses/pi-sdk.md) · [OpenHands](harnesses/openhands-agent-server.md) ·
30
+ [DeepSeek Harness](harnesses/deepseek-harness.md)
31
+
32
+ These guides own runtime pins, configuration, native evidence, and known gaps.
33
+
34
+ ## Evaluate behavior
35
+
36
+ The [evaluation reading path](evaluation/README.md) explains how the parts fit:
37
+
38
+ - [Uniform events](evaluation/uniform-events.md) and
39
+ [normalization integrity](evaluation/normalization-integrity.md)
40
+ - [Structural observations](evaluation/structural-observations.md)
41
+ - [Behavior assertions](evaluation/behavior-assertions.md) and
42
+ [semantic judging](evaluation/semantic-judge.md)
43
+ - [Human calibration](evaluation/human-calibration.md)
44
+ - [Comparison and aggregation](evaluation/aggregation.md)
45
+
46
+ ## Build and look things up
47
+
48
+ - [Reference index](reference/README.md): grouped CLI syntax, packet/experiment
49
+ contracts, run bundles, lifecycle, and Agent SDK configuration.
50
+ - [Contributor guide](development/README.md): checks, documentation ownership,
51
+ and release preparation.
52
+ - [Extension contracts](development/extension-contracts.md): adapter, extractor,
53
+ rubric, verifier, and export-policy development.
54
+ - [Documentation sync findings](development/documentation-sync.md): why generated
55
+ change provenance is kept separate from public documentation.
56
+ - [Release records](../release/README.md): versioned verification and limitations.
57
+
58
+ For agents: read the repository's `AGENTS.md`, then the relevant guide and
59
+ contract. CLI syntax is in [one grouped reference](reference/cli.md), not
60
+ scattered through release history.
@@ -0,0 +1,71 @@
1
+ # Contribute to EBO
2
+
3
+ Start with the repository's `AGENTS.md`, then
4
+ [the documentation index](../README.md). The TypeScript coordinator owns
5
+ evidence capture and evaluation; each harness retains its native protocol and
6
+ semantics.
7
+
8
+ ## Set up and check
9
+
10
+ Use the pinned Node version and build before invoking the checkout's CLI:
11
+
12
+ ```sh
13
+ nvm use
14
+ npm ci
15
+ npm run build
16
+ npm link
17
+ ebo --help
18
+ npm run typecheck
19
+ npm test
20
+ git diff --check
21
+ ```
22
+
23
+ The test suite is deterministic by default. Live provider smokes are opt-in,
24
+ consume external capacity, and require approved credentials. Their instructions
25
+ belong in the relevant harness guide. Do not treat a skipped live test as proof
26
+ of provider access.
27
+
28
+ Use [extension contracts](extension-contracts.md) when adding an adapter,
29
+ structural extractor, rubric, verifier, or export policy. Preserve native
30
+ records and qualified partial attempts before adding derived views.
31
+
32
+ ## Documentation ownership
33
+
34
+ - **README:** motivation, harness overview, installation, quickstart, navigation.
35
+ - **Guides:** executable workflows and recovery.
36
+ - **Harnesses:** source-specific setup, pins, policies, evidence, and limitations.
37
+ - **Evaluation:** methods, semantics, interpretation boundaries.
38
+ - **Reference:** exact syntax and artifact contracts.
39
+ - **Release records:** version-specific changes, verification, and limitations.
40
+
41
+ Before adding a page, check whether an existing page already owns the topic.
42
+ Link every new page from a relevant index. Update relative links when moving
43
+ documents, including links from release records and packaged examples.
44
+ Use `ebo` in operator commands; keep Node invocations for tests and scripts
45
+ that are not CLI commands. Do not duplicate the complete CLI in the README.
46
+
47
+ Run a cold-reader check: can a newcomer follow the commands without knowing
48
+ which placeholders, working directory, credentials, or prior artifacts they
49
+ need? Keep runnable examples separate from syntax templates.
50
+
51
+ ## Memory and documentation sync
52
+
53
+ OpenSymphony memory remains available for implementation provenance. In this
54
+ repository its generated topic notes are private and excluded from the package.
55
+ Public docs are authored from current implementation and verified behavior,
56
+ not generated issue lists.
57
+
58
+ `opensymphony memory sync-docs` updates private notes; it no longer writes
59
+ the public guide tree. After a feature change, use those notes to find evidence
60
+ and deliberately update the relevant authored guide. See
61
+ [the sync failure analysis](documentation-sync.md).
62
+
63
+ ## Release gate
64
+
65
+ From a **clean** checkout, `npm run acceptance` runs the deterministic suite,
66
+ checks fixture digests and documentation links, scans the package, and verifies
67
+ two byte-identical package builds. It writes under
68
+ `.ebo/releases/<package-version>/`; it does not publish or tag.
69
+
70
+ See [release records](../../release/README.md). A documentation edit after a
71
+ tag belongs to a later change; do not move an existing release tag to include it.
@@ -0,0 +1,58 @@
1
+ # Documentation sync: failure signals and local policy
2
+
3
+ ## What failed
4
+
5
+ The public docs tree had 58 Markdown files. After removing managed sync blocks
6
+ and frontmatter, **39 contained only a title**. They were provenance summaries,
7
+ not documentation a reader could use.
8
+
9
+ Examples included `adapters.md` and `documentation.md`. Their managed
10
+ sections listed issue contributions, generic invariants, and statements that
11
+ no gotchas or diagrams had been inferred. They provided no operational model,
12
+ configuration, example, or recovery instructions.
13
+
14
+ Area associations also spread across unrelated topics: foundation and SDK
15
+ integration issues appeared in many area lists. A high mapping confidence and
16
+ a clean public-link lint did not establish topical relevance or usefulness.
17
+
18
+ ## Why it matters
19
+
20
+ - Alphabetical navigation gave empty pages the same prominence as real guides.
21
+ - Repeated contribution lists obscured substantive content.
22
+ - `Docs pending: 0` meant the sync ran, not that user-facing behavior was
23
+ documented.
24
+ - Incremental syncing encouraged append-only maintenance rather than designing
25
+ an entry point around reader tasks.
26
+
27
+ ## EBO's containment change
28
+
29
+ The repository memory configuration routes generated topic notes to the
30
+ ignored private memory tree and sets their visibility to private. Public
31
+ managed blocks and title-only pages are removed. Authored guides are grouped
32
+ by purpose and linked from a documentation index.
33
+
34
+ Memory capture and provenance are preserved. The broad learned area mappings
35
+ are not repaired by this change; their output is simply kept out of public
36
+ docs. No OpenSymphony engine behavior is changed.
37
+
38
+ Future captures may learn mappings again. Inspect sync's dry-run target paths
39
+ before applying it, and keep public authored guides out of generated targets.
40
+ Do not rerun initialization with force merely to regenerate public pages.
41
+
42
+ ## Candidate acceptance checks for a future OpenSymphony fix
43
+
44
+ These are proposed product checks, not implemented EBO features:
45
+
46
+ 1. A page consisting only of a title and generated boilerplate should remain
47
+ private provenance, not become public documentation.
48
+ 2. Topic relevance should come from narrow source evidence; shared milestone
49
+ membership alone should not spread every issue across every area.
50
+ 3. A sync report should distinguish provenance updated, authored documentation
51
+ reviewed, and actual user-facing coverage.
52
+ 4. Generated output should identify concrete behaviors and limitations, or
53
+ state that human authoring is needed without publishing an empty page.
54
+ 5. Validation should test whether an intended reader can perform a documented
55
+ action, in addition to checking links and privacy.
56
+
57
+ Use this case as a reproducible failure example when revisiting sync-docs.
58
+ Do not replace the issue lists with invented implementation prose.
@@ -0,0 +1,307 @@
1
+ # Extension contracts
2
+
3
+ EBO extensions are ordinary, explicit TypeScript registrations backed by
4
+ contract tests. There is no dynamic package discovery, plugin registry, or
5
+ universal harness protocol. Add a loader only if separately installed
6
+ third-party extensions become a demonstrated requirement.
7
+
8
+ Start with `npm ci && npm run build`. Every command below runs a checked-in
9
+ contract fixture with Node's test runner. The snippets are minimal shapes;
10
+ copy the nearby production adapter/test that matches the source boundary and
11
+ replace only source-owned fields.
12
+
13
+ ## Harness capture and normalization adapter
14
+
15
+ Implement a `HarnessAdapter<Request, NativeRecord>` with separate capture and
16
+ normalization members, then register it explicitly:
17
+
18
+ ```ts
19
+ import {
20
+ AdapterRegistry,
21
+ assertAdapterContract,
22
+ type HarnessAdapter,
23
+ } from "engineering-behavior-observatory";
24
+
25
+ type NativeRecord = { kind: string };
26
+
27
+ const adapter: HarnessAdapter<Request, NativeRecord> = {
28
+ capture: {
29
+ id: "example-adapter",
30
+ harness: "example-harness",
31
+ capture: async (request) => captureNativeRecords(request),
32
+ },
33
+ normalization: {
34
+ id: "example-adapter",
35
+ harness: "example-harness",
36
+ capabilityProfile,
37
+ normalize: async (capture) => projectQualifiedCapture(capture),
38
+ },
39
+ };
40
+
41
+ await assertAdapterContract(adapter, request, nativeEvidenceResolver);
42
+ const adapters = new AdapterRegistry([adapter]);
43
+ ```
44
+
45
+ Evidence obligations:
46
+
47
+ - persist native records before projection and give every mapped event a
48
+ resolvable `artifactId` plus `recordLocator`;
49
+ - normalize only qualified or qualified-with-gaps capture;
50
+ - declare every native type and available/partial/unsupported capability;
51
+ - keep unknown, unsupported, and missing values explicit;
52
+ - retain every native record as mapped or explicitly unmapped; and
53
+ - preserve source run/attempt identity, ordering domains, timestamps,
54
+ parentage, content references, and terminal semantics without inference.
55
+
56
+ Runnable contract fixture:
57
+
58
+ ```sh
59
+ node --test --test-name-pattern='minimal capture and normalization adapter contract' \
60
+ dist/test/uniform-events.test.js
61
+ ```
62
+
63
+ Current source-specific examples are the Agent SDK normalizer,
64
+ `createOpenHandsHarnessAdapter`, `createDeepSeekHarnessAdapter`, and
65
+ `createCodexHarnessAdapter`; `createCursorSdkBehaviorEvidence` applies the same
66
+ native-first contract to the direct SDK boundary. A new adapter must not weaken
67
+ their rules to fit a common convenience model.
68
+
69
+ The Pi integration uses direct public SDK session/history/subscription APIs in
70
+ `src/pi.ts`, without forcing Pi into a process protocol.
71
+
72
+ ## Source-specific process or API boundary
73
+
74
+ Use `runProtocolProcess` only for a source whose owned child actually speaks
75
+ newline-delimited JSON on stdout:
76
+
77
+ ```ts
78
+ import { runProtocolProcess } from "engineering-behavior-observatory";
79
+
80
+ const result = await runProtocolProcess({
81
+ command: runtime,
82
+ args: ["serve"],
83
+ source: "example-harness",
84
+ evidencePath: "evidence/protocol.jsonl",
85
+ stderrPath: "evidence/stderr.log",
86
+ onFrame: async (frame, recorder) => {
87
+ await recorder.recordNotification({
88
+ source: "example-harness",
89
+ method: nativeMethod(frame),
90
+ payload: frame,
91
+ });
92
+ },
93
+ });
94
+ ```
95
+
96
+ This reusable layer owns bounded frame parsing, append-only JSONL recording,
97
+ stderr capture, interruption, teardown, and partial evidence. The adapter still
98
+ owns source method schemas, request/response correlation, session identities,
99
+ capabilities, and completion evidence. A clean process exit is not a source
100
+ completion event.
101
+
102
+ DeepSeek integrations must use the official
103
+ `@deepseek-ai/dsh-sdk-client` and `@deepseek-ai/dsh-sdk-protocol` packages. The
104
+ official client owns JSON-RPC framing, transport, and teardown; EBO records its
105
+ typed calls/notifications rather than recreating the protocol client with
106
+ `runProtocolProcess`. OpenHands uses its pinned REST/WebSocket contract, and
107
+ Codex uses its pinned app-server contract; neither is forced into a generic
108
+ JSONL method schema.
109
+
110
+ Cursor uses direct `Agent.create({ local: ... })`, `send`, detailed callbacks,
111
+ stream/wait/conversation, and the exported `JsonlLocalAgentStore`. Its durable
112
+ store and callback channels overlap with the stream; retain them separately and
113
+ select one authoritative normalized source instead of counting them twice.
114
+
115
+ Evidence obligations:
116
+
117
+ - append source frames/observations in observed order before deriving state;
118
+ - bound stdout frames and diagnostics before parsing/retention;
119
+ - distinguish malformed output, recorder failure, interruption, shutdown, and
120
+ natural exit; and
121
+ - retain a readable partial record when the source terminates unexpectedly.
122
+
123
+ Runnable contract fixture:
124
+
125
+ ```sh
126
+ node --test --test-name-pattern='records JSONL frames' \
127
+ dist/test/process-protocol.test.js
128
+ ```
129
+
130
+ ## Structural extractor
131
+
132
+ Structural extractors are deterministic entries in
133
+ `STRUCTURAL_EXTRACTOR_REGISTRY`, not runtime-loaded classes:
134
+
135
+ ```ts
136
+ const registration = {
137
+ id: "example-explicit-tool-failure-count",
138
+ requiredCapabilities: ["family:tool"],
139
+ definition: "Distinct native tool operations with an explicit failure.",
140
+ } as const;
141
+ ```
142
+
143
+ Add the registration and its extraction function in
144
+ `src/structural-observations.ts`, bump `STRUCTURAL_EXTRACTOR_VERSION`, update
145
+ both extractor-version constraints in
146
+ `schemas/structural-observations.v1.json`, then add one small golden fixture
147
+ case. Schema/readback changes must continue accepting already retained
148
+ extractor versions. Every registry or extraction-behavior change requires this
149
+ coordinated version update so new observations validate and retained provenance
150
+ stays unambiguous. Do not use a semantic heuristic for a structural fact.
151
+
152
+ Evidence obligations:
153
+
154
+ - state the denominator, unit, and exact native condition being counted;
155
+ - cite every contributing native record and normalized source event;
156
+ - deduplicate only on a source-owned stable identity;
157
+ - emit `unavailable` with a reason when required capability/order/identity is
158
+ absent; and
159
+ - never turn missing evidence into zero or combine cumulative usage snapshots.
160
+
161
+ Runnable contract fixtures:
162
+
163
+ ```sh
164
+ node --test --test-name-pattern='golden structural facts|available zero' \
165
+ dist/test/structural-observations.test.js
166
+ ```
167
+
168
+ ## Rubric and semantic judge backend
169
+
170
+ A rubric is caller-owned data inside a versioned
171
+ `SemanticJudgeRequest`; it is not executable plugin code:
172
+
173
+ ```ts
174
+ import type { SemanticJudgeRequest } from "engineering-behavior-observatory";
175
+
176
+ const rubric: SemanticJudgeRequest["rubric"] = {
177
+ id: "example-verification-rubric",
178
+ version: "1.0.0",
179
+ instructions: "Assess only whether cited evidence shows validation.",
180
+ };
181
+ ```
182
+
183
+ The caller also selects the evaluator. Omit `backend` (or use
184
+ `claude-agent-sdk`) with provider `anthropic`, or use `codex-app-server` with
185
+ provider `openai`. Model, effort, limits, and optional executable remain
186
+ configuration; no automatic fallback occurs. Extend backend code only when an
187
+ issue explicitly requires another trusted execution boundary.
188
+
189
+ Evidence obligations:
190
+
191
+ - select bounded event and structural-observation IDs explicitly;
192
+ - treat packaged trajectory text as untrusted data;
193
+ - require citations for assessed claims plus rationale and an alternative
194
+ explanation;
195
+ - allow abstention when evidence is insufficient; and
196
+ - retain evaluator configuration identity without exposing secrets or
197
+ silently changing model/effort.
198
+
199
+ Runnable contract fixtures:
200
+
201
+ ```sh
202
+ node --test --test-name-pattern='request schema admits exactly|packages bounded blinded' \
203
+ dist/test/semantic-judge.test.js
204
+ node --test dist/test/codex-judge.test.js
205
+ ```
206
+
207
+ ## Verified-task verifier
208
+
209
+ A verifier is an admitted, digest-pinned CommonJS or ESM file, not an arbitrary
210
+ command. It reads the private workspace snapshot path from `process.argv[2]`
211
+ and writes one JSON object to stdout. Compile this TypeScript example to
212
+ CommonJS or ESM before digesting and admitting it. Admit compiled ESM with an
213
+ `.mjs` locator; the operational runners treat every other locator as CommonJS:
214
+
215
+ ```ts
216
+ import { readFileSync } from "node:fs";
217
+ import { join } from "node:path";
218
+
219
+ const workspace = process.argv[2];
220
+ const passed = readFileSync(join(workspace, "result.txt"), "utf8") === "done\n";
221
+ if (!passed) process.exitCode = 1;
222
+ process.stdout.write(JSON.stringify({
223
+ assertions: [{ id: "expected-result", status: passed ? "passed" : "failed" }],
224
+ }));
225
+ ```
226
+
227
+ Evidence obligations:
228
+
229
+ - use only the admitted digest-pinned verifier and evaluated workspace
230
+ snapshot;
231
+ - emit bounded assertion IDs with `passed` or `failed` status;
232
+ - write diagnostics to stderr, not additional stdout records;
233
+ - preserve timeout/crash/malformed output as verifier error evidence; and
234
+ - never run a verifier for an observational packet or treat not-run as passed.
235
+
236
+ Runnable contract fixtures:
237
+
238
+ ```sh
239
+ node --test --test-name-pattern='executes a verifier outside|preserves ESM' \
240
+ dist/test/verifiers.test.js
241
+ ```
242
+
243
+ ## Export policy
244
+
245
+ An export policy is explicit caller data passed to the existing fail-closed
246
+ export implementation:
247
+
248
+ ```ts
249
+ import {
250
+ createPortableRunBundleExport,
251
+ type PortableExportPolicy,
252
+ } from "engineering-behavior-observatory";
253
+
254
+ const policy: PortableExportPolicy = {
255
+ sharingClass: "partner",
256
+ maxArtifactBytes: 16 * 1024 * 1024,
257
+ maxStringBytes: 8192,
258
+ sensitiveValues: [callerKnownConfidentialValue],
259
+ };
260
+
261
+ await createPortableRunBundleExport({
262
+ sourceRoot: restrictedRunBundle,
263
+ destinationRoot: newExportRoot,
264
+ policy,
265
+ });
266
+ ```
267
+
268
+ Keep caller-known `sensitiveValues` with restricted study inputs and never
269
+ commit real values. See [the operator guide](../guides/operator-guide.md#5-export-an-approved-derivative)
270
+ for handling details.
271
+
272
+ Extend the existing sanitizer/readback only when a new artifact kind has an
273
+ explicit sharing classification. Do not add a permissive fallback.
274
+
275
+ Evidence obligations:
276
+
277
+ - leave the restricted source bundle unchanged;
278
+ - allow only known artifact kinds and classifications;
279
+ - remove hidden reasoning, secrets, environment values, and local identifiers;
280
+ - rewrite correlation IDs and verify every output digest; and
281
+ - rerun policy-bound readback and secret scanning before use or packing.
282
+
283
+ Runnable contract fixtures:
284
+
285
+ ```sh
286
+ node --test --test-name-pattern='exports a sanitized|fails closed on unknown' \
287
+ dist/test/exports.test.js
288
+ ```
289
+
290
+ ## Contract checklist
291
+
292
+ For every extension:
293
+
294
+ 1. Pin the source runtime/API/schema version and record configuration digests.
295
+ 2. Name native evidence, completion semantics, known gaps, and unsupported
296
+ capabilities before mapping anything.
297
+ 3. Preserve partial/failed attempts and immutable native evidence.
298
+ 4. Add one focused `node:test` contract fixture beside the closest existing
299
+ test; use synthetic or permissively licensed data only.
300
+ 5. Run the targeted fixture, `npm run build`, `npm run typecheck`, `npm test`,
301
+ and `git diff --check`.
302
+ 6. Update [the operator guide](../guides/operator-guide.md) only if the public command or
303
+ operational recovery path changed.
304
+
305
+ These are implementation workflows. Task authorship, model selection, study
306
+ execution, human corpus review, trial counts, and partner delivery remain
307
+ caller-owned operations outside the extension contract.
@@ -0,0 +1,48 @@
1
+ # Evaluate behavior
2
+
3
+ Start with a captured run and a question: did the agent validate its changes?
4
+ How did it respond to a tool failure? Was repeated investigation useful or
5
+ redundant? A tool count alone cannot answer those questions.
6
+
7
+ ```mermaid
8
+ flowchart TD
9
+ A["Qualified native bundle"] --> B["Uniform events with native citations"]
10
+ B --> C["Structural observations"]
11
+ B --> D["Bounded evidence + rubric"]
12
+ C --> D
13
+ D --> E["Judge proposal or abstention"]
14
+ E --> F["Human review / adjudication"]
15
+ C --> G["Comparison and aggregation"]
16
+ F --> G
17
+ G --> H["Atlas / report"]
18
+ ```
19
+
20
+ ## Reading path
21
+
22
+ | Step | Read | Result |
23
+ | :--- | :--- | :--- |
24
+ | 1. Establish what is observable | [Uniform events](uniform-events.md), [integrity gates](normalization-integrity.md) | Source-bound events, unmapped records, capability coverage |
25
+ | 2. Extract exact facts | [Structural observations](structural-observations.md) | Counts, resource observations, explicit denominators and missing values |
26
+ | 3. Ask a behavioral question | [Assertions](behavior-assertions.md), [semantic judge](semantic-judge.md) | Cited proposal or abstention for one rubric dimension |
27
+ | 4. Review the interpretation | [Human calibration](human-calibration.md) | Human-authored decisions and agreement populations |
28
+ | 5. Compare declared conditions | [Aggregation](aggregation.md) | Descriptive distributions, matched differences, exclusions and caveats |
29
+ | 6. Explore or report | [Behavior Atlas](../guides/atlas.md) | Local drilldown and reproducible report |
30
+
31
+ The judge backend is independent of the evaluated harness: choose the Claude
32
+ Agent SDK or native Codex backend explicitly. EBO retains the evaluator, rubric,
33
+ evidence selection, limits, and configuration identity.
34
+
35
+ ## Keep three questions separate
36
+
37
+ - **Did execution end?** The terminal record answers this.
38
+ - **Did we capture enough evidence?** Qualification and capability coverage
39
+ answer this, separately for each relevant signal.
40
+ - **Was the behavior useful?** A cited interpretation and review can address
41
+ this; execution status and counts cannot substitute for it.
42
+
43
+ A failure followed by another operation is a structural observation, not proof
44
+ of recovery. An absent tool family is not necessarily zero activity. Repeated
45
+ judge calls are not additional independent engineering trials.
46
+
47
+ Use the [operator workflow](../guides/operator-guide.md#6-normalize-and-extract-structural-observations)
48
+ for commands, and [CLI reference](../reference/cli.md) for complete syntax.
@@ -0,0 +1,112 @@
1
+ # Comparison and aggregation
2
+
3
+ `ebo aggregate build` creates a deterministic local read model from a current
4
+ corpus index, structural observations, proposed behavior assertions, human
5
+ calibration lineage, and per-measure comparison eligibility reports. It reads
6
+ JSON/JSONL directly and writes one derived JSON document outside the source
7
+ corpus; it does not alter retained bundles.
8
+
9
+ ```sh
10
+ ebo aggregate build request.json aggregate.json
11
+ ```
12
+
13
+ The request uses paths relative to `request.json`:
14
+
15
+ ```json
16
+ {
17
+ "schemaVersion": "ebo.aggregation-request/v1",
18
+ "sources": {
19
+ "corpusRoot": "corpus",
20
+ "corpusIndex": "corpus-index.jsonl",
21
+ "observationSets": [{ "bundleRoot": "corpus/run-a", "path": "derived/observations/run-a.json" }],
22
+ "assertions": [{ "bundleRoot": "corpus/run-a", "path": "derived/assertions/assertion-a.json" }],
23
+ "calibrations": [{
24
+ "selection": "derived/review/selection.json",
25
+ "history": "derived/review/history.json"
26
+ }]
27
+ },
28
+ "groupBy": ["task", "model", "harness"],
29
+ "selectedAttemptPolicy": "all-attempts",
30
+ "recurrence": { "minimumOccurrences": 2 },
31
+ "comparisons": [{
32
+ "id": "matched-verifier-outcome",
33
+ "measure": "verified:verifier-passed",
34
+ "left": { "model": "model-a" },
35
+ "right": { "model": "model-b" },
36
+ "matchBy": ["task", "harness", "trial"],
37
+ "eligibilityGates": [{
38
+ "request": "derived/comparisons/model-a-model-b.request.json",
39
+ "report": "derived/comparisons/model-a-model-b.report.json"
40
+ }]
41
+ }]
42
+ }
43
+ ```
44
+
45
+ Grouping dimensions are `task`, `model`, `harness`, `trial`, and
46
+ `capture-qualification`. `all-attempts` is the evidence-preserving default.
47
+ `trial` is the queue's retained numeric trial index; legacy bundles without it
48
+ remain unavailable and are excluded from trial-matched comparisons.
49
+ `latest-attempt-per-run` selects the highest declared attempt number regardless
50
+ of terminal state and reports every excluded earlier attempt; it never searches
51
+ for or substitutes a completed retry.
52
+
53
+ Each metric names its run, attempt, operation, assertion, or reviewed-assertion
54
+ population and carries numerator, denominator, units, exclusions, and a claim
55
+ status. Empty denominators are `unavailable`. Observational completion is only
56
+ a terminal-state measure; task-success rates come only from available verified
57
+ attempt outcomes. Duplicate identical inputs are counted once, while conflicting
58
+ records with the same identity fail the build.
59
+
60
+ Each group's additive `behaviors` array retains constructive, adverse, mixed,
61
+ and context-dependent distributions separately from review agreement. A
62
+ partition names the exact vocabulary/category/dimension, rubric id/version,
63
+ and evaluator identity/version/configuration digest. Missing legacy evaluator
64
+ configuration stays a separate partition. Its denominator is distinct
65
+ confirmed attempt-dimensions, not judge calls: agreeing confirmed reruns count
66
+ once and conflicting confirmed reruns exclude that attempt. Disputed,
67
+ unreviewed, rejected, and abstained assertions cannot contribute; assertion
68
+ references expose review outcome and inclusion alongside run/attempt/digest.
69
+ An empty denominator is unavailable. These distributions remain descriptive.
70
+
71
+ Corpus indexing prefers `structuralQualification.status` over legacy
72
+ `qualification`. Legacy-only known states remain readable; missing or invalid
73
+ structural status is unavailable. Qualified-with-gaps, incomplete, unqualified,
74
+ and unavailable remain distinct. Rebuild older corpus indexes before using
75
+ them; source manifests and native artifacts are never rewritten.
76
+
77
+ Every report retains the logical request and corpus-index digests plus the
78
+ exact admitted manifest, observation-set, assertion, calibration, and
79
+ comparison-gate digests. Moving the report therefore does not detach its
80
+ metrics from their rebuild inputs.
81
+
82
+ Observation and assertion sources name their retained bundle so the build can
83
+ recompute structural observations, resolve assertion citations, and reject
84
+ stale derived evidence. Calibration selections are likewise reloaded from their
85
+ digest-bound sources before their review history contributes to a metric.
86
+ Legacy v1 structural sets without the optional declared capability profile stay
87
+ readable; aggregation and semantic judging rebuild that profile from the native
88
+ bundle before use.
89
+
90
+ Matched comparisons require one candidate on each side for every caller-chosen
91
+ match key and one eligibility report whose candidate IDs are those two run IDs.
92
+ Missing, ambiguous, unavailable, or ungated pairs are exclusions. Supported
93
+ measures are `attempt:infrastructure-failure`,
94
+ `attempt:terminal-completed`, `verified:verifier-passed`, and any numeric
95
+ `structural:<extractor-id>` observation. The supplied comparison gate applies
96
+ only to that candidate pair, must name the exact measure, and must include the
97
+ measure's required capability;
98
+ an unsupported gate makes the comparison
99
+ unavailable, and partial-capability or declared-condition caveats remain in the
100
+ output.
101
+
102
+ Each eligibility report carries the digest of its source comparison request.
103
+ The aggregate build reloads that request and recomputes the report before using
104
+ the gate, so an edited or stale report cannot authorize a matched difference.
105
+ These bindings use `ebo.comparison-request/v2` and
106
+ `ebo.comparison-report/v2`; the original v1 request/report remain supported by
107
+ `ebo comparison check` but do not contain enough lineage for aggregation.
108
+
109
+ The caller supplies the recurrence threshold. A divergent matched unit below
110
+ that threshold is a `case-study`; reaching it yields only a
111
+ `recurring-description`. Neither state establishes causality or statistical
112
+ significance, and the report never emits a composite model ranking.