engineering-behavior-observatory 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/codex-run.d.ts +2 -0
- package/dist/src/codex-run.js +5 -1
- package/dist/src/codex.d.ts +2 -0
- package/dist/src/codex.js +10 -7
- package/dist/src/run-bundles.js +12 -0
- package/docs/guides/evidence-and-sharing.md +5 -0
- package/docs/harnesses/codex-harness.md +19 -1
- package/package.json +3 -1
- package/release/0.2.2/KNOWN_LIMITATIONS.md +13 -0
- package/release/0.2.2/README.md +29 -0
- package/release/0.2.2/reproducibility.json +73 -0
- package/release/0.2.3/KNOWN_LIMITATIONS.md +10 -0
- package/release/0.2.3/README.md +25 -0
- package/release/0.2.3/reproducibility.json +74 -0
- package/release/README.md +2 -0
package/dist/src/codex-run.d.ts
CHANGED
|
@@ -32,6 +32,8 @@ export type CodexNativeToolPolicyConfiguration = {
|
|
|
32
32
|
kind: "native-tool-policy";
|
|
33
33
|
approvalPolicy: CodexApprovalPolicy;
|
|
34
34
|
sandbox: CodexSandbox;
|
|
35
|
+
/** Only for workspace-write; defaults to true. */
|
|
36
|
+
networkAccess?: boolean;
|
|
35
37
|
};
|
|
36
38
|
export type CodexCaptureProfileConfiguration = {
|
|
37
39
|
schemaVersion: typeof CODEX_CONFIG_SCHEMA_VERSION;
|
package/dist/src/codex-run.js
CHANGED
|
@@ -281,6 +281,7 @@ export async function runCodexQueueEntry(options) {
|
|
|
281
281
|
effort: model.effort,
|
|
282
282
|
approvalPolicy: toolPolicy.approvalPolicy,
|
|
283
283
|
sandbox: toolPolicy.sandbox,
|
|
284
|
+
...(toolPolicy.networkAccess === undefined ? {} : { networkAccess: toolPolicy.networkAccess }),
|
|
284
285
|
...(captureProfile.telemetrySignals === undefined ? {} : {
|
|
285
286
|
telemetry: { signals: captureProfile.telemetrySignals },
|
|
286
287
|
}),
|
|
@@ -378,11 +379,14 @@ function validateConfiguration(record, kind, reference) {
|
|
|
378
379
|
throw configError(reference, "shutdownGraceMs must be positive");
|
|
379
380
|
}
|
|
380
381
|
else if (kind === "native-tool-policy") {
|
|
381
|
-
keys(record, ["schemaVersion", "kind", "approvalPolicy", "sandbox"], ["approvalPolicy", "sandbox"], reference);
|
|
382
|
+
keys(record, ["schemaVersion", "kind", "approvalPolicy", "sandbox", "networkAccess"], ["approvalPolicy", "sandbox"], reference);
|
|
382
383
|
if (!["untrusted", "on-request", "never"].includes(String(record.approvalPolicy)))
|
|
383
384
|
throw configError(reference, "approvalPolicy is invalid");
|
|
384
385
|
if (!["read-only", "workspace-write", "danger-full-access"].includes(String(record.sandbox)))
|
|
385
386
|
throw configError(reference, "sandbox is invalid");
|
|
387
|
+
if (record.networkAccess !== undefined && (typeof record.networkAccess !== "boolean" || record.sandbox !== "workspace-write")) {
|
|
388
|
+
throw configError(reference, "networkAccess must be a boolean and requires workspace-write");
|
|
389
|
+
}
|
|
386
390
|
}
|
|
387
391
|
else {
|
|
388
392
|
keys(record, ["schemaVersion", "kind", "telemetrySignals", "workspaceOutcome"], [], reference);
|
package/dist/src/codex.d.ts
CHANGED
|
@@ -20,6 +20,8 @@ export type CodexAppServerConfiguration = {
|
|
|
20
20
|
effort: CodexReasoningEffort;
|
|
21
21
|
approvalPolicy: CodexApprovalPolicy;
|
|
22
22
|
sandbox: CodexSandbox;
|
|
23
|
+
/** Workspace-write tool network access; defaults to true. Other sandbox modes retain their native policy. */
|
|
24
|
+
networkAccess?: boolean;
|
|
23
25
|
/** Test-only executable prefix; production uses the pinned executable directly. */
|
|
24
26
|
executableArgs?: readonly string[];
|
|
25
27
|
telemetry?: {
|
package/dist/src/codex.js
CHANGED
|
@@ -71,6 +71,9 @@ export async function captureCodexAppServer(request) {
|
|
|
71
71
|
requireText(request.configuration.executable, "Codex executable");
|
|
72
72
|
if (request.configuration.version !== CODEX_APP_SERVER_VERSION)
|
|
73
73
|
throw new Error(`Codex capture requires pinned runtime ${CODEX_APP_SERVER_VERSION}.`);
|
|
74
|
+
if (request.configuration.networkAccess !== undefined && (typeof request.configuration.networkAccess !== "boolean" || request.configuration.sandbox !== "workspace-write")) {
|
|
75
|
+
throw new Error("Codex networkAccess must be a boolean and requires workspace-write.");
|
|
76
|
+
}
|
|
74
77
|
const shutdownGraceMs = request.shutdownGraceMs ?? CODEX_DEFAULT_SHUTDOWN_GRACE_MS;
|
|
75
78
|
let abortRequested = request.signal?.aborted ?? false;
|
|
76
79
|
let abortDeadline = abortRequested ? performance.now() + shutdownGraceMs : undefined;
|
|
@@ -328,7 +331,7 @@ export async function captureCodexAppServer(request) {
|
|
|
328
331
|
config: { model_instructions_file: instructionsPath, model_reasoning_effort: request.configuration.effort,
|
|
329
332
|
mcp_servers: {}, hooks: {}, plugins: {},
|
|
330
333
|
...(request.configuration.sandbox === "workspace-write" ? { sandbox_workspace_write: {
|
|
331
|
-
writable_roots: [request.workspacePath], network_access:
|
|
334
|
+
writable_roots: [request.workspacePath], network_access: request.configuration.networkAccess ?? true, exclude_tmpdir_env_var: true, exclude_slash_tmp: true,
|
|
332
335
|
} } : {}),
|
|
333
336
|
},
|
|
334
337
|
developerInstructions: "Work only in the supplied workspace. Do not request interactive input or broaden permissions.",
|
|
@@ -351,7 +354,7 @@ export async function captureCodexAppServer(request) {
|
|
|
351
354
|
if (threadStart.approvalPolicy !== request.configuration.approvalPolicy) {
|
|
352
355
|
addGap({ kind: "approval-policy-mismatch", detail: `Requested ${request.configuration.approvalPolicy}; applied ${JSON.stringify(threadStart.approvalPolicy)}.` });
|
|
353
356
|
}
|
|
354
|
-
if (!sandboxMatches(request.configuration.sandbox, threadStart.sandbox, request.workspacePath, threadStart.cwd)) {
|
|
357
|
+
if (!sandboxMatches(request.configuration.sandbox, threadStart.sandbox, request.workspacePath, threadStart.cwd, request.configuration.networkAccess ?? true)) {
|
|
355
358
|
addGap({ kind: "sandbox-mismatch", detail: `Requested ${request.configuration.sandbox}; applied ${JSON.stringify(threadStart.sandbox)}.` });
|
|
356
359
|
}
|
|
357
360
|
const started = await sendRequest("turn/start", {
|
|
@@ -359,7 +362,7 @@ export async function captureCodexAppServer(request) {
|
|
|
359
362
|
input: [{ type: "text", text: request.prompt }],
|
|
360
363
|
cwd: request.workspacePath,
|
|
361
364
|
approvalPolicy: request.configuration.approvalPolicy,
|
|
362
|
-
sandboxPolicy: turnSandboxPolicy(request.configuration.sandbox, request.workspacePath),
|
|
365
|
+
sandboxPolicy: turnSandboxPolicy(request.configuration.sandbox, request.workspacePath, request.configuration.networkAccess ?? true),
|
|
363
366
|
model: request.configuration.model,
|
|
364
367
|
effort: request.configuration.effort,
|
|
365
368
|
});
|
|
@@ -675,7 +678,7 @@ function responseSourceIdentity(method, payload) {
|
|
|
675
678
|
return text(isRecord(payload.thread) ? payload.thread.id : undefined);
|
|
676
679
|
return undefined;
|
|
677
680
|
}
|
|
678
|
-
function turnSandboxPolicy(sandbox, workspace) {
|
|
681
|
+
function turnSandboxPolicy(sandbox, workspace, networkAccess) {
|
|
679
682
|
if (sandbox === "danger-full-access")
|
|
680
683
|
return { type: "dangerFullAccess" };
|
|
681
684
|
if (sandbox === "read-only")
|
|
@@ -683,12 +686,12 @@ function turnSandboxPolicy(sandbox, workspace) {
|
|
|
683
686
|
return {
|
|
684
687
|
type: "workspaceWrite",
|
|
685
688
|
writableRoots: [workspace],
|
|
686
|
-
networkAccess
|
|
689
|
+
networkAccess,
|
|
687
690
|
excludeTmpdirEnvVar: true,
|
|
688
691
|
excludeSlashTmp: true,
|
|
689
692
|
};
|
|
690
693
|
}
|
|
691
|
-
function sandboxMatches(requested, applied, workspace, appliedCwd) {
|
|
694
|
+
function sandboxMatches(requested, applied, workspace, appliedCwd, networkAccess) {
|
|
692
695
|
if (!isRecord(applied) || appliedCwd !== workspace)
|
|
693
696
|
return false;
|
|
694
697
|
if (requested === "danger-full-access")
|
|
@@ -696,7 +699,7 @@ function sandboxMatches(requested, applied, workspace, appliedCwd) {
|
|
|
696
699
|
if (requested === "read-only")
|
|
697
700
|
return applied.type === "readOnly" && applied.networkAccess === false;
|
|
698
701
|
return applied.type === "workspaceWrite"
|
|
699
|
-
&& applied.networkAccess ===
|
|
702
|
+
&& applied.networkAccess === networkAccess
|
|
700
703
|
&& Array.isArray(applied.writableRoots)
|
|
701
704
|
// Codex 0.153.4 roots are additional to cwd; it removes redundant cwd entries.
|
|
702
705
|
&& applied.writableRoots.length <= 1
|
package/dist/src/run-bundles.js
CHANGED
|
@@ -168,6 +168,18 @@ export class RunBundleAssembler {
|
|
|
168
168
|
const outcome = { descriptor, fingerprint, treeDigest, format };
|
|
169
169
|
await whileProjected?.(capturedPath, outcome);
|
|
170
170
|
return outcome;
|
|
171
|
+
}).then((outcome) => {
|
|
172
|
+
this.captureMissing = this.captureMissing.filter(({ kind }) => kind !== "workspace-capture-error");
|
|
173
|
+
return outcome;
|
|
174
|
+
}, (error) => {
|
|
175
|
+
// Keep the cause when a harness retains a partial bundle after packaging fails.
|
|
176
|
+
// Otherwise its terminal classification alone loses the actionable exception.
|
|
177
|
+
this.captureMissing = this.captureMissing.filter(({ kind }) => kind !== "workspace-capture-error");
|
|
178
|
+
this.captureMissing.push({
|
|
179
|
+
kind: "workspace-capture-error", reason: "not-collected", affects: ["outcome"],
|
|
180
|
+
detail: errorMessage(error),
|
|
181
|
+
});
|
|
182
|
+
throw error;
|
|
171
183
|
});
|
|
172
184
|
}
|
|
173
185
|
async finalize(input) {
|
|
@@ -41,6 +41,11 @@ events can still explain a stop. Keep terminal state, infrastructure failure,
|
|
|
41
41
|
and capture quality separate. Preserve `retainedWorkspacePath` when outcome
|
|
42
42
|
packaging failed; that path is a recovery location, not qualified evidence.
|
|
43
43
|
|
|
44
|
+
Capture reports retain packaging exceptions as `workspace-capture-error`
|
|
45
|
+
missing-evidence entries. Inspect the detail before retrying capture against a
|
|
46
|
+
retained workspace. A later successful capture is recovery evidence; it does
|
|
47
|
+
not change the original attempt's recorded failure.
|
|
48
|
+
|
|
44
49
|
Use a new output destination for derived records and reruns. Do not edit a
|
|
45
50
|
native bundle to make a validator accept it.
|
|
46
51
|
|
|
@@ -63,7 +63,7 @@ the actual provider model and effort. A minimal observational configuration is:
|
|
|
63
63
|
{ "schemaVersion": "ebo.codex-config/v1", "kind": "model", "provider": "openai", "model": "gpt-5.6-sol", "effort": "high" }
|
|
64
64
|
{ "schemaVersion": "ebo.codex-config/v1", "kind": "harness", "adapter": "codex-app-server", "executable": "/opt/homebrew/bin/codex", "version": "0.153.4", "contractDigest": "sha256:e5f798fd1343c539f01fedea0e8a84a43c080fcca4615c80eb04a5edab4f7d0a" }
|
|
65
65
|
{ "schemaVersion": "ebo.codex-config/v1", "kind": "native-limits", "shutdownGraceMs": 2000 }
|
|
66
|
-
{ "schemaVersion": "ebo.codex-config/v1", "kind": "native-tool-policy", "approvalPolicy": "never", "sandbox": "workspace-write" }
|
|
66
|
+
{ "schemaVersion": "ebo.codex-config/v1", "kind": "native-tool-policy", "approvalPolicy": "never", "sandbox": "workspace-write", "networkAccess": true }
|
|
67
67
|
{ "schemaVersion": "ebo.codex-config/v1", "kind": "capture-profile", "telemetrySignals": ["logs", "traces", "metrics"], "workspaceOutcome": { "excludeDirectoryNames": ["node_modules"] } }
|
|
68
68
|
```
|
|
69
69
|
|
|
@@ -79,6 +79,24 @@ ebo codex run \
|
|
|
79
79
|
`runCodexQueueEntry` is the equivalent library API. It does not iterate, retry,
|
|
80
80
|
resume, or overwrite an existing attempt destination.
|
|
81
81
|
|
|
82
|
+
Workspace-write runs enable outbound tool network access by default, allowing
|
|
83
|
+
dependency installation and documentation retrieval. Set `networkAccess: false`
|
|
84
|
+
in the digest-pinned tool policy for an offline condition. The setting applies
|
|
85
|
+
to both thread and turn policy; EBO verifies the applied thread policy and retains
|
|
86
|
+
the request and response as native evidence. Filesystem writable roots and
|
|
87
|
+
noninteractive approvals are unchanged. Network access permits outbound data
|
|
88
|
+
transfer, so use only the admitted task inputs and permitted destinations.
|
|
89
|
+
|
|
90
|
+
`networkAccess` is supported only with `workspace-write`. Read-only mode keeps
|
|
91
|
+
tool network access disabled; danger-full-access retains its native unrestricted
|
|
92
|
+
policy. Invalid combinations are rejected rather than silently ignored. This
|
|
93
|
+
setting does not block model-provider traffic or configure the separate Codex
|
|
94
|
+
web-search tool. Context-window and compaction settings remain unspecified.
|
|
95
|
+
|
|
96
|
+
Before this change, workspace-write runs always disabled tool network access.
|
|
97
|
+
Historical bundles retain their recorded policy. For reproducible re-execution,
|
|
98
|
+
compile a new queue with an explicit boolean; an omitted value now means `true`.
|
|
99
|
+
|
|
82
100
|
## Lifecycle and evidence
|
|
83
101
|
|
|
84
102
|
The client sends `initialize`, `initialized`, `thread/start`, and `turn/start`.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "engineering-behavior-observatory",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.3",
|
|
4
4
|
"description": "Capture engineering-agent trajectories, evaluate behavior, and inspect cited evidence across harnesses.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -45,6 +45,8 @@
|
|
|
45
45
|
"release/0.1.0/",
|
|
46
46
|
"release/0.2.0/",
|
|
47
47
|
"release/0.2.1/",
|
|
48
|
+
"release/0.2.2/",
|
|
49
|
+
"release/0.2.3/",
|
|
48
50
|
"schemas/",
|
|
49
51
|
"scripts/atlas-grafana.sh"
|
|
50
52
|
],
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# v0.2.2 known limitations
|
|
2
|
+
|
|
3
|
+
The [v0.2.1 limitations](../0.2.1/KNOWN_LIMITATIONS.md) continue to apply.
|
|
4
|
+
|
|
5
|
+
- `networkAccess` configures workspace-write command networking, not the model
|
|
6
|
+
provider connection or the separate Codex web-search tool.
|
|
7
|
+
- Read-only mode remains offline for tools. Danger-full-access retains its
|
|
8
|
+
unrestricted native behavior. Supplying `networkAccess` with either mode is
|
|
9
|
+
rejected.
|
|
10
|
+
- Platform or organization network restrictions can still prevent a request.
|
|
11
|
+
A configured allowance is not proof that every destination is reachable.
|
|
12
|
+
- The default changed from offline to online for workspace-write runs. Pin the
|
|
13
|
+
boolean explicitly in new experiment configurations.
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# v0.2.2: Configurable Codex tool network access
|
|
2
|
+
|
|
3
|
+
Codex workspace-write runs now enable outbound tool network access by default.
|
|
4
|
+
Set `networkAccess: false` in the digest-pinned native tool policy to run offline.
|
|
5
|
+
Both thread and turn receive the selected policy; native evidence retains the
|
|
6
|
+
request and applied policy, and mismatches remain explicit capture gaps.
|
|
7
|
+
|
|
8
|
+
Filesystem writable roots, temporary-directory restrictions, approval handling,
|
|
9
|
+
credential isolation, and read-only judging are unchanged. Invalid types and
|
|
10
|
+
settings on incompatible sandbox modes are rejected. No dependencies or runtime
|
|
11
|
+
pins changed.
|
|
12
|
+
|
|
13
|
+
## Reproduction and migration
|
|
14
|
+
|
|
15
|
+
See the [Codex guide](../../docs/harnesses/codex-harness.md) for the configuration.
|
|
16
|
+
Historical bundles are unchanged. An omitted setting previously meant offline;
|
|
17
|
+
it now means network enabled. Compile a new queue with an explicit boolean when
|
|
18
|
+
reproducing an earlier condition. Network access permits outbound transfer, so
|
|
19
|
+
admit appropriate task data before execution.
|
|
20
|
+
|
|
21
|
+
Run `npm ci` and `npm run acceptance` with Node 24.19.0. The release checks include
|
|
22
|
+
policy defaults, explicit on/off behavior, invalid settings, mismatch evidence,
|
|
23
|
+
and offline queue-to-runtime propagation, plus the full existing test suite.
|
|
24
|
+
A live ChatGPT-authenticated Astra check also fetched example.com successfully
|
|
25
|
+
through the workspace-write sandbox; this is infrastructure proof, not a scored
|
|
26
|
+
task result.
|
|
27
|
+
|
|
28
|
+
See [known limitations](KNOWN_LIMITATIONS.md) and the
|
|
29
|
+
[reproducibility manifest](reproducibility.json).
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": "ebo.release-reproducibility/v1",
|
|
3
|
+
"release": {
|
|
4
|
+
"name": "engineering-behavior-observatory",
|
|
5
|
+
"version": "0.2.2"
|
|
6
|
+
},
|
|
7
|
+
"runtime": {
|
|
8
|
+
"node": "24.19.0",
|
|
9
|
+
"claudeAgentSdk": "0.3.258",
|
|
10
|
+
"openhandsAgentServer": "1.46.0",
|
|
11
|
+
"deepseekClient": "0.1.1-rc.2",
|
|
12
|
+
"deepseekProtocol": "0.1.1-rc.2",
|
|
13
|
+
"deepseekRuntime": "0.1.1-rc.2",
|
|
14
|
+
"codexAppServer": "0.153.4",
|
|
15
|
+
"grafana": "13.2.0",
|
|
16
|
+
"grafanaInfinity": "4.0.0",
|
|
17
|
+
"piSdk": "0.85.1",
|
|
18
|
+
"cursorSdk": "1.0.31"
|
|
19
|
+
},
|
|
20
|
+
"commands": [
|
|
21
|
+
"npm ci",
|
|
22
|
+
"npm run acceptance"
|
|
23
|
+
],
|
|
24
|
+
"fixtureCoverage": {
|
|
25
|
+
"agent-sdk": "frozen queue entry through qualified capture, approved export, normalization, configurable judging, review, aggregation, and Atlas",
|
|
26
|
+
"openhands-agent-server": "pinned REST/WebSocket stream-final reconciliation, workspace/outcome evidence, partial capture, and retained evaluation",
|
|
27
|
+
"deepseek-harness": "official-client JSON-RPC composition, receipt-to-idle completion, stderr, interruption, shutdown, swaps, and retained evaluation",
|
|
28
|
+
"codex-app-server": "owned stdio lifecycle, full history, OTLP, interruption/failure, export, and retained evaluation; configurable workspace-write networking with explicit offline queue propagation",
|
|
29
|
+
"acceptance-cases": "seeded ordering, secret scanning, partial attempts, native references, abstention, disputes, denominators, and unsupported comparisons",
|
|
30
|
+
"pi-sdk": "frozen queue, passive hooks, durable session, partial cleanup, export, retained normalization and evaluation",
|
|
31
|
+
"cursor-sdk": "frozen queue, official store, callbacks/history, bounded cancellation, export, retained normalization and evaluation"
|
|
32
|
+
},
|
|
33
|
+
"determinism": {
|
|
34
|
+
"stable": [
|
|
35
|
+
"task, run, attempt, event, assertion, aggregate, and Atlas case identities",
|
|
36
|
+
"fixture, native-reference, normalized-dataset, source/cohort, archive, and package digests",
|
|
37
|
+
"seeded queue and review-sample ordering"
|
|
38
|
+
],
|
|
39
|
+
"excluded": [
|
|
40
|
+
"native and EBO observation timestamps",
|
|
41
|
+
"lifecycle start/finish timestamps",
|
|
42
|
+
"provider timing and usage",
|
|
43
|
+
"temporary workspace and output paths"
|
|
44
|
+
]
|
|
45
|
+
},
|
|
46
|
+
"fixtures": {
|
|
47
|
+
"tests/fixtures/task-packet.valid.v1.json": "ac2ef1043c0cc16bf4c21d05c1dc880ca1b4b8cc2f9ea7653328e91f0b0e2681",
|
|
48
|
+
"test/fixtures/agent-sdk-normalizer/complete.input.json": "7285dee088517e5de38ceda84fc7c58d9303949c7448ab3b5e10a5a2d2ebdfe3",
|
|
49
|
+
"test/fixtures/agent-sdk-normalizer/complete.expected.jsonl": "d32fa3a013bc179f1673f7e1a89b8a721bb8ddaf63eba5bc8a3f14f1cc148ed8",
|
|
50
|
+
"contracts/openhands-agent-server-v1.44.1.json": "e7a07977688a0703b15751a8a1ab17f29be45f9fdf438017e0dbcbc49cca37b0",
|
|
51
|
+
"contracts/openhands-agent-server-v1.46.0.json": "455cdddd2c206cb2f20245c2189b92e773657ec14d651236baa532b2dbadeb28",
|
|
52
|
+
"test/fixtures/openhands/v1.44.1/streamed-events.json": "d751083ba2ef94c9865117db481cc2f4933772bbbbb6b9fcf60571563715c1ea",
|
|
53
|
+
"test/fixtures/openhands/v1.44.1/final-events.json": "551db9904e860004d604c923692b5c5d3aaee966220492214c60003d2df415b6",
|
|
54
|
+
"test/fixtures/deepseek/golden-success.json": "f3867d38169bf0e28f64aba1dd60553f21cf07f41f81b4bad48261e49fa3ca88",
|
|
55
|
+
"test/fixtures/deepseek/golden-interrupted.json": "79e1f8404df1ee3b562d0bdbc0b2ee712ffc6d5774992b92970814ea815677b7",
|
|
56
|
+
"test/fixtures/deepseek/compositions/minimal/composition.json": "72fe4368e105f1bad82b7fdf8380e92c9fa689d8985f6ab91f534f0baf3eda02",
|
|
57
|
+
"contracts/codex-app-server-0.153.4/manifest.json": "e62281ff5e5d1cc71d07763b85e997ccc9ddbad9aa82a469da4181ee4b1890a8",
|
|
58
|
+
"test/fixtures/codex/legacy-0.150.1.dataset.json": "f71906b009e546afbf3b4e79396f223181382f3b3d8795e9cfcd032398a9d6bf",
|
|
59
|
+
"test/fixtures/behavior-assertions/abstained.json": "7f1962864f664c794925a67d93074e7b150af0f57c58041a81e062990d641197",
|
|
60
|
+
"test/fixtures/behavior-assertions/disputed.review.json": "450d3d9efe7feefaf7500b3ad20ab2803b412c7e53b91763125e15a83be87981",
|
|
61
|
+
"test/fixtures/comparison/exact.json": "cafa68e94b8d50dc308ddcfc2e73aa139c828b4ecd1e1c762bbc51bf1b44501e",
|
|
62
|
+
"test/fixtures/structural-observations/golden.json": "bad71a3a93524b76c0a2fdbb58e98e1e65be46cae14acf18ac89d9ba76e03e8a",
|
|
63
|
+
"test/atlas-fixture.ts": "9f03ce9cbcd0e5d3e3a5916866554ceb1c21f1c8712e58529499c1ea499f048d",
|
|
64
|
+
"test/pi.test.ts": "a00997c5617ca178e19b25b0d483cbdce8ce7c306d4c871557945b31a2e657cf",
|
|
65
|
+
"test/cursor-sdk.test.ts": "753094f127b11dc392ec2244c9edddabec27aa94347bb6159a292129ed2a2822",
|
|
66
|
+
"examples/cursor-sdk/README.md": "a2cf481123e7f3deb1c5a682ba208a775c3f91c35e1a6c1a428c517c79692618",
|
|
67
|
+
"examples/cursor-sdk/capture-profile.json": "d1a5787875daf5523b3783f259b891c256a904cbb10279d5be214af846305a2d",
|
|
68
|
+
"examples/cursor-sdk/harness.json": "2f1da6f2b17258d0ee59431f687696901ce3692589175d182f3a4bbe8dd4d689",
|
|
69
|
+
"examples/cursor-sdk/model.json": "ad8f31ff5d2387c02d79226679022358afafc61ba5dda73040e3f754530a230a",
|
|
70
|
+
"examples/cursor-sdk/native-limits.json": "40fe451f56b0863e79b42f5c85defbce678e1dc90dd248d15c430db176d71511",
|
|
71
|
+
"examples/cursor-sdk/native-tool-policy.json": "0fef98b36cd39de82914c5b068022c1335dadda24b7d6e7df398b2430e6a16e0"
|
|
72
|
+
}
|
|
73
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# v0.2.3 known limitations
|
|
2
|
+
|
|
3
|
+
The [v0.2.2 limitations](../0.2.2/KNOWN_LIMITATIONS.md) continue to apply.
|
|
4
|
+
|
|
5
|
+
- Error details are retained for future captures. Exceptions discarded by older
|
|
6
|
+
versions cannot be reconstructed from a generic missing-workspace report.
|
|
7
|
+
- A later successful workspace capture is recovery evidence, not proof that the
|
|
8
|
+
original capture succeeded or that its transient failure cannot recur.
|
|
9
|
+
- The existing OTLP size bounds, missing-signal reporting, and bounded native
|
|
10
|
+
normalization projection are unchanged.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# v0.2.3: Workspace capture error diagnostics
|
|
2
|
+
|
|
3
|
+
The shared run-bundle assembler now preserves workspace packaging exceptions
|
|
4
|
+
in capture reports as `workspace-capture-error` missing-evidence entries.
|
|
5
|
+
Previously, a harness could finalize a partial bundle with only a generic
|
|
6
|
+
missing-workspace report, losing the exception needed to diagnose the failure.
|
|
7
|
+
|
|
8
|
+
Capture still rejects failures and preserves the source workspace through the
|
|
9
|
+
existing harness retention paths. A successful explicit capture retry clears
|
|
10
|
+
the prior diagnostic. No automatic retry, weaker validation, dependency update,
|
|
11
|
+
runtime-pin change, or artifact-schema change is introduced.
|
|
12
|
+
|
|
13
|
+
This release fixes diagnostic loss. It does not claim to eliminate every
|
|
14
|
+
workspace packaging failure. Existing immutable bundles are not rewritten.
|
|
15
|
+
|
|
16
|
+
## Verification
|
|
17
|
+
|
|
18
|
+
Run `npm ci` and `npm run acceptance` on Node 24.19.0. Regression coverage checks
|
|
19
|
+
retention of the original capture error, successful retry clearing, and valid
|
|
20
|
+
partial manifests. Acceptance also runs the full suite, checks documentation
|
|
21
|
+
and package contents, and compares two independently built package archives.
|
|
22
|
+
|
|
23
|
+
See [known limitations](KNOWN_LIMITATIONS.md), the
|
|
24
|
+
[reproducibility manifest](reproducibility.json), and
|
|
25
|
+
[recovery guidance](../../docs/guides/evidence-and-sharing.md).
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": "ebo.release-reproducibility/v1",
|
|
3
|
+
"release": {
|
|
4
|
+
"name": "engineering-behavior-observatory",
|
|
5
|
+
"version": "0.2.3"
|
|
6
|
+
},
|
|
7
|
+
"runtime": {
|
|
8
|
+
"node": "24.19.0",
|
|
9
|
+
"claudeAgentSdk": "0.3.258",
|
|
10
|
+
"openhandsAgentServer": "1.46.0",
|
|
11
|
+
"deepseekClient": "0.1.1-rc.2",
|
|
12
|
+
"deepseekProtocol": "0.1.1-rc.2",
|
|
13
|
+
"deepseekRuntime": "0.1.1-rc.2",
|
|
14
|
+
"codexAppServer": "0.153.4",
|
|
15
|
+
"grafana": "13.2.0",
|
|
16
|
+
"grafanaInfinity": "4.0.0",
|
|
17
|
+
"piSdk": "0.85.1",
|
|
18
|
+
"cursorSdk": "1.0.31"
|
|
19
|
+
},
|
|
20
|
+
"commands": [
|
|
21
|
+
"npm ci",
|
|
22
|
+
"npm run acceptance"
|
|
23
|
+
],
|
|
24
|
+
"fixtureCoverage": {
|
|
25
|
+
"agent-sdk": "frozen queue entry through qualified capture, approved export, normalization, configurable judging, review, aggregation, and Atlas",
|
|
26
|
+
"openhands-agent-server": "pinned REST/WebSocket stream-final reconciliation, workspace/outcome evidence, partial capture, and retained evaluation",
|
|
27
|
+
"deepseek-harness": "official-client JSON-RPC composition, receipt-to-idle completion, stderr, interruption, shutdown, swaps, and retained evaluation",
|
|
28
|
+
"codex-app-server": "owned stdio lifecycle, full history, OTLP, interruption/failure, export, and retained evaluation; configurable workspace-write networking with explicit offline queue propagation",
|
|
29
|
+
"acceptance-cases": "seeded ordering, secret scanning, partial attempts, native references, abstention, disputes, denominators, and unsupported comparisons",
|
|
30
|
+
"pi-sdk": "frozen queue, passive hooks, durable session, partial cleanup, export, retained normalization and evaluation",
|
|
31
|
+
"cursor-sdk": "frozen queue, official store, callbacks/history, bounded cancellation, export, retained normalization and evaluation",
|
|
32
|
+
"workspace-capture": "shared packaging error retained in partial reports; explicit successful retry clears prior error"
|
|
33
|
+
},
|
|
34
|
+
"determinism": {
|
|
35
|
+
"stable": [
|
|
36
|
+
"task, run, attempt, event, assertion, aggregate, and Atlas case identities",
|
|
37
|
+
"fixture, native-reference, normalized-dataset, source/cohort, archive, and package digests",
|
|
38
|
+
"seeded queue and review-sample ordering"
|
|
39
|
+
],
|
|
40
|
+
"excluded": [
|
|
41
|
+
"native and EBO observation timestamps",
|
|
42
|
+
"lifecycle start/finish timestamps",
|
|
43
|
+
"provider timing and usage",
|
|
44
|
+
"temporary workspace and output paths"
|
|
45
|
+
]
|
|
46
|
+
},
|
|
47
|
+
"fixtures": {
|
|
48
|
+
"tests/fixtures/task-packet.valid.v1.json": "ac2ef1043c0cc16bf4c21d05c1dc880ca1b4b8cc2f9ea7653328e91f0b0e2681",
|
|
49
|
+
"test/fixtures/agent-sdk-normalizer/complete.input.json": "7285dee088517e5de38ceda84fc7c58d9303949c7448ab3b5e10a5a2d2ebdfe3",
|
|
50
|
+
"test/fixtures/agent-sdk-normalizer/complete.expected.jsonl": "d32fa3a013bc179f1673f7e1a89b8a721bb8ddaf63eba5bc8a3f14f1cc148ed8",
|
|
51
|
+
"contracts/openhands-agent-server-v1.44.1.json": "e7a07977688a0703b15751a8a1ab17f29be45f9fdf438017e0dbcbc49cca37b0",
|
|
52
|
+
"contracts/openhands-agent-server-v1.46.0.json": "455cdddd2c206cb2f20245c2189b92e773657ec14d651236baa532b2dbadeb28",
|
|
53
|
+
"test/fixtures/openhands/v1.44.1/streamed-events.json": "d751083ba2ef94c9865117db481cc2f4933772bbbbb6b9fcf60571563715c1ea",
|
|
54
|
+
"test/fixtures/openhands/v1.44.1/final-events.json": "551db9904e860004d604c923692b5c5d3aaee966220492214c60003d2df415b6",
|
|
55
|
+
"test/fixtures/deepseek/golden-success.json": "f3867d38169bf0e28f64aba1dd60553f21cf07f41f81b4bad48261e49fa3ca88",
|
|
56
|
+
"test/fixtures/deepseek/golden-interrupted.json": "79e1f8404df1ee3b562d0bdbc0b2ee712ffc6d5774992b92970814ea815677b7",
|
|
57
|
+
"test/fixtures/deepseek/compositions/minimal/composition.json": "72fe4368e105f1bad82b7fdf8380e92c9fa689d8985f6ab91f534f0baf3eda02",
|
|
58
|
+
"contracts/codex-app-server-0.153.4/manifest.json": "e62281ff5e5d1cc71d07763b85e997ccc9ddbad9aa82a469da4181ee4b1890a8",
|
|
59
|
+
"test/fixtures/codex/legacy-0.150.1.dataset.json": "f71906b009e546afbf3b4e79396f223181382f3b3d8795e9cfcd032398a9d6bf",
|
|
60
|
+
"test/fixtures/behavior-assertions/abstained.json": "7f1962864f664c794925a67d93074e7b150af0f57c58041a81e062990d641197",
|
|
61
|
+
"test/fixtures/behavior-assertions/disputed.review.json": "450d3d9efe7feefaf7500b3ad20ab2803b412c7e53b91763125e15a83be87981",
|
|
62
|
+
"test/fixtures/comparison/exact.json": "cafa68e94b8d50dc308ddcfc2e73aa139c828b4ecd1e1c762bbc51bf1b44501e",
|
|
63
|
+
"test/fixtures/structural-observations/golden.json": "bad71a3a93524b76c0a2fdbb58e98e1e65be46cae14acf18ac89d9ba76e03e8a",
|
|
64
|
+
"test/atlas-fixture.ts": "9f03ce9cbcd0e5d3e3a5916866554ceb1c21f1c8712e58529499c1ea499f048d",
|
|
65
|
+
"test/pi.test.ts": "a00997c5617ca178e19b25b0d483cbdce8ce7c306d4c871557945b31a2e657cf",
|
|
66
|
+
"test/cursor-sdk.test.ts": "753094f127b11dc392ec2244c9edddabec27aa94347bb6159a292129ed2a2822",
|
|
67
|
+
"examples/cursor-sdk/README.md": "a2cf481123e7f3deb1c5a682ba208a775c3f91c35e1a6c1a428c517c79692618",
|
|
68
|
+
"examples/cursor-sdk/capture-profile.json": "d1a5787875daf5523b3783f259b891c256a904cbb10279d5be214af846305a2d",
|
|
69
|
+
"examples/cursor-sdk/harness.json": "2f1da6f2b17258d0ee59431f687696901ce3692589175d182f3a4bbe8dd4d689",
|
|
70
|
+
"examples/cursor-sdk/model.json": "ad8f31ff5d2387c02d79226679022358afafc61ba5dda73040e3f754530a230a",
|
|
71
|
+
"examples/cursor-sdk/native-limits.json": "40fe451f56b0863e79b42f5c85defbce678e1dc90dd248d15c430db176d71511",
|
|
72
|
+
"examples/cursor-sdk/native-tool-policy.json": "0fef98b36cd39de82914c5b068022c1335dadda24b7d6e7df398b2430e6a16e0"
|
|
73
|
+
}
|
|
74
|
+
}
|
package/release/README.md
CHANGED
|
@@ -5,6 +5,8 @@ For downloads and published artifacts, see
|
|
|
5
5
|
|
|
6
6
|
| Version | Changes and verification | Support boundary |
|
|
7
7
|
| :--- | :--- | :--- |
|
|
8
|
+
| [0.2.3](0.2.3/README.md) | Retain workspace capture errors in partial-bundle reports | [Known limitations](0.2.3/KNOWN_LIMITATIONS.md) |
|
|
9
|
+
| [0.2.2](0.2.2/README.md) | Configurable Codex tool networking, enabled by default for workspace-write | [Known limitations](0.2.2/KNOWN_LIMITATIONS.md) |
|
|
8
10
|
| [0.2.1](0.2.1/README.md) | Documentation redesign, Apache-2.0, and npm distribution setup | [Known limitations](0.2.1/KNOWN_LIMITATIONS.md) |
|
|
9
11
|
| [0.2.0](0.2.0/README.md) | Pi and Cursor SDK integrations; reproducible package | [Known limitations](0.2.0/KNOWN_LIMITATIONS.md) |
|
|
10
12
|
| [0.1.0](0.1.0/README.md) | Initial capture, behavioral evaluation, Atlas, and release gate | [Known limitations](0.1.0/KNOWN_LIMITATIONS.md) |
|