engineering-behavior-observatory 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +151 -0
- package/contracts/codex-app-server-0.150.1/manifest.json +25 -0
- package/contracts/codex-app-server-0.150.1/schema/InitializeParams.json +84 -0
- package/contracts/codex-app-server-0.150.1/schema/JSONRPCMessage.json +137 -0
- package/contracts/codex-app-server-0.150.1/schema/v2/ThreadReadParams.json +17 -0
- package/contracts/codex-app-server-0.150.1/schema/v2/ThreadTokenUsageUpdatedNotification.json +82 -0
- package/contracts/codex-app-server-0.150.1/schema/v2/TurnInterruptParams.json +17 -0
- package/contracts/codex-app-server-0.150.1/types/AskForApproval.ts +5 -0
- package/contracts/codex-app-server-0.150.1/types/ClientNotification.ts +5 -0
- package/contracts/codex-app-server-0.150.1/types/SandboxMode.ts +5 -0
- package/contracts/codex-app-server-0.150.1/types/ThreadReadParams.ts +9 -0
- package/contracts/codex-app-server-0.150.1/types/TokenUsageBreakdown.ts +5 -0
- package/contracts/codex-app-server-0.150.1/types/TurnInterruptParams.ts +5 -0
- package/contracts/codex-app-server-0.153.4/manifest.json +27 -0
- package/contracts/codex-app-server-0.153.4/schema/JSONRPCMessage.json +137 -0
- package/contracts/codex-app-server-0.153.4/schema/v1/InitializeParams.json +84 -0
- package/contracts/codex-app-server-0.153.4/schema/v2/ThreadReadParams.json +17 -0
- package/contracts/codex-app-server-0.153.4/schema/v2/ThreadStartParams.json +515 -0
- package/contracts/codex-app-server-0.153.4/schema/v2/ThreadTokenUsageUpdatedNotification.json +82 -0
- package/contracts/codex-app-server-0.153.4/schema/v2/TurnInterruptParams.json +17 -0
- package/contracts/codex-app-server-0.153.4/schema/v2/TurnStartParams.json +921 -0
- package/contracts/codex-app-server-0.153.4/types/AskForApproval.ts +5 -0
- package/contracts/codex-app-server-0.153.4/types/ClientNotification.ts +5 -0
- package/contracts/codex-app-server-0.153.4/types/SandboxMode.ts +5 -0
- package/contracts/codex-app-server-0.153.4/types/ThreadReadParams.ts +12 -0
- package/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.ts +5 -0
- package/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.ts +5 -0
- package/contracts/openhands-agent-server-v1.44.1.json +65 -0
- package/contracts/openhands-agent-server-v1.46.0.json +46 -0
- package/dist/contracts/codex-app-server-0.153.4/types/AskForApproval.d.ts +9 -0
- package/dist/contracts/codex-app-server-0.153.4/types/AskForApproval.js +2 -0
- package/dist/contracts/codex-app-server-0.153.4/types/ClientNotification.d.ts +3 -0
- package/dist/contracts/codex-app-server-0.153.4/types/ClientNotification.js +2 -0
- package/dist/contracts/codex-app-server-0.153.4/types/SandboxMode.d.ts +1 -0
- package/dist/contracts/codex-app-server-0.153.4/types/SandboxMode.js +2 -0
- package/dist/contracts/codex-app-server-0.153.4/types/ThreadReadParams.d.ts +10 -0
- package/dist/contracts/codex-app-server-0.153.4/types/ThreadReadParams.js +2 -0
- package/dist/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.d.ts +8 -0
- package/dist/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.js +2 -0
- package/dist/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.d.ts +4 -0
- package/dist/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.js +2 -0
- package/dist/src/agent-sdk-normalizer.d.ts +22 -0
- package/dist/src/agent-sdk-normalizer.js +995 -0
- package/dist/src/agent-sdk-run.d.ts +31 -0
- package/dist/src/agent-sdk-run.js +242 -0
- package/dist/src/agent-sdk-runner.d.ts +96 -0
- package/dist/src/agent-sdk-runner.js +407 -0
- package/dist/src/agent-sdk.d.ts +294 -0
- package/dist/src/agent-sdk.js +644 -0
- package/dist/src/aggregation.d.ts +183 -0
- package/dist/src/aggregation.js +513 -0
- package/dist/src/artifacts.d.ts +36 -0
- package/dist/src/artifacts.js +1372 -0
- package/dist/src/atlas-grafana.d.ts +377 -0
- package/dist/src/atlas-grafana.js +47 -0
- package/dist/src/atlas-html.d.ts +2 -0
- package/dist/src/atlas-html.js +34 -0
- package/dist/src/atlas.d.ts +130 -0
- package/dist/src/atlas.js +235 -0
- package/dist/src/behavior-assertions.d.ts +95 -0
- package/dist/src/behavior-assertions.js +154 -0
- package/dist/src/cli.d.ts +5 -0
- package/dist/src/cli.js +957 -0
- package/dist/src/codex-judge.d.ts +44 -0
- package/dist/src/codex-judge.js +214 -0
- package/dist/src/codex-run.d.ts +114 -0
- package/dist/src/codex-run.js +519 -0
- package/dist/src/codex.d.ts +187 -0
- package/dist/src/codex.js +1028 -0
- package/dist/src/contracts.d.ts +155 -0
- package/dist/src/contracts.js +866 -0
- package/dist/src/corpus.d.ts +67 -0
- package/dist/src/corpus.js +540 -0
- package/dist/src/cursor-sdk-runner.d.ts +81 -0
- package/dist/src/cursor-sdk-runner.js +362 -0
- package/dist/src/cursor-sdk.d.ts +81 -0
- package/dist/src/cursor-sdk.js +1078 -0
- package/dist/src/deepseek-adapter.d.ts +152 -0
- package/dist/src/deepseek-adapter.js +777 -0
- package/dist/src/exports.d.ts +69 -0
- package/dist/src/exports.js +966 -0
- package/dist/src/human-calibration.d.ts +167 -0
- package/dist/src/human-calibration.js +618 -0
- package/dist/src/index.d.ts +71 -0
- package/dist/src/index.js +38 -0
- package/dist/src/lifecycle.d.ts +196 -0
- package/dist/src/lifecycle.js +1889 -0
- package/dist/src/normalization-integrity.d.ts +128 -0
- package/dist/src/normalization-integrity.js +429 -0
- package/dist/src/openhands-run.d.ts +36 -0
- package/dist/src/openhands-run.js +415 -0
- package/dist/src/openhands.d.ts +130 -0
- package/dist/src/openhands.js +859 -0
- package/dist/src/pi.d.ts +203 -0
- package/dist/src/pi.js +1345 -0
- package/dist/src/process-protocol.d.ts +245 -0
- package/dist/src/process-protocol.js +1261 -0
- package/dist/src/retained-evidence.d.ts +12 -0
- package/dist/src/retained-evidence.js +201 -0
- package/dist/src/run-bundles.d.ts +193 -0
- package/dist/src/run-bundles.js +993 -0
- package/dist/src/scheduler.d.ts +109 -0
- package/dist/src/scheduler.js +1080 -0
- package/dist/src/semantic-judge.d.ts +194 -0
- package/dist/src/semantic-judge.js +875 -0
- package/dist/src/structural-observations.d.ts +124 -0
- package/dist/src/structural-observations.js +671 -0
- package/dist/src/task-packets.d.ts +115 -0
- package/dist/src/task-packets.js +683 -0
- package/dist/src/uniform-events.d.ts +120 -0
- package/dist/src/uniform-events.js +158 -0
- package/dist/src/verifiers.d.ts +116 -0
- package/dist/src/verifiers.js +819 -0
- package/dist/src/workspaces.d.ts +56 -0
- package/dist/src/workspaces.js +1314 -0
- package/docs/README.md +60 -0
- package/docs/development/README.md +71 -0
- package/docs/development/documentation-sync.md +58 -0
- package/docs/development/extension-contracts.md +307 -0
- package/docs/evaluation/README.md +48 -0
- package/docs/evaluation/aggregation.md +112 -0
- package/docs/evaluation/behavior-assertions.md +45 -0
- package/docs/evaluation/human-calibration.md +170 -0
- package/docs/evaluation/normalization-integrity.md +80 -0
- package/docs/evaluation/semantic-judge.md +188 -0
- package/docs/evaluation/structural-observations.md +84 -0
- package/docs/evaluation/uniform-events.md +131 -0
- package/docs/guides/atlas.md +144 -0
- package/docs/guides/evidence-and-sharing.md +100 -0
- package/docs/guides/operator-guide.md +424 -0
- package/docs/guides/quickstart.md +152 -0
- package/docs/guides/telemetry.md +66 -0
- package/docs/harnesses/README.md +31 -0
- package/docs/harnesses/claude-agent-sdk.md +61 -0
- package/docs/harnesses/codex-harness.md +174 -0
- package/docs/harnesses/cursor-sdk.md +149 -0
- package/docs/harnesses/deepseek-harness.md +134 -0
- package/docs/harnesses/openhands-agent-server.md +153 -0
- package/docs/harnesses/pi-sdk.md +66 -0
- package/docs/reference/README.md +19 -0
- package/docs/reference/agent-sdk-operational-runner.md +118 -0
- package/docs/reference/cli.md +114 -0
- package/docs/reference/contracts.md +222 -0
- package/docs/reference/run-bundle-contract.md +354 -0
- package/docs/reference/run-lifecycle.md +54 -0
- package/examples/cursor-sdk/README.md +11 -0
- package/examples/cursor-sdk/capture-profile.json +1 -0
- package/examples/cursor-sdk/harness.json +1 -0
- package/examples/cursor-sdk/model.json +1 -0
- package/examples/cursor-sdk/native-limits.json +1 -0
- package/examples/cursor-sdk/native-tool-policy.json +1 -0
- package/ontology/behavior-categories.v1.json +46 -0
- package/package.json +66 -0
- package/release/0.1.0/KNOWN_LIMITATIONS.md +35 -0
- package/release/0.1.0/README.md +46 -0
- package/release/0.1.0/reproducibility.json +61 -0
- package/release/0.2.0/KNOWN_LIMITATIONS.md +40 -0
- package/release/0.2.0/README.md +42 -0
- package/release/0.2.0/reproducibility.json +73 -0
- package/release/0.2.1/KNOWN_LIMITATIONS.md +18 -0
- package/release/0.2.1/README.md +63 -0
- package/release/0.2.1/reproducibility.json +73 -0
- package/release/README.md +13 -0
- package/schemas/aggregation.v1.json +293 -0
- package/schemas/behavior-assertions.v1.json +220 -0
- package/schemas/deepseek-runtime-composition.v1.schema.json +122 -0
- package/schemas/experiment.v1.schema.json +188 -0
- package/schemas/human-calibration.v1.json +336 -0
- package/schemas/normalization-integrity.v1.json +302 -0
- package/schemas/run-bundles/v1.json +1040 -0
- package/schemas/run-queue.v1.schema.json +206 -0
- package/schemas/semantic-judge.v1.json +380 -0
- package/schemas/structural-observations.v1.json +131 -0
- package/schemas/task-packet-freeze.v1.schema.json +106 -0
- package/schemas/task-packet.v1.schema.json +234 -0
- package/schemas/uniform-events/v1.json +289 -0
- package/scripts/atlas-grafana.sh +18 -0
|
@@ -0,0 +1,519 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
import { cp, lstat, mkdtemp, realpath, rm, stat } from "node:fs/promises";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
6
|
+
import { promisify } from "node:util";
|
|
7
|
+
import { assertNoDuplicateJsonKeys, digestMetadata, validateArtifact, validateRunManifestEvidence } from "./artifacts.js";
|
|
8
|
+
import { captureCodexAppServer, CODEX_ADAPTER_VERSION, CODEX_APP_SERVER_VERSION, CODEX_DEFAULT_SHUTDOWN_GRACE_MS, CODEX_HARNESS, describeAndValidateCodexDataset, } from "./codex.js";
|
|
9
|
+
import { isSafeArtifactRelativePath, resolveBundleConfiguration } from "./contracts.js";
|
|
10
|
+
import { createAttemptIdentity, createRunIdentity, executeRunAttempt, } from "./lifecycle.js";
|
|
11
|
+
import { createRunBundleAssembler, qualifyRunBundle, } from "./run-bundles.js";
|
|
12
|
+
import { readBoundedFile, readRunQueue } from "./scheduler.js";
|
|
13
|
+
import { assertTaskPacketAdmitted, formatErrors } from "./task-packets.js";
|
|
14
|
+
import { executeVerifier } from "./verifiers.js";
|
|
15
|
+
import { cleanupWorkspace, materializeWorkspace } from "./workspaces.js";
|
|
16
|
+
export const CODEX_CONFIG_SCHEMA_VERSION = "ebo.codex-config/v1";
|
|
17
|
+
export const CODEX_CONTRACT_DIGEST = "sha256:e5f798fd1343c539f01fedea0e8a84a43c080fcca4615c80eb04a5edab4f7d0a";
|
|
18
|
+
const execFileAsync = promisify(execFile);
|
|
19
|
+
const ATTEMPT_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
|
|
20
|
+
/** Execute and retain one caller-configured Codex app-server attempt without scheduling or retrying it. */
|
|
21
|
+
export async function captureCodexAppServerRun(options) {
|
|
22
|
+
if (options.definition.run.harness.id !== CODEX_HARNESS) {
|
|
23
|
+
throw new Error(`Codex run harness must be ${CODEX_HARNESS}.`);
|
|
24
|
+
}
|
|
25
|
+
if (options.definition.run.model.id !== options.configuration.model) {
|
|
26
|
+
throw new Error("The declared model must match the Codex app-server configuration.");
|
|
27
|
+
}
|
|
28
|
+
const shutdownGraceMs = options.shutdownGraceMs ?? CODEX_DEFAULT_SHUTDOWN_GRACE_MS;
|
|
29
|
+
const assembler = await createRunBundleAssembler(withPinnedRuntime(options.definition));
|
|
30
|
+
let workspace;
|
|
31
|
+
let workspaceOutcome;
|
|
32
|
+
let workspaceOutcomePromise;
|
|
33
|
+
let verifierResult;
|
|
34
|
+
let capture;
|
|
35
|
+
let nativeEvidenceRegistered = false;
|
|
36
|
+
const captureWorkspace = async (context) => {
|
|
37
|
+
if (workspaceOutcome !== undefined)
|
|
38
|
+
return workspaceOutcome;
|
|
39
|
+
if (workspace?.status !== "ready" || workspace.path === undefined || workspace.artifactId === undefined) {
|
|
40
|
+
throw new Error("Codex capture requires a ready retained workspace before outcome packaging.");
|
|
41
|
+
}
|
|
42
|
+
workspaceOutcomePromise ??= assembler.captureWorkspaceOutcome({
|
|
43
|
+
startPath: options.startingWorkspacePath,
|
|
44
|
+
finalPath: workspace.path,
|
|
45
|
+
id: workspace.artifactId,
|
|
46
|
+
source: "codex-workspace-outcome",
|
|
47
|
+
...(options.workspaceOutcomeExcludedDirectoryNames === undefined ? {} : {
|
|
48
|
+
excludeDirectoryNames: options.workspaceOutcomeExcludedDirectoryNames,
|
|
49
|
+
}),
|
|
50
|
+
...(options.workspaceOutcomeRespectsGitignore === undefined ? {} : {
|
|
51
|
+
respectGitignore: options.workspaceOutcomeRespectsGitignore,
|
|
52
|
+
}),
|
|
53
|
+
...(options.workspaceOutcomeOmitsEmptyDirectories === undefined ? {} : {
|
|
54
|
+
omitEmptyDirectories: options.workspaceOutcomeOmitsEmptyDirectories,
|
|
55
|
+
}),
|
|
56
|
+
}, context === undefined || options.verifier === undefined ? undefined : async (path, outcome) => {
|
|
57
|
+
verifierResult = await options.verifier(context, outcome, path);
|
|
58
|
+
});
|
|
59
|
+
workspaceOutcome = await workspaceOutcomePromise;
|
|
60
|
+
return workspaceOutcome;
|
|
61
|
+
};
|
|
62
|
+
const registerNativeEvidence = async () => {
|
|
63
|
+
if (nativeEvidenceRegistered || capture === undefined)
|
|
64
|
+
return;
|
|
65
|
+
if (await nonempty(`${assembler.bundleRoot}/session.jsonl`)) {
|
|
66
|
+
await assembler.registerArtifact({
|
|
67
|
+
id: "session",
|
|
68
|
+
source: CODEX_HARNESS,
|
|
69
|
+
kind: "session",
|
|
70
|
+
mediaType: "application/x-ndjson",
|
|
71
|
+
sharingClass: "restricted",
|
|
72
|
+
relativePath: "session.jsonl",
|
|
73
|
+
...(capture.threadId === undefined ? {} : { nativeReference: { type: "session", id: capture.threadId } }),
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
await assembler.writeJsonArtifact({
|
|
77
|
+
id: "telemetry",
|
|
78
|
+
source: CODEX_HARNESS,
|
|
79
|
+
kind: "telemetry",
|
|
80
|
+
mediaType: "application/json",
|
|
81
|
+
sharingClass: "restricted",
|
|
82
|
+
relativePath: "telemetry/codex.json",
|
|
83
|
+
}, capture.telemetry);
|
|
84
|
+
nativeEvidenceRegistered = true;
|
|
85
|
+
};
|
|
86
|
+
const coordinatedWorkspace = {
|
|
87
|
+
setup: async (context) => {
|
|
88
|
+
workspace = await options.workspace.setup(context);
|
|
89
|
+
return workspace;
|
|
90
|
+
},
|
|
91
|
+
cleanup: async (context) => {
|
|
92
|
+
if (workspace?.status === "ready")
|
|
93
|
+
await captureWorkspace();
|
|
94
|
+
await options.workspace.cleanup?.(context);
|
|
95
|
+
},
|
|
96
|
+
};
|
|
97
|
+
const run = createRunIdentity({
|
|
98
|
+
id: options.definition.run.id,
|
|
99
|
+
taskId: options.definition.run.task.id,
|
|
100
|
+
modelId: options.definition.run.model.id,
|
|
101
|
+
harnessId: options.definition.run.harness.id,
|
|
102
|
+
});
|
|
103
|
+
const attemptIdentity = createAttemptIdentity(options.definition.run.id, options.definition.attempt.number, options.definition.attempt.id, options.definition.attempt.retryOf);
|
|
104
|
+
const performCapture = options.capture ?? captureCodexAppServer;
|
|
105
|
+
const attempt = await executeRunAttempt({
|
|
106
|
+
run,
|
|
107
|
+
assessmentMode: options.definition.run.assessmentMode,
|
|
108
|
+
attempt: attemptIdentity,
|
|
109
|
+
workspace: coordinatedWorkspace,
|
|
110
|
+
harness: async ({ signal, registerShutdown }) => {
|
|
111
|
+
if (workspace?.status !== "ready" || workspace.path === undefined)
|
|
112
|
+
throw new Error("Codex run requires a ready workspace.");
|
|
113
|
+
capture = await performCapture({
|
|
114
|
+
runId: options.definition.run.id,
|
|
115
|
+
attemptId: options.definition.attempt.id,
|
|
116
|
+
workspacePath: workspace.path,
|
|
117
|
+
prompt: options.prompt,
|
|
118
|
+
configuration: options.configuration,
|
|
119
|
+
evidencePath: `${assembler.bundleRoot}/session.jsonl`,
|
|
120
|
+
signal,
|
|
121
|
+
registerShutdown,
|
|
122
|
+
shutdownGraceMs,
|
|
123
|
+
});
|
|
124
|
+
if (capture.terminalStatus === "completed") {
|
|
125
|
+
return { status: "completed", completionEvidence: capture.terminal, evidence: durableCaptureEvidence(capture) };
|
|
126
|
+
}
|
|
127
|
+
if (capture.terminalStatus === "interrupted") {
|
|
128
|
+
return { status: "interrupted", reason: "Codex turn was interrupted.", evidence: durableCaptureEvidence(capture) };
|
|
129
|
+
}
|
|
130
|
+
return {
|
|
131
|
+
status: "failed",
|
|
132
|
+
failureClass: "infrastructure",
|
|
133
|
+
reason: capture.gaps.find(({ kind }) => kind === "capture-error")?.detail ?? `Codex turn ended as ${String(capture.terminalStatus)}.`,
|
|
134
|
+
evidence: durableCaptureEvidence(capture),
|
|
135
|
+
};
|
|
136
|
+
},
|
|
137
|
+
...(options.verifier === undefined ? {} : {
|
|
138
|
+
verifier: async (context) => {
|
|
139
|
+
await captureWorkspace(context);
|
|
140
|
+
if (verifierResult === undefined)
|
|
141
|
+
throw new Error("Codex verifier did not return a result.");
|
|
142
|
+
await assembler.writeJsonArtifact({
|
|
143
|
+
id: "verifier",
|
|
144
|
+
source: "ebo-verifier",
|
|
145
|
+
kind: "verifier",
|
|
146
|
+
mediaType: "application/json",
|
|
147
|
+
sharingClass: "restricted",
|
|
148
|
+
relativePath: "verifier.json",
|
|
149
|
+
}, verifierResult);
|
|
150
|
+
return { status: verifierResult.status, ...(verifierResult.error === undefined ? {} : { error: verifierResult.error }), evidence: verifierResult };
|
|
151
|
+
},
|
|
152
|
+
}),
|
|
153
|
+
evidence: { flush: registerNativeEvidence },
|
|
154
|
+
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
155
|
+
...(options.maxWallClockMs === undefined ? {} : { maxWallClockMs: options.maxWallClockMs }),
|
|
156
|
+
shutdownGraceMs,
|
|
157
|
+
});
|
|
158
|
+
if (workspace?.status === "ready")
|
|
159
|
+
await captureWorkspace().catch(() => undefined);
|
|
160
|
+
await registerNativeEvidence();
|
|
161
|
+
const missingEvidence = [];
|
|
162
|
+
if (capture?.threadId === undefined)
|
|
163
|
+
missingEvidence.push({
|
|
164
|
+
kind: "session-identity", reason: "not-collected", affects: ["semantic"], detail: "Codex did not return an owned thread identity.",
|
|
165
|
+
});
|
|
166
|
+
for (const gap of capture?.gaps ?? [])
|
|
167
|
+
missingEvidence.push({
|
|
168
|
+
kind: gap.kind,
|
|
169
|
+
reason: gap.kind === "interrupt-acknowledgement" ? "process-interrupted" : "not-collected",
|
|
170
|
+
affects: [gap.kind.includes("telemetry") ? "timing-resource" : "semantic"],
|
|
171
|
+
detail: gap.detail,
|
|
172
|
+
});
|
|
173
|
+
const receipt = capture?.telemetry.telemetry.receipt;
|
|
174
|
+
if (receipt?.status !== "received")
|
|
175
|
+
missingEvidence.push({
|
|
176
|
+
kind: "telemetry-receipt",
|
|
177
|
+
reason: receipt?.status === "not-checked" ? "not-checked" : "not-collected",
|
|
178
|
+
affects: ["timing-resource"],
|
|
179
|
+
detail: receipt === undefined ? "Codex telemetry evidence was not produced." : `Codex OTLP receipt is ${receipt.status}.`,
|
|
180
|
+
});
|
|
181
|
+
missingEvidence.push({
|
|
182
|
+
kind: "child-history",
|
|
183
|
+
reason: "not-checked",
|
|
184
|
+
affects: ["semantic"],
|
|
185
|
+
detail: "Child histories are not claimed unless explicit native delegation identities are collected.",
|
|
186
|
+
});
|
|
187
|
+
const qualificationOptions = {
|
|
188
|
+
startingWorkspacePath: options.startingWorkspacePath,
|
|
189
|
+
semanticEvidenceKinds: ["session"],
|
|
190
|
+
relatedSessionIds: [],
|
|
191
|
+
...(options.workspaceOutcomeExcludedDirectoryNames === undefined ? {} : {
|
|
192
|
+
workspaceOutcomeExcludedDirectoryNames: options.workspaceOutcomeExcludedDirectoryNames,
|
|
193
|
+
}),
|
|
194
|
+
...(options.workspaceOutcomeRespectsGitignore === undefined ? {} : {
|
|
195
|
+
workspaceOutcomeRespectsGitignore: options.workspaceOutcomeRespectsGitignore,
|
|
196
|
+
}),
|
|
197
|
+
...(options.workspaceOutcomeOmitsEmptyDirectories === undefined ? {} : {
|
|
198
|
+
workspaceOutcomeOmitsEmptyDirectories: options.workspaceOutcomeOmitsEmptyDirectories,
|
|
199
|
+
}),
|
|
200
|
+
};
|
|
201
|
+
const terminal = structuredClone(attempt.terminal);
|
|
202
|
+
if (workspaceOutcome === undefined)
|
|
203
|
+
delete terminal.workspaceArtifactId;
|
|
204
|
+
const manifest = await assembler.finalize({ terminal, missingEvidence, qualification: qualificationOptions });
|
|
205
|
+
const qualification = await qualifyRunBundle(assembler.bundleRoot, qualificationOptions);
|
|
206
|
+
if (capture === undefined || capture.threadId === undefined)
|
|
207
|
+
return { attempt, manifest, qualification, capture };
|
|
208
|
+
const { dataset, coverage } = await describeAndValidateCodexDataset(capture);
|
|
209
|
+
return { attempt, manifest, qualification, capture, normalized: dataset, coverage };
|
|
210
|
+
}
|
|
211
|
+
/** Execute exactly one persisted frozen queue entry through the pinned Codex app-server adapter. */
|
|
212
|
+
export async function runCodexQueueEntry(options) {
|
|
213
|
+
const bundleRoot = resolve(options.bundleRoot);
|
|
214
|
+
const queue = readRunQueue(options.queuePath, undefined, { bundleRoot });
|
|
215
|
+
const matches = queue.entries.filter((entry) => entry.runId === options.runId);
|
|
216
|
+
if (matches.length !== 1)
|
|
217
|
+
throw new Error(matches.length === 0
|
|
218
|
+
? `Run "${options.runId}" is not in the queue.` : `Run "${options.runId}" matches more than one queue entry.`);
|
|
219
|
+
const entry = matches[0];
|
|
220
|
+
const model = resolveCodexConfigurationRecord(bundleRoot, entry.configuration.model, "model");
|
|
221
|
+
const harness = resolveCodexConfigurationRecord(bundleRoot, entry.configuration.harness, "harness");
|
|
222
|
+
const limits = resolveCodexConfigurationRecord(bundleRoot, entry.configuration.nativeLimits, "native-limits");
|
|
223
|
+
const toolPolicy = resolveCodexConfigurationRecord(bundleRoot, entry.configuration.nativeToolPolicy, "native-tool-policy");
|
|
224
|
+
const captureProfile = resolveCodexConfigurationRecord(bundleRoot, queue.captureProfile, "capture-profile");
|
|
225
|
+
if (entry.harness.id !== CODEX_HARNESS)
|
|
226
|
+
throw new Error(`Queue harness must be ${CODEX_HARNESS}.`);
|
|
227
|
+
const runtime = await (options.probeRuntime ?? probeCodexRuntime)(harness.executable);
|
|
228
|
+
if (runtime.version !== `codex-cli ${harness.version}`) {
|
|
229
|
+
throw new Error(`Codex ${harness.version} is required; received ${runtime.version}.`);
|
|
230
|
+
}
|
|
231
|
+
const inspection = assertTaskPacketAdmitted(bundleRoot, entry.task.packetRef.locator);
|
|
232
|
+
const packet = inspection.packet;
|
|
233
|
+
if (inspection.packetDigest === null || inspection.packetDigest.value !== entry.task.packetRef.digest.value
|
|
234
|
+
|| inspection.packetDigest.algorithm !== entry.task.packetRef.digest.algorithm) {
|
|
235
|
+
throw new Error(`Task packet "${entry.task.packetRef.locator}" changed from its frozen queue reference.`);
|
|
236
|
+
}
|
|
237
|
+
const attemptId = options.attemptId ?? randomUUID();
|
|
238
|
+
if (!ATTEMPT_ID_PATTERN.test(attemptId))
|
|
239
|
+
throw new Error("Attempt ID must be one safe path component.");
|
|
240
|
+
const outputRoot = resolve(options.outputRoot);
|
|
241
|
+
const destination = join(outputRoot, entry.runId, attemptId);
|
|
242
|
+
const fromOutput = relative(outputRoot, destination);
|
|
243
|
+
if (fromOutput === "" || isAbsolute(fromOutput) || fromOutput === ".." || fromOutput.startsWith(`..${sep}`)) {
|
|
244
|
+
throw new Error("Attempt destination escapes the selected output root.");
|
|
245
|
+
}
|
|
246
|
+
if (await exists(destination))
|
|
247
|
+
throw new Error(`Attempt destination "${destination}" already exists and is never replaced.`);
|
|
248
|
+
const workspace = await materializeWorkspace({
|
|
249
|
+
bundleRoot,
|
|
250
|
+
packetLocator: entry.task.packetRef.locator,
|
|
251
|
+
freezeLocator: entry.task.freezeLocator,
|
|
252
|
+
...(options.workspaceRoot === undefined ? {} : { workspaceParent: options.workspaceRoot }),
|
|
253
|
+
});
|
|
254
|
+
if (workspace.status !== "ready")
|
|
255
|
+
throw new Error(`Workspace materialization failed: ${workspace.error ?? "unknown failure"}`);
|
|
256
|
+
let baselineRoot;
|
|
257
|
+
let captureStarted = false;
|
|
258
|
+
try {
|
|
259
|
+
baselineRoot = await mkdtemp(join(tmpdir(), "ebo-codex-baseline-"));
|
|
260
|
+
const startingWorkspacePath = join(baselineRoot, "workspace");
|
|
261
|
+
await cp(workspace.path, startingWorkspacePath, { recursive: true, preserveTimestamps: true, force: false });
|
|
262
|
+
const verifierReference = packet.assessmentMode === "verified" ? packet.restricted.verifier : undefined;
|
|
263
|
+
const verifierFormat = verifierReference?.locator.toLowerCase().endsWith(".mjs") ? "module" : "commonjs";
|
|
264
|
+
const definition = buildDefinition(queue.captureProfile, entry, packet, destination, attemptId, model, harness, verifierFormat);
|
|
265
|
+
const result = await captureCodexAppServerRun({
|
|
266
|
+
definition,
|
|
267
|
+
startingWorkspacePath,
|
|
268
|
+
workspace: {
|
|
269
|
+
setup: () => {
|
|
270
|
+
captureStarted = true;
|
|
271
|
+
return { status: "ready", path: workspace.path, artifactId: "workspace", retained: true };
|
|
272
|
+
},
|
|
273
|
+
cleanup: () => cleanupWorkspace(workspace),
|
|
274
|
+
},
|
|
275
|
+
configuration: {
|
|
276
|
+
executable: runtime.path,
|
|
277
|
+
...(options.executableArgs === undefined ? {} : { executableArgs: options.executableArgs }),
|
|
278
|
+
version: harness.version,
|
|
279
|
+
provider: model.provider,
|
|
280
|
+
model: model.model,
|
|
281
|
+
effort: model.effort,
|
|
282
|
+
approvalPolicy: toolPolicy.approvalPolicy,
|
|
283
|
+
sandbox: toolPolicy.sandbox,
|
|
284
|
+
...(captureProfile.telemetrySignals === undefined ? {} : {
|
|
285
|
+
telemetry: { signals: captureProfile.telemetrySignals },
|
|
286
|
+
}),
|
|
287
|
+
},
|
|
288
|
+
prompt: packet.agentInput.prompt,
|
|
289
|
+
...(verifierReference === undefined ? {} : {
|
|
290
|
+
verifier: (context, outcome, projectedPath) => executeVerifier({
|
|
291
|
+
bundleId: definition.bundleId,
|
|
292
|
+
verifierRoot: bundleRoot,
|
|
293
|
+
verifier: verifierReference,
|
|
294
|
+
workspacePath: projectedPath,
|
|
295
|
+
workspaceFingerprint: outcome.fingerprint,
|
|
296
|
+
workspace: { artifactId: outcome.descriptor.id, digest: outcome.descriptor.digest, fingerprint: outcome.fingerprint },
|
|
297
|
+
artifactRoot: destination,
|
|
298
|
+
moduleFormat: verifierFormat,
|
|
299
|
+
signal: context.signal,
|
|
300
|
+
}),
|
|
301
|
+
}),
|
|
302
|
+
maxWallClockMs: queue.coordinatorBudget.maxWallClockMs,
|
|
303
|
+
...(limits.shutdownGraceMs === undefined ? {} : { shutdownGraceMs: limits.shutdownGraceMs }),
|
|
304
|
+
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
305
|
+
...(options.capture === undefined ? {} : { capture: options.capture }),
|
|
306
|
+
...(captureProfile.workspaceOutcome === undefined ? {} : {
|
|
307
|
+
workspaceOutcomeExcludedDirectoryNames: captureProfile.workspaceOutcome.excludeDirectoryNames,
|
|
308
|
+
...(captureProfile.workspaceOutcome.respectGitignore === undefined ? {} : {
|
|
309
|
+
workspaceOutcomeRespectsGitignore: captureProfile.workspaceOutcome.respectGitignore,
|
|
310
|
+
}),
|
|
311
|
+
...(captureProfile.workspaceOutcome.omitEmptyDirectories === undefined ? {} : {
|
|
312
|
+
workspaceOutcomeOmitsEmptyDirectories: captureProfile.workspaceOutcome.omitEmptyDirectories,
|
|
313
|
+
}),
|
|
314
|
+
}),
|
|
315
|
+
});
|
|
316
|
+
const manifest = reopenManifest(destination);
|
|
317
|
+
return {
|
|
318
|
+
runId: entry.runId,
|
|
319
|
+
attemptId,
|
|
320
|
+
bundlePath: destination,
|
|
321
|
+
terminal: manifest.terminal,
|
|
322
|
+
classification: result.attempt.classification.kind,
|
|
323
|
+
captureQualification: result.qualification.status,
|
|
324
|
+
assessmentMode: packet.assessmentMode,
|
|
325
|
+
...(result.capture?.threadId === undefined ? {} : { threadId: result.capture.threadId }),
|
|
326
|
+
...(result.capture?.turnId === undefined ? {} : { turnId: result.capture.turnId }),
|
|
327
|
+
normalizedEvents: result.normalized?.events.length ?? 0,
|
|
328
|
+
unmappedNativeRecords: result.normalized?.unmapped.length ?? 0,
|
|
329
|
+
...(workspace.state === "ready" ? { retainedWorkspacePath: workspace.path } : {}),
|
|
330
|
+
};
|
|
331
|
+
}
|
|
332
|
+
finally {
|
|
333
|
+
if (baselineRoot !== undefined)
|
|
334
|
+
await rm(baselineRoot, { recursive: true, force: true });
|
|
335
|
+
if (!captureStarted && workspace.state === "ready")
|
|
336
|
+
await cleanupWorkspace(workspace).catch(() => undefined);
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
export function resolveCodexConfigurationRecord(bundleRoot, reference, kind) {
|
|
340
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(resolveBundleConfiguration(bundleRoot, reference));
|
|
341
|
+
assertNoDuplicateJsonKeys(text);
|
|
342
|
+
const value = JSON.parse(text);
|
|
343
|
+
if (!isRecord(value) || value.schemaVersion !== CODEX_CONFIG_SCHEMA_VERSION || value.kind !== kind) {
|
|
344
|
+
throw configError(reference, `must declare ${CODEX_CONFIG_SCHEMA_VERSION} kind ${kind}`);
|
|
345
|
+
}
|
|
346
|
+
validateConfiguration(value, kind, reference);
|
|
347
|
+
return value;
|
|
348
|
+
}
|
|
349
|
+
async function probeCodexRuntime(executable) {
|
|
350
|
+
if (!isAbsolute(executable))
|
|
351
|
+
throw new Error("Codex harness executable must be an absolute path.");
|
|
352
|
+
const path = await realpath(executable);
|
|
353
|
+
const { stdout } = await execFileAsync(path, ["--version"], { encoding: "utf8", timeout: 5_000 });
|
|
354
|
+
return { path, version: stdout.trim() };
|
|
355
|
+
}
|
|
356
|
+
function validateConfiguration(record, kind, reference) {
|
|
357
|
+
if (kind === "model") {
|
|
358
|
+
keys(record, ["schemaVersion", "kind", "provider", "model", "effort"], ["provider", "model", "effort"], reference);
|
|
359
|
+
if (record.provider !== "openai")
|
|
360
|
+
throw configError(reference, "provider must be openai");
|
|
361
|
+
requiredText(record.model, "model", reference);
|
|
362
|
+
if (!["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"].includes(String(record.effort)))
|
|
363
|
+
throw configError(reference, "effort is invalid");
|
|
364
|
+
}
|
|
365
|
+
else if (kind === "harness") {
|
|
366
|
+
keys(record, ["schemaVersion", "kind", "adapter", "executable", "version", "contractDigest"], ["adapter", "executable", "version", "contractDigest"], reference);
|
|
367
|
+
if (record.adapter !== CODEX_HARNESS || record.version !== CODEX_APP_SERVER_VERSION)
|
|
368
|
+
throw configError(reference, "adapter or version is not pinned");
|
|
369
|
+
requiredText(record.executable, "executable", reference);
|
|
370
|
+
if (!isAbsolute(record.executable))
|
|
371
|
+
throw configError(reference, "executable must be absolute");
|
|
372
|
+
if (record.contractDigest !== CODEX_CONTRACT_DIGEST)
|
|
373
|
+
throw configError(reference, `contractDigest must be ${CODEX_CONTRACT_DIGEST}`);
|
|
374
|
+
}
|
|
375
|
+
else if (kind === "native-limits") {
|
|
376
|
+
keys(record, ["schemaVersion", "kind", "shutdownGraceMs"], [], reference);
|
|
377
|
+
if (record.shutdownGraceMs !== undefined && (!Number.isSafeInteger(record.shutdownGraceMs) || record.shutdownGraceMs < 1))
|
|
378
|
+
throw configError(reference, "shutdownGraceMs must be positive");
|
|
379
|
+
}
|
|
380
|
+
else if (kind === "native-tool-policy") {
|
|
381
|
+
keys(record, ["schemaVersion", "kind", "approvalPolicy", "sandbox"], ["approvalPolicy", "sandbox"], reference);
|
|
382
|
+
if (!["untrusted", "on-request", "never"].includes(String(record.approvalPolicy)))
|
|
383
|
+
throw configError(reference, "approvalPolicy is invalid");
|
|
384
|
+
if (!["read-only", "workspace-write", "danger-full-access"].includes(String(record.sandbox)))
|
|
385
|
+
throw configError(reference, "sandbox is invalid");
|
|
386
|
+
}
|
|
387
|
+
else {
|
|
388
|
+
keys(record, ["schemaVersion", "kind", "telemetrySignals", "workspaceOutcome"], [], reference);
|
|
389
|
+
if (record.telemetrySignals !== undefined && (!Array.isArray(record.telemetrySignals)
|
|
390
|
+
|| new Set(record.telemetrySignals).size !== record.telemetrySignals.length
|
|
391
|
+
|| record.telemetrySignals.some((signal) => !["logs", "traces", "metrics"].includes(String(signal))))) {
|
|
392
|
+
throw configError(reference, "telemetrySignals is invalid");
|
|
393
|
+
}
|
|
394
|
+
if (record.workspaceOutcome !== undefined)
|
|
395
|
+
validateWorkspaceOutcome(record.workspaceOutcome, reference);
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
function validateWorkspaceOutcome(value, reference) {
|
|
399
|
+
if (!isRecord(value))
|
|
400
|
+
throw configError(reference, "workspaceOutcome must be an object");
|
|
401
|
+
keys(value, ["excludeDirectoryNames", "respectGitignore", "omitEmptyDirectories"], ["excludeDirectoryNames"], reference);
|
|
402
|
+
if (!Array.isArray(value.excludeDirectoryNames) || value.excludeDirectoryNames.some((name) => typeof name !== "string"
|
|
403
|
+
|| name.includes("/") || !isSafeArtifactRelativePath(name)))
|
|
404
|
+
throw configError(reference, "workspace exclusions are invalid");
|
|
405
|
+
for (const flag of ["respectGitignore", "omitEmptyDirectories"]) {
|
|
406
|
+
if (value[flag] !== undefined && typeof value[flag] !== "boolean")
|
|
407
|
+
throw configError(reference, `${flag} must be boolean`);
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
function buildDefinition(captureProfile, entry, packet, bundleRoot, attemptId, model, harness, verifierFormat) {
|
|
411
|
+
return {
|
|
412
|
+
bundleRoot,
|
|
413
|
+
bundleId: `bundle-${attemptId}`,
|
|
414
|
+
run: {
|
|
415
|
+
id: entry.runId,
|
|
416
|
+
trial: { index: entry.trial.index },
|
|
417
|
+
assessmentMode: packet.assessmentMode,
|
|
418
|
+
task: { id: entry.task.id, digest: `sha256:${entry.task.packetRef.digest.value}` },
|
|
419
|
+
fixture: { id: packet.agentInput.fixture.source.locator, digest: `sha256:${packet.agentInput.fixture.source.digest.value}` },
|
|
420
|
+
model: { provider: model.provider, id: model.model, configurationDigest: `sha256:${entry.configuration.model.digest.value}` },
|
|
421
|
+
harness: { id: CODEX_HARNESS, version: harness.version, configurationDigest: `sha256:${entry.configuration.harness.digest.value}` },
|
|
422
|
+
runtime: [
|
|
423
|
+
{ source: "OpenAI", name: "codex-app-server", version: harness.version },
|
|
424
|
+
{ source: "EBO", name: "codex-adapter", version: CODEX_ADAPTER_VERSION },
|
|
425
|
+
],
|
|
426
|
+
...(packet.assessmentMode === "verified" ? {
|
|
427
|
+
verifier: { locator: packet.restricted.verifier.locator, digest: `sha256:${packet.restricted.verifier.digest.value}`, format: verifierFormat },
|
|
428
|
+
} : {}),
|
|
429
|
+
},
|
|
430
|
+
attempt: { id: attemptId, number: 1 },
|
|
431
|
+
configuration: {
|
|
432
|
+
digest: `sha256:${digestMetadata({ model: entry.configuration.model, harness: entry.configuration.harness, captureProfile }).value}`,
|
|
433
|
+
captureProfileDigest: `sha256:${captureProfile.digest.value}`,
|
|
434
|
+
budgetDigest: `sha256:${entry.configuration.nativeLimits.digest.value}`,
|
|
435
|
+
toolPolicyDigest: `sha256:${entry.configuration.nativeToolPolicy.digest.value}`,
|
|
436
|
+
},
|
|
437
|
+
};
|
|
438
|
+
}
|
|
439
|
+
function withPinnedRuntime(definition) {
|
|
440
|
+
return {
|
|
441
|
+
...structuredClone(definition),
|
|
442
|
+
run: {
|
|
443
|
+
...structuredClone(definition.run),
|
|
444
|
+
harness: { ...structuredClone(definition.run.harness), id: CODEX_HARNESS, version: CODEX_APP_SERVER_VERSION },
|
|
445
|
+
runtime: definition.run.runtime.filter(({ name }) => !["codex-app-server", "codex-adapter"].includes(name)).concat([
|
|
446
|
+
{ source: "OpenAI", name: "codex-app-server", version: CODEX_APP_SERVER_VERSION },
|
|
447
|
+
{ source: "EBO", name: "codex-adapter", version: CODEX_ADAPTER_VERSION },
|
|
448
|
+
]),
|
|
449
|
+
},
|
|
450
|
+
};
|
|
451
|
+
}
|
|
452
|
+
function durableCaptureEvidence(capture) {
|
|
453
|
+
return {
|
|
454
|
+
qualification: capture.qualification,
|
|
455
|
+
...(capture.threadId === undefined ? {} : { threadId: capture.threadId }),
|
|
456
|
+
...(capture.turnId === undefined ? {} : { turnId: capture.turnId }),
|
|
457
|
+
...(capture.terminalStatus === undefined ? {} : { terminalStatus: capture.terminalStatus }),
|
|
458
|
+
gaps: capture.gaps,
|
|
459
|
+
process: {
|
|
460
|
+
status: capture.process.status,
|
|
461
|
+
partial: capture.process.partial,
|
|
462
|
+
termination: capture.process.termination,
|
|
463
|
+
stdoutFrames: capture.process.stdoutFrames,
|
|
464
|
+
exitCode: capture.process.launch.exitCode,
|
|
465
|
+
signal: capture.process.launch.signal,
|
|
466
|
+
stderrSizeBytes: capture.process.stderr.sizeBytes,
|
|
467
|
+
stderrTruncated: capture.process.stderr.truncated,
|
|
468
|
+
},
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
function reopenManifest(bundleRoot) {
|
|
472
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(readBoundedFile(join(bundleRoot, "manifest.json"), "Run manifest"));
|
|
473
|
+
assertNoDuplicateJsonKeys(text);
|
|
474
|
+
const value = JSON.parse(text);
|
|
475
|
+
const errors = [...validateArtifact("manifest.json", value), ...validateRunManifestEvidence("manifest.json", value, bundleRoot)];
|
|
476
|
+
if (errors.length > 0)
|
|
477
|
+
throw new Error(`Retained run manifest failed validation:\n${formatErrors(errors)}`);
|
|
478
|
+
return value;
|
|
479
|
+
}
|
|
480
|
+
async function nonempty(path) {
|
|
481
|
+
try {
|
|
482
|
+
return (await stat(path)).size > 0;
|
|
483
|
+
}
|
|
484
|
+
catch (error) {
|
|
485
|
+
if (error.code === "ENOENT")
|
|
486
|
+
return false;
|
|
487
|
+
throw error;
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
async function exists(path) {
|
|
491
|
+
try {
|
|
492
|
+
await lstat(path);
|
|
493
|
+
return true;
|
|
494
|
+
}
|
|
495
|
+
catch (error) {
|
|
496
|
+
if (error.code === "ENOENT")
|
|
497
|
+
return false;
|
|
498
|
+
throw error;
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
function keys(record, allowed, required, reference) {
|
|
502
|
+
const set = new Set(allowed);
|
|
503
|
+
const unexpected = Object.keys(record).find((key) => !set.has(key));
|
|
504
|
+
if (unexpected !== undefined)
|
|
505
|
+
throw configError(reference, `contains unknown field "${unexpected}"`);
|
|
506
|
+
const missing = required.find((key) => record[key] === undefined);
|
|
507
|
+
if (missing !== undefined)
|
|
508
|
+
throw configError(reference, `is missing "${missing}"`);
|
|
509
|
+
}
|
|
510
|
+
function requiredText(value, field, reference) {
|
|
511
|
+
if (typeof value !== "string" || value.trim() === "")
|
|
512
|
+
throw configError(reference, `${field} must be nonempty`);
|
|
513
|
+
}
|
|
514
|
+
function configError(reference, detail) {
|
|
515
|
+
return new Error(`Codex configuration "${reference.locator}" ${detail}.`);
|
|
516
|
+
}
|
|
517
|
+
function isRecord(value) {
|
|
518
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
519
|
+
}
|