engineering-behavior-observatory 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +151 -0
- package/contracts/codex-app-server-0.150.1/manifest.json +25 -0
- package/contracts/codex-app-server-0.150.1/schema/InitializeParams.json +84 -0
- package/contracts/codex-app-server-0.150.1/schema/JSONRPCMessage.json +137 -0
- package/contracts/codex-app-server-0.150.1/schema/v2/ThreadReadParams.json +17 -0
- package/contracts/codex-app-server-0.150.1/schema/v2/ThreadTokenUsageUpdatedNotification.json +82 -0
- package/contracts/codex-app-server-0.150.1/schema/v2/TurnInterruptParams.json +17 -0
- package/contracts/codex-app-server-0.150.1/types/AskForApproval.ts +5 -0
- package/contracts/codex-app-server-0.150.1/types/ClientNotification.ts +5 -0
- package/contracts/codex-app-server-0.150.1/types/SandboxMode.ts +5 -0
- package/contracts/codex-app-server-0.150.1/types/ThreadReadParams.ts +9 -0
- package/contracts/codex-app-server-0.150.1/types/TokenUsageBreakdown.ts +5 -0
- package/contracts/codex-app-server-0.150.1/types/TurnInterruptParams.ts +5 -0
- package/contracts/codex-app-server-0.153.4/manifest.json +27 -0
- package/contracts/codex-app-server-0.153.4/schema/JSONRPCMessage.json +137 -0
- package/contracts/codex-app-server-0.153.4/schema/v1/InitializeParams.json +84 -0
- package/contracts/codex-app-server-0.153.4/schema/v2/ThreadReadParams.json +17 -0
- package/contracts/codex-app-server-0.153.4/schema/v2/ThreadStartParams.json +515 -0
- package/contracts/codex-app-server-0.153.4/schema/v2/ThreadTokenUsageUpdatedNotification.json +82 -0
- package/contracts/codex-app-server-0.153.4/schema/v2/TurnInterruptParams.json +17 -0
- package/contracts/codex-app-server-0.153.4/schema/v2/TurnStartParams.json +921 -0
- package/contracts/codex-app-server-0.153.4/types/AskForApproval.ts +5 -0
- package/contracts/codex-app-server-0.153.4/types/ClientNotification.ts +5 -0
- package/contracts/codex-app-server-0.153.4/types/SandboxMode.ts +5 -0
- package/contracts/codex-app-server-0.153.4/types/ThreadReadParams.ts +12 -0
- package/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.ts +5 -0
- package/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.ts +5 -0
- package/contracts/openhands-agent-server-v1.44.1.json +65 -0
- package/contracts/openhands-agent-server-v1.46.0.json +46 -0
- package/dist/contracts/codex-app-server-0.153.4/types/AskForApproval.d.ts +9 -0
- package/dist/contracts/codex-app-server-0.153.4/types/AskForApproval.js +2 -0
- package/dist/contracts/codex-app-server-0.153.4/types/ClientNotification.d.ts +3 -0
- package/dist/contracts/codex-app-server-0.153.4/types/ClientNotification.js +2 -0
- package/dist/contracts/codex-app-server-0.153.4/types/SandboxMode.d.ts +1 -0
- package/dist/contracts/codex-app-server-0.153.4/types/SandboxMode.js +2 -0
- package/dist/contracts/codex-app-server-0.153.4/types/ThreadReadParams.d.ts +10 -0
- package/dist/contracts/codex-app-server-0.153.4/types/ThreadReadParams.js +2 -0
- package/dist/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.d.ts +8 -0
- package/dist/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.js +2 -0
- package/dist/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.d.ts +4 -0
- package/dist/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.js +2 -0
- package/dist/src/agent-sdk-normalizer.d.ts +22 -0
- package/dist/src/agent-sdk-normalizer.js +995 -0
- package/dist/src/agent-sdk-run.d.ts +31 -0
- package/dist/src/agent-sdk-run.js +242 -0
- package/dist/src/agent-sdk-runner.d.ts +96 -0
- package/dist/src/agent-sdk-runner.js +407 -0
- package/dist/src/agent-sdk.d.ts +294 -0
- package/dist/src/agent-sdk.js +644 -0
- package/dist/src/aggregation.d.ts +183 -0
- package/dist/src/aggregation.js +513 -0
- package/dist/src/artifacts.d.ts +36 -0
- package/dist/src/artifacts.js +1372 -0
- package/dist/src/atlas-grafana.d.ts +377 -0
- package/dist/src/atlas-grafana.js +47 -0
- package/dist/src/atlas-html.d.ts +2 -0
- package/dist/src/atlas-html.js +34 -0
- package/dist/src/atlas.d.ts +130 -0
- package/dist/src/atlas.js +235 -0
- package/dist/src/behavior-assertions.d.ts +95 -0
- package/dist/src/behavior-assertions.js +154 -0
- package/dist/src/cli.d.ts +5 -0
- package/dist/src/cli.js +957 -0
- package/dist/src/codex-judge.d.ts +44 -0
- package/dist/src/codex-judge.js +214 -0
- package/dist/src/codex-run.d.ts +114 -0
- package/dist/src/codex-run.js +519 -0
- package/dist/src/codex.d.ts +187 -0
- package/dist/src/codex.js +1028 -0
- package/dist/src/contracts.d.ts +155 -0
- package/dist/src/contracts.js +866 -0
- package/dist/src/corpus.d.ts +67 -0
- package/dist/src/corpus.js +540 -0
- package/dist/src/cursor-sdk-runner.d.ts +81 -0
- package/dist/src/cursor-sdk-runner.js +362 -0
- package/dist/src/cursor-sdk.d.ts +81 -0
- package/dist/src/cursor-sdk.js +1078 -0
- package/dist/src/deepseek-adapter.d.ts +152 -0
- package/dist/src/deepseek-adapter.js +777 -0
- package/dist/src/exports.d.ts +69 -0
- package/dist/src/exports.js +966 -0
- package/dist/src/human-calibration.d.ts +167 -0
- package/dist/src/human-calibration.js +618 -0
- package/dist/src/index.d.ts +71 -0
- package/dist/src/index.js +38 -0
- package/dist/src/lifecycle.d.ts +196 -0
- package/dist/src/lifecycle.js +1889 -0
- package/dist/src/normalization-integrity.d.ts +128 -0
- package/dist/src/normalization-integrity.js +429 -0
- package/dist/src/openhands-run.d.ts +36 -0
- package/dist/src/openhands-run.js +415 -0
- package/dist/src/openhands.d.ts +130 -0
- package/dist/src/openhands.js +859 -0
- package/dist/src/pi.d.ts +203 -0
- package/dist/src/pi.js +1345 -0
- package/dist/src/process-protocol.d.ts +245 -0
- package/dist/src/process-protocol.js +1261 -0
- package/dist/src/retained-evidence.d.ts +12 -0
- package/dist/src/retained-evidence.js +201 -0
- package/dist/src/run-bundles.d.ts +193 -0
- package/dist/src/run-bundles.js +993 -0
- package/dist/src/scheduler.d.ts +109 -0
- package/dist/src/scheduler.js +1080 -0
- package/dist/src/semantic-judge.d.ts +194 -0
- package/dist/src/semantic-judge.js +875 -0
- package/dist/src/structural-observations.d.ts +124 -0
- package/dist/src/structural-observations.js +671 -0
- package/dist/src/task-packets.d.ts +115 -0
- package/dist/src/task-packets.js +683 -0
- package/dist/src/uniform-events.d.ts +120 -0
- package/dist/src/uniform-events.js +158 -0
- package/dist/src/verifiers.d.ts +116 -0
- package/dist/src/verifiers.js +819 -0
- package/dist/src/workspaces.d.ts +56 -0
- package/dist/src/workspaces.js +1314 -0
- package/docs/README.md +60 -0
- package/docs/development/README.md +71 -0
- package/docs/development/documentation-sync.md +58 -0
- package/docs/development/extension-contracts.md +307 -0
- package/docs/evaluation/README.md +48 -0
- package/docs/evaluation/aggregation.md +112 -0
- package/docs/evaluation/behavior-assertions.md +45 -0
- package/docs/evaluation/human-calibration.md +170 -0
- package/docs/evaluation/normalization-integrity.md +80 -0
- package/docs/evaluation/semantic-judge.md +188 -0
- package/docs/evaluation/structural-observations.md +84 -0
- package/docs/evaluation/uniform-events.md +131 -0
- package/docs/guides/atlas.md +144 -0
- package/docs/guides/evidence-and-sharing.md +100 -0
- package/docs/guides/operator-guide.md +424 -0
- package/docs/guides/quickstart.md +152 -0
- package/docs/guides/telemetry.md +66 -0
- package/docs/harnesses/README.md +31 -0
- package/docs/harnesses/claude-agent-sdk.md +61 -0
- package/docs/harnesses/codex-harness.md +174 -0
- package/docs/harnesses/cursor-sdk.md +149 -0
- package/docs/harnesses/deepseek-harness.md +134 -0
- package/docs/harnesses/openhands-agent-server.md +153 -0
- package/docs/harnesses/pi-sdk.md +66 -0
- package/docs/reference/README.md +19 -0
- package/docs/reference/agent-sdk-operational-runner.md +118 -0
- package/docs/reference/cli.md +114 -0
- package/docs/reference/contracts.md +222 -0
- package/docs/reference/run-bundle-contract.md +354 -0
- package/docs/reference/run-lifecycle.md +54 -0
- package/examples/cursor-sdk/README.md +11 -0
- package/examples/cursor-sdk/capture-profile.json +1 -0
- package/examples/cursor-sdk/harness.json +1 -0
- package/examples/cursor-sdk/model.json +1 -0
- package/examples/cursor-sdk/native-limits.json +1 -0
- package/examples/cursor-sdk/native-tool-policy.json +1 -0
- package/ontology/behavior-categories.v1.json +46 -0
- package/package.json +66 -0
- package/release/0.1.0/KNOWN_LIMITATIONS.md +35 -0
- package/release/0.1.0/README.md +46 -0
- package/release/0.1.0/reproducibility.json +61 -0
- package/release/0.2.0/KNOWN_LIMITATIONS.md +40 -0
- package/release/0.2.0/README.md +42 -0
- package/release/0.2.0/reproducibility.json +73 -0
- package/release/0.2.1/KNOWN_LIMITATIONS.md +18 -0
- package/release/0.2.1/README.md +63 -0
- package/release/0.2.1/reproducibility.json +73 -0
- package/release/README.md +13 -0
- package/schemas/aggregation.v1.json +293 -0
- package/schemas/behavior-assertions.v1.json +220 -0
- package/schemas/deepseek-runtime-composition.v1.schema.json +122 -0
- package/schemas/experiment.v1.schema.json +188 -0
- package/schemas/human-calibration.v1.json +336 -0
- package/schemas/normalization-integrity.v1.json +302 -0
- package/schemas/run-bundles/v1.json +1040 -0
- package/schemas/run-queue.v1.schema.json +206 -0
- package/schemas/semantic-judge.v1.json +380 -0
- package/schemas/structural-observations.v1.json +131 -0
- package/schemas/task-packet-freeze.v1.schema.json +106 -0
- package/schemas/task-packet.v1.schema.json +234 -0
- package/schemas/uniform-events/v1.json +289 -0
- package/scripts/atlas-grafana.sh +18 -0
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
import type { AskForApproval } from "../contracts/codex-app-server-0.153.4/types/AskForApproval.js";
|
|
2
|
+
import type { SandboxMode } from "../contracts/codex-app-server-0.153.4/types/SandboxMode.js";
|
|
3
|
+
import type { TokenUsageBreakdown } from "../contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.js";
|
|
4
|
+
import { type ProtocolObservation, type ProtocolProcessResult } from "./process-protocol.js";
|
|
5
|
+
import { type AdapterCoverageReport, type NormalizedDataset } from "./normalization-integrity.js";
|
|
6
|
+
import { type HarnessAdapter, type NormalizationResult, type QualifiedNativeCapture } from "./uniform-events.js";
|
|
7
|
+
export declare const CODEX_APP_SERVER_VERSION = "0.153.4";
|
|
8
|
+
export declare const CODEX_ADAPTER_VERSION = "0.1.0";
|
|
9
|
+
export declare const CODEX_HARNESS = "codex-app-server";
|
|
10
|
+
export declare const CODEX_DEFAULT_SHUTDOWN_GRACE_MS = 2000;
|
|
11
|
+
export type CodexReasoningEffort = "none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max" | "ultra";
|
|
12
|
+
export type CodexApprovalPolicy = Extract<AskForApproval, string>;
|
|
13
|
+
export type CodexSandbox = SandboxMode;
|
|
14
|
+
export type CodexTelemetrySignal = "logs" | "traces" | "metrics";
|
|
15
|
+
export type CodexAppServerConfiguration = {
|
|
16
|
+
executable: string;
|
|
17
|
+
version: typeof CODEX_APP_SERVER_VERSION;
|
|
18
|
+
provider: "openai";
|
|
19
|
+
model: string;
|
|
20
|
+
effort: CodexReasoningEffort;
|
|
21
|
+
approvalPolicy: CodexApprovalPolicy;
|
|
22
|
+
sandbox: CodexSandbox;
|
|
23
|
+
/** Test-only executable prefix; production uses the pinned executable directly. */
|
|
24
|
+
executableArgs?: readonly string[];
|
|
25
|
+
telemetry?: {
|
|
26
|
+
signals: readonly CodexTelemetrySignal[];
|
|
27
|
+
};
|
|
28
|
+
};
|
|
29
|
+
export type CodexOtlpRecord = {
|
|
30
|
+
signal: CodexTelemetrySignal;
|
|
31
|
+
receivedAt: string;
|
|
32
|
+
contentType?: string;
|
|
33
|
+
payload?: unknown;
|
|
34
|
+
raw?: string;
|
|
35
|
+
parseError?: string;
|
|
36
|
+
};
|
|
37
|
+
export type CodexTelemetryEvidence = {
|
|
38
|
+
schemaVersion: "ebo.codex-telemetry/v1";
|
|
39
|
+
attemptId: string;
|
|
40
|
+
runtime: {
|
|
41
|
+
executable: string;
|
|
42
|
+
version: string;
|
|
43
|
+
adapterVersion: string;
|
|
44
|
+
userConfiguration: "isolated";
|
|
45
|
+
localLoginReference: "available" | "unavailable";
|
|
46
|
+
};
|
|
47
|
+
effectiveConfiguration: {
|
|
48
|
+
model: string;
|
|
49
|
+
provider: "openai";
|
|
50
|
+
effort: CodexReasoningEffort;
|
|
51
|
+
approvalPolicy: CodexApprovalPolicy;
|
|
52
|
+
sandbox: CodexSandbox;
|
|
53
|
+
workspace: string;
|
|
54
|
+
environmentKeys: readonly string[];
|
|
55
|
+
threadStart?: Record<string, unknown>;
|
|
56
|
+
};
|
|
57
|
+
telemetry: {
|
|
58
|
+
receipt: {
|
|
59
|
+
status: "received" | "missing" | "not-checked";
|
|
60
|
+
signals: Record<CodexTelemetrySignal, {
|
|
61
|
+
status: "received" | "missing" | "disabled";
|
|
62
|
+
count: number;
|
|
63
|
+
}>;
|
|
64
|
+
};
|
|
65
|
+
records: readonly CodexOtlpRecord[];
|
|
66
|
+
receiverErrors: readonly string[];
|
|
67
|
+
};
|
|
68
|
+
usage: {
|
|
69
|
+
updates: ReadonlyArray<{
|
|
70
|
+
sequence: number;
|
|
71
|
+
threadId: string;
|
|
72
|
+
turnId: string;
|
|
73
|
+
total: Partial<TokenUsageBreakdown>;
|
|
74
|
+
last: Partial<TokenUsageBreakdown>;
|
|
75
|
+
modelContextWindow?: number;
|
|
76
|
+
}>;
|
|
77
|
+
final?: {
|
|
78
|
+
total: Partial<TokenUsageBreakdown>;
|
|
79
|
+
last: Partial<TokenUsageBreakdown>;
|
|
80
|
+
modelContextWindow?: number;
|
|
81
|
+
};
|
|
82
|
+
};
|
|
83
|
+
};
|
|
84
|
+
export type CodexCaptureGap = {
|
|
85
|
+
kind: string;
|
|
86
|
+
detail: string;
|
|
87
|
+
};
|
|
88
|
+
export type CodexAppServerCaptureRequest = {
|
|
89
|
+
runId: string;
|
|
90
|
+
attemptId: string;
|
|
91
|
+
workspacePath: string;
|
|
92
|
+
prompt: string;
|
|
93
|
+
configuration: CodexAppServerConfiguration;
|
|
94
|
+
evidencePath: string;
|
|
95
|
+
stderrPath?: string;
|
|
96
|
+
signal?: AbortSignal;
|
|
97
|
+
shutdownGraceMs?: number;
|
|
98
|
+
maxLineBytes?: number;
|
|
99
|
+
maxInMemoryObservations?: number;
|
|
100
|
+
now?: () => string;
|
|
101
|
+
registerShutdown?: (shutdown: () => Promise<void>) => void;
|
|
102
|
+
};
|
|
103
|
+
export type CodexAppServerCapture = QualifiedNativeCapture<ProtocolObservation> & {
|
|
104
|
+
threadId?: string;
|
|
105
|
+
turnId?: string;
|
|
106
|
+
terminalStatus?: "completed" | "interrupted" | "failed";
|
|
107
|
+
terminal?: Record<string, unknown>;
|
|
108
|
+
history?: Record<string, unknown>;
|
|
109
|
+
process: ProtocolProcessResult;
|
|
110
|
+
gaps: readonly CodexCaptureGap[];
|
|
111
|
+
telemetry: CodexTelemetryEvidence;
|
|
112
|
+
};
|
|
113
|
+
export declare const CODEX_APP_SERVER_CAPABILITIES: {
|
|
114
|
+
readonly adapterId: "ebo-codex-app-server-v0.153.4";
|
|
115
|
+
readonly schemaVersion: "ebo.adapter-capability-profile/v1";
|
|
116
|
+
readonly harness: "codex-app-server";
|
|
117
|
+
readonly nativeTypes: readonly ["item/completed", "turn/completed", "turn/plan/updated", "thread/compacted", "thread/tokenUsage/updated", "model/rerouted", "hook/completed", "server-request"];
|
|
118
|
+
readonly families: {
|
|
119
|
+
readonly message: {
|
|
120
|
+
readonly status: "partial";
|
|
121
|
+
readonly detail: "Completed user and agent message items are projected.";
|
|
122
|
+
};
|
|
123
|
+
readonly "model-request": {
|
|
124
|
+
readonly status: "unsupported";
|
|
125
|
+
readonly detail: "Inference requests are not inferred from turns or model changes.";
|
|
126
|
+
};
|
|
127
|
+
readonly tool: {
|
|
128
|
+
readonly status: "partial";
|
|
129
|
+
readonly detail: "Only completed native tool items are projected; deltas stay native.";
|
|
130
|
+
};
|
|
131
|
+
readonly context: {
|
|
132
|
+
readonly status: "partial";
|
|
133
|
+
readonly detail: "Plans and compaction are projected when emitted.";
|
|
134
|
+
};
|
|
135
|
+
readonly permission: {
|
|
136
|
+
readonly status: "partial";
|
|
137
|
+
readonly detail: "Server requests and unattended decisions stay native; request events are projected.";
|
|
138
|
+
};
|
|
139
|
+
readonly delegation: {
|
|
140
|
+
readonly status: "partial";
|
|
141
|
+
readonly detail: "Completed exposed collaboration items are projected when present.";
|
|
142
|
+
};
|
|
143
|
+
readonly artifact: {
|
|
144
|
+
readonly status: "partial";
|
|
145
|
+
readonly detail: "Completed file-change items are projected.";
|
|
146
|
+
};
|
|
147
|
+
readonly validation: {
|
|
148
|
+
readonly status: "unsupported";
|
|
149
|
+
readonly detail: "Verifier evidence is packaged outside the Codex native normalizer.";
|
|
150
|
+
};
|
|
151
|
+
readonly runtime: {
|
|
152
|
+
readonly status: "partial";
|
|
153
|
+
readonly detail: "Usage, reroutes, and completed hooks are projected when emitted.";
|
|
154
|
+
};
|
|
155
|
+
readonly outcome: {
|
|
156
|
+
readonly status: "available";
|
|
157
|
+
readonly detail: "Matching turn/completed is authoritative for turn outcome.";
|
|
158
|
+
};
|
|
159
|
+
};
|
|
160
|
+
readonly evidence: {
|
|
161
|
+
readonly nativeOrder: {
|
|
162
|
+
readonly status: "available";
|
|
163
|
+
readonly detail: "The retained receive/write sequence is one stdio ordering domain.";
|
|
164
|
+
};
|
|
165
|
+
readonly nativeTime: {
|
|
166
|
+
readonly status: "partial";
|
|
167
|
+
readonly detail: "Item lifecycle timestamps are native; other records retain observation time separately.";
|
|
168
|
+
};
|
|
169
|
+
readonly parentage: {
|
|
170
|
+
readonly status: "partial";
|
|
171
|
+
readonly detail: "Thread and turn scopes are retained; item relations remain source-specific.";
|
|
172
|
+
};
|
|
173
|
+
readonly content: {
|
|
174
|
+
readonly status: "partial";
|
|
175
|
+
readonly detail: "Mapped content references retained native payloads without copying bodies.";
|
|
176
|
+
};
|
|
177
|
+
};
|
|
178
|
+
};
|
|
179
|
+
export declare function createCodexHarnessAdapter(): HarnessAdapter<CodexAppServerCaptureRequest, ProtocolObservation>;
|
|
180
|
+
/** Drive one owned Codex app-server thread and finish only on its matching turn/completed notification. */
|
|
181
|
+
export declare function captureCodexAppServer(request: CodexAppServerCaptureRequest): Promise<CodexAppServerCapture>;
|
|
182
|
+
export declare function normalizeCodexCapture(capture: QualifiedNativeCapture<ProtocolObservation>): Promise<NormalizationResult>;
|
|
183
|
+
export declare function describeAndValidateCodexDataset(capture: QualifiedNativeCapture<ProtocolObservation>, runtimeVersion?: string): Promise<{
|
|
184
|
+
dataset: NormalizedDataset;
|
|
185
|
+
coverage: AdapterCoverageReport;
|
|
186
|
+
}>;
|
|
187
|
+
export declare function writeProtocolLine(stream: NodeJS.WritableStream, message: unknown): Promise<void>;
|