engineering-behavior-observatory 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +151 -0
  3. package/contracts/codex-app-server-0.150.1/manifest.json +25 -0
  4. package/contracts/codex-app-server-0.150.1/schema/InitializeParams.json +84 -0
  5. package/contracts/codex-app-server-0.150.1/schema/JSONRPCMessage.json +137 -0
  6. package/contracts/codex-app-server-0.150.1/schema/v2/ThreadReadParams.json +17 -0
  7. package/contracts/codex-app-server-0.150.1/schema/v2/ThreadTokenUsageUpdatedNotification.json +82 -0
  8. package/contracts/codex-app-server-0.150.1/schema/v2/TurnInterruptParams.json +17 -0
  9. package/contracts/codex-app-server-0.150.1/types/AskForApproval.ts +5 -0
  10. package/contracts/codex-app-server-0.150.1/types/ClientNotification.ts +5 -0
  11. package/contracts/codex-app-server-0.150.1/types/SandboxMode.ts +5 -0
  12. package/contracts/codex-app-server-0.150.1/types/ThreadReadParams.ts +9 -0
  13. package/contracts/codex-app-server-0.150.1/types/TokenUsageBreakdown.ts +5 -0
  14. package/contracts/codex-app-server-0.150.1/types/TurnInterruptParams.ts +5 -0
  15. package/contracts/codex-app-server-0.153.4/manifest.json +27 -0
  16. package/contracts/codex-app-server-0.153.4/schema/JSONRPCMessage.json +137 -0
  17. package/contracts/codex-app-server-0.153.4/schema/v1/InitializeParams.json +84 -0
  18. package/contracts/codex-app-server-0.153.4/schema/v2/ThreadReadParams.json +17 -0
  19. package/contracts/codex-app-server-0.153.4/schema/v2/ThreadStartParams.json +515 -0
  20. package/contracts/codex-app-server-0.153.4/schema/v2/ThreadTokenUsageUpdatedNotification.json +82 -0
  21. package/contracts/codex-app-server-0.153.4/schema/v2/TurnInterruptParams.json +17 -0
  22. package/contracts/codex-app-server-0.153.4/schema/v2/TurnStartParams.json +921 -0
  23. package/contracts/codex-app-server-0.153.4/types/AskForApproval.ts +5 -0
  24. package/contracts/codex-app-server-0.153.4/types/ClientNotification.ts +5 -0
  25. package/contracts/codex-app-server-0.153.4/types/SandboxMode.ts +5 -0
  26. package/contracts/codex-app-server-0.153.4/types/ThreadReadParams.ts +12 -0
  27. package/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.ts +5 -0
  28. package/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.ts +5 -0
  29. package/contracts/openhands-agent-server-v1.44.1.json +65 -0
  30. package/contracts/openhands-agent-server-v1.46.0.json +46 -0
  31. package/dist/contracts/codex-app-server-0.153.4/types/AskForApproval.d.ts +9 -0
  32. package/dist/contracts/codex-app-server-0.153.4/types/AskForApproval.js +2 -0
  33. package/dist/contracts/codex-app-server-0.153.4/types/ClientNotification.d.ts +3 -0
  34. package/dist/contracts/codex-app-server-0.153.4/types/ClientNotification.js +2 -0
  35. package/dist/contracts/codex-app-server-0.153.4/types/SandboxMode.d.ts +1 -0
  36. package/dist/contracts/codex-app-server-0.153.4/types/SandboxMode.js +2 -0
  37. package/dist/contracts/codex-app-server-0.153.4/types/ThreadReadParams.d.ts +10 -0
  38. package/dist/contracts/codex-app-server-0.153.4/types/ThreadReadParams.js +2 -0
  39. package/dist/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.d.ts +8 -0
  40. package/dist/contracts/codex-app-server-0.153.4/types/TokenUsageBreakdown.js +2 -0
  41. package/dist/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.d.ts +4 -0
  42. package/dist/contracts/codex-app-server-0.153.4/types/TurnInterruptParams.js +2 -0
  43. package/dist/src/agent-sdk-normalizer.d.ts +22 -0
  44. package/dist/src/agent-sdk-normalizer.js +995 -0
  45. package/dist/src/agent-sdk-run.d.ts +31 -0
  46. package/dist/src/agent-sdk-run.js +242 -0
  47. package/dist/src/agent-sdk-runner.d.ts +96 -0
  48. package/dist/src/agent-sdk-runner.js +407 -0
  49. package/dist/src/agent-sdk.d.ts +294 -0
  50. package/dist/src/agent-sdk.js +644 -0
  51. package/dist/src/aggregation.d.ts +183 -0
  52. package/dist/src/aggregation.js +513 -0
  53. package/dist/src/artifacts.d.ts +36 -0
  54. package/dist/src/artifacts.js +1372 -0
  55. package/dist/src/atlas-grafana.d.ts +377 -0
  56. package/dist/src/atlas-grafana.js +47 -0
  57. package/dist/src/atlas-html.d.ts +2 -0
  58. package/dist/src/atlas-html.js +34 -0
  59. package/dist/src/atlas.d.ts +130 -0
  60. package/dist/src/atlas.js +235 -0
  61. package/dist/src/behavior-assertions.d.ts +95 -0
  62. package/dist/src/behavior-assertions.js +154 -0
  63. package/dist/src/cli.d.ts +5 -0
  64. package/dist/src/cli.js +957 -0
  65. package/dist/src/codex-judge.d.ts +44 -0
  66. package/dist/src/codex-judge.js +214 -0
  67. package/dist/src/codex-run.d.ts +114 -0
  68. package/dist/src/codex-run.js +519 -0
  69. package/dist/src/codex.d.ts +187 -0
  70. package/dist/src/codex.js +1028 -0
  71. package/dist/src/contracts.d.ts +155 -0
  72. package/dist/src/contracts.js +866 -0
  73. package/dist/src/corpus.d.ts +67 -0
  74. package/dist/src/corpus.js +540 -0
  75. package/dist/src/cursor-sdk-runner.d.ts +81 -0
  76. package/dist/src/cursor-sdk-runner.js +362 -0
  77. package/dist/src/cursor-sdk.d.ts +81 -0
  78. package/dist/src/cursor-sdk.js +1078 -0
  79. package/dist/src/deepseek-adapter.d.ts +152 -0
  80. package/dist/src/deepseek-adapter.js +777 -0
  81. package/dist/src/exports.d.ts +69 -0
  82. package/dist/src/exports.js +966 -0
  83. package/dist/src/human-calibration.d.ts +167 -0
  84. package/dist/src/human-calibration.js +618 -0
  85. package/dist/src/index.d.ts +71 -0
  86. package/dist/src/index.js +38 -0
  87. package/dist/src/lifecycle.d.ts +196 -0
  88. package/dist/src/lifecycle.js +1889 -0
  89. package/dist/src/normalization-integrity.d.ts +128 -0
  90. package/dist/src/normalization-integrity.js +429 -0
  91. package/dist/src/openhands-run.d.ts +36 -0
  92. package/dist/src/openhands-run.js +415 -0
  93. package/dist/src/openhands.d.ts +130 -0
  94. package/dist/src/openhands.js +859 -0
  95. package/dist/src/pi.d.ts +203 -0
  96. package/dist/src/pi.js +1345 -0
  97. package/dist/src/process-protocol.d.ts +245 -0
  98. package/dist/src/process-protocol.js +1261 -0
  99. package/dist/src/retained-evidence.d.ts +12 -0
  100. package/dist/src/retained-evidence.js +201 -0
  101. package/dist/src/run-bundles.d.ts +193 -0
  102. package/dist/src/run-bundles.js +993 -0
  103. package/dist/src/scheduler.d.ts +109 -0
  104. package/dist/src/scheduler.js +1080 -0
  105. package/dist/src/semantic-judge.d.ts +194 -0
  106. package/dist/src/semantic-judge.js +875 -0
  107. package/dist/src/structural-observations.d.ts +124 -0
  108. package/dist/src/structural-observations.js +671 -0
  109. package/dist/src/task-packets.d.ts +115 -0
  110. package/dist/src/task-packets.js +683 -0
  111. package/dist/src/uniform-events.d.ts +120 -0
  112. package/dist/src/uniform-events.js +158 -0
  113. package/dist/src/verifiers.d.ts +116 -0
  114. package/dist/src/verifiers.js +819 -0
  115. package/dist/src/workspaces.d.ts +56 -0
  116. package/dist/src/workspaces.js +1314 -0
  117. package/docs/README.md +60 -0
  118. package/docs/development/README.md +71 -0
  119. package/docs/development/documentation-sync.md +58 -0
  120. package/docs/development/extension-contracts.md +307 -0
  121. package/docs/evaluation/README.md +48 -0
  122. package/docs/evaluation/aggregation.md +112 -0
  123. package/docs/evaluation/behavior-assertions.md +45 -0
  124. package/docs/evaluation/human-calibration.md +170 -0
  125. package/docs/evaluation/normalization-integrity.md +80 -0
  126. package/docs/evaluation/semantic-judge.md +188 -0
  127. package/docs/evaluation/structural-observations.md +84 -0
  128. package/docs/evaluation/uniform-events.md +131 -0
  129. package/docs/guides/atlas.md +144 -0
  130. package/docs/guides/evidence-and-sharing.md +100 -0
  131. package/docs/guides/operator-guide.md +424 -0
  132. package/docs/guides/quickstart.md +152 -0
  133. package/docs/guides/telemetry.md +66 -0
  134. package/docs/harnesses/README.md +31 -0
  135. package/docs/harnesses/claude-agent-sdk.md +61 -0
  136. package/docs/harnesses/codex-harness.md +174 -0
  137. package/docs/harnesses/cursor-sdk.md +149 -0
  138. package/docs/harnesses/deepseek-harness.md +134 -0
  139. package/docs/harnesses/openhands-agent-server.md +153 -0
  140. package/docs/harnesses/pi-sdk.md +66 -0
  141. package/docs/reference/README.md +19 -0
  142. package/docs/reference/agent-sdk-operational-runner.md +118 -0
  143. package/docs/reference/cli.md +114 -0
  144. package/docs/reference/contracts.md +222 -0
  145. package/docs/reference/run-bundle-contract.md +354 -0
  146. package/docs/reference/run-lifecycle.md +54 -0
  147. package/examples/cursor-sdk/README.md +11 -0
  148. package/examples/cursor-sdk/capture-profile.json +1 -0
  149. package/examples/cursor-sdk/harness.json +1 -0
  150. package/examples/cursor-sdk/model.json +1 -0
  151. package/examples/cursor-sdk/native-limits.json +1 -0
  152. package/examples/cursor-sdk/native-tool-policy.json +1 -0
  153. package/ontology/behavior-categories.v1.json +46 -0
  154. package/package.json +66 -0
  155. package/release/0.1.0/KNOWN_LIMITATIONS.md +35 -0
  156. package/release/0.1.0/README.md +46 -0
  157. package/release/0.1.0/reproducibility.json +61 -0
  158. package/release/0.2.0/KNOWN_LIMITATIONS.md +40 -0
  159. package/release/0.2.0/README.md +42 -0
  160. package/release/0.2.0/reproducibility.json +73 -0
  161. package/release/0.2.1/KNOWN_LIMITATIONS.md +18 -0
  162. package/release/0.2.1/README.md +63 -0
  163. package/release/0.2.1/reproducibility.json +73 -0
  164. package/release/README.md +13 -0
  165. package/schemas/aggregation.v1.json +293 -0
  166. package/schemas/behavior-assertions.v1.json +220 -0
  167. package/schemas/deepseek-runtime-composition.v1.schema.json +122 -0
  168. package/schemas/experiment.v1.schema.json +188 -0
  169. package/schemas/human-calibration.v1.json +336 -0
  170. package/schemas/normalization-integrity.v1.json +302 -0
  171. package/schemas/run-bundles/v1.json +1040 -0
  172. package/schemas/run-queue.v1.schema.json +206 -0
  173. package/schemas/semantic-judge.v1.json +380 -0
  174. package/schemas/structural-observations.v1.json +131 -0
  175. package/schemas/task-packet-freeze.v1.schema.json +106 -0
  176. package/schemas/task-packet.v1.schema.json +234 -0
  177. package/schemas/uniform-events/v1.json +289 -0
  178. package/scripts/atlas-grafana.sh +18 -0
@@ -0,0 +1,203 @@
1
+ import { type AgentSessionEvent, type InlineExtension } from "@earendil-works/pi-coding-agent";
2
+ import { type ArtifactReference } from "./contracts.js";
3
+ import { type RunAttemptResult, type TerminalRecord, type VerifierExecutionContext, type WorkspaceCoordinator } from "./lifecycle.js";
4
+ import { type AdapterCoverageReport, type NormalizedDataset } from "./normalization-integrity.js";
5
+ import { JsonlEvidenceWriter } from "./process-protocol.js";
6
+ import { type CaptureQualificationReport, type CaptureQualificationStatus, type CapturedWorkspaceOutcome, type RunBundleDefinition, type RunManifest } from "./run-bundles.js";
7
+ import { type TaskPacket } from "./task-packets.js";
8
+ import { type VerifierResult } from "./verifiers.js";
9
+ import { type AdapterCapabilityProfile, type NativeEvidenceResolver, type NormalizationInput, type NormalizationResult } from "./uniform-events.js";
10
+ export declare const PI_HARNESS = "pi-sdk";
11
+ export declare const PI_CONFIG_SCHEMA_VERSION = "ebo.pi-config/v1";
12
+ export declare const PI_ADAPTER_VERSION = "1.0.0";
13
+ export declare const PINNED_PI_SDK_VERSION = "0.85.1";
14
+ declare const PI_APIS: readonly ["anthropic-messages", "openai-completions", "openai-responses", "google-generative-ai"];
15
+ declare const PI_THINKING_LEVELS: readonly ["off", "minimal", "low", "medium", "high", "xhigh", "max"];
16
+ declare const PI_TOOLS: readonly ["read", "bash", "edit", "write", "grep", "find", "ls"];
17
+ type PiCost = {
18
+ input: number;
19
+ output: number;
20
+ cacheRead: number;
21
+ cacheWrite: number;
22
+ };
23
+ type PiThinkingLevel = typeof PI_THINKING_LEVELS[number];
24
+ export type PiModelConfiguration = {
25
+ schemaVersion: typeof PI_CONFIG_SCHEMA_VERSION;
26
+ kind: "model";
27
+ provider: string;
28
+ queueModelId: string;
29
+ model: string;
30
+ api: typeof PI_APIS[number];
31
+ baseUrl: string;
32
+ apiKeyEnv: string;
33
+ thinkingLevel: typeof PI_THINKING_LEVELS[number];
34
+ thinkingLevelMap?: Partial<Record<PiThinkingLevel, string | null>>;
35
+ reasoning: boolean;
36
+ input: Array<"text" | "image">;
37
+ contextWindow: number;
38
+ maxTokens: number;
39
+ cost: PiCost;
40
+ };
41
+ export type PiHarnessConfiguration = {
42
+ schemaVersion: typeof PI_CONFIG_SCHEMA_VERSION;
43
+ kind: "harness";
44
+ adapter: typeof PI_HARNESS;
45
+ version: typeof PINNED_PI_SDK_VERSION;
46
+ extensions?: ArtifactReference[];
47
+ skills?: ArtifactReference[];
48
+ prompts?: ArtifactReference[];
49
+ contextFiles?: ArtifactReference[];
50
+ systemPrompt?: ArtifactReference;
51
+ };
52
+ export type PiNativeLimitsConfiguration = {
53
+ schemaVersion: typeof PI_CONFIG_SCHEMA_VERSION;
54
+ kind: "native-limits";
55
+ shutdownGraceMs?: number;
56
+ maxRetries?: number;
57
+ retryBaseDelayMs?: number;
58
+ providerTimeoutMs?: number;
59
+ };
60
+ export type PiNativeToolPolicyConfiguration = {
61
+ schemaVersion: typeof PI_CONFIG_SCHEMA_VERSION;
62
+ kind: "native-tool-policy";
63
+ tools: typeof PI_TOOLS[number][];
64
+ excludeTools?: typeof PI_TOOLS[number][];
65
+ environmentAllowlist: string[];
66
+ };
67
+ export type PiCaptureProfileConfiguration = {
68
+ schemaVersion: typeof PI_CONFIG_SCHEMA_VERSION;
69
+ kind: "capture-profile";
70
+ passiveObserver?: boolean;
71
+ includeProviderPayloads?: boolean;
72
+ workspaceOutcome?: {
73
+ excludeDirectoryNames: string[];
74
+ respectGitignore?: boolean;
75
+ omitEmptyDirectories?: boolean;
76
+ };
77
+ };
78
+ export type PiConfigurationRecord = PiModelConfiguration | PiHarnessConfiguration | PiNativeLimitsConfiguration | PiNativeToolPolicyConfiguration | PiCaptureProfileConfiguration;
79
+ export type PiConfigurationKind = PiConfigurationRecord["kind"];
80
+ export type PiNativeRecord = Record<string, unknown>;
81
+ export declare const piCapabilityProfile: AdapterCapabilityProfile;
82
+ type PiRecordedEnvelope = {
83
+ schemaVersion: "ebo.pi-native-record/v1";
84
+ sequence: number;
85
+ receivedAt: string;
86
+ channel: "stream" | "observer" | "adapter";
87
+ nativeType: string;
88
+ sessionId?: string;
89
+ hook?: string;
90
+ stage?: string;
91
+ extensionOrder?: number;
92
+ payload: unknown;
93
+ };
94
+ declare class PiEvidenceRecorder {
95
+ readonly writer: JsonlEvidenceWriter;
96
+ private sequence;
97
+ private queue;
98
+ private error;
99
+ constructor(path: string);
100
+ record(input: Omit<PiRecordedEnvelope, "schemaVersion" | "sequence" | "receivedAt">): Promise<void>;
101
+ flush(): Promise<void>;
102
+ close(): Promise<void>;
103
+ }
104
+ export type PiSession = {
105
+ readonly sessionId: string;
106
+ readonly sessionFile?: string;
107
+ readonly messages: unknown[];
108
+ readonly extensionRunner?: {
109
+ emit(event: unknown): Promise<unknown>;
110
+ };
111
+ subscribe(listener: (event: AgentSessionEvent) => void): () => void;
112
+ prompt(text: string, options?: {
113
+ expandPromptTemplates?: boolean;
114
+ source?: "extension";
115
+ }): Promise<void>;
116
+ waitForIdle(): Promise<void>;
117
+ abort(): Promise<void>;
118
+ exportToJsonl(outputPath?: string): string;
119
+ getActiveToolNames(): string[];
120
+ dispose(): void;
121
+ };
122
+ export type PiSessionFactoryInput = {
123
+ workspacePath: string;
124
+ sessionDirectory: string;
125
+ model: PiModelConfiguration;
126
+ harness: PiHarnessConfiguration;
127
+ limits: PiNativeLimitsConfiguration;
128
+ toolPolicy: PiNativeToolPolicyConfiguration;
129
+ captureProfile: PiCaptureProfileConfiguration;
130
+ bundleRoot: string;
131
+ observer: PiEvidenceRecorder;
132
+ signal: AbortSignal;
133
+ };
134
+ export type PiSessionFactory = (input: PiSessionFactoryInput) => Promise<PiSession>;
135
+ export type CapturePiSdkRunOptions = {
136
+ definition: RunBundleDefinition;
137
+ configurationRoot: string;
138
+ startingWorkspacePath: string;
139
+ workspace: WorkspaceCoordinator;
140
+ model: PiModelConfiguration;
141
+ harness: PiHarnessConfiguration;
142
+ limits: PiNativeLimitsConfiguration;
143
+ toolPolicy: PiNativeToolPolicyConfiguration;
144
+ captureProfile: PiCaptureProfileConfiguration;
145
+ prompt: string;
146
+ verifier?: (context: VerifierExecutionContext, workspace: CapturedWorkspaceOutcome, workspacePath: string) => VerifierResult | Promise<VerifierResult>;
147
+ workspaceOutcomeExcludedDirectoryNames?: readonly string[];
148
+ workspaceOutcomeRespectsGitignore?: boolean;
149
+ workspaceOutcomeOmitsEmptyDirectories?: boolean;
150
+ signal?: AbortSignal;
151
+ maxWallClockMs?: number;
152
+ createSession?: PiSessionFactory;
153
+ };
154
+ export type CapturePiSdkRunResult = {
155
+ attempt: RunAttemptResult;
156
+ manifest: RunManifest;
157
+ qualification: CaptureQualificationReport;
158
+ };
159
+ /** Execute one direct Pi TypeScript SDK session and retain native history, transient events, and passive observations. */
160
+ export declare function capturePiSdkRun(options: CapturePiSdkRunOptions): Promise<CapturePiSdkRunResult>;
161
+ export declare function filterPiToolEnvironment(environment: NodeJS.ProcessEnv, allowlist: ReadonlySet<string>, apiKeyEnv: string): NodeJS.ProcessEnv;
162
+ export declare function createPiPassiveObserver(recorder: PiEvidenceRecorder, extensionOrder: number, includeProviderPayloads: boolean): InlineExtension;
163
+ export declare function normalizePiCapture(input: NormalizationInput<PiNativeRecord>): Promise<NormalizationResult>;
164
+ export declare function describeAndValidatePiDataset(capture: NormalizationInput<PiNativeRecord>): Promise<{
165
+ dataset: NormalizedDataset;
166
+ coverage: AdapterCoverageReport;
167
+ resolver: NativeEvidenceResolver;
168
+ }>;
169
+ export declare function piNativeType(record: PiNativeRecord): string;
170
+ export declare function assertPiNativeRecord(record: unknown, line: number): asserts record is PiNativeRecord;
171
+ export declare function qualifyRetainedPiCapture(capture: NormalizationInput<PiNativeRecord>, sessionId: string | undefined, expectsCompletion: boolean): NormalizationInput<PiNativeRecord>;
172
+ export type RunPiQueueEntryOptions = {
173
+ bundleRoot: string;
174
+ queuePath: string;
175
+ runId: string;
176
+ outputRoot: string;
177
+ workspaceRoot?: string;
178
+ attemptId?: string;
179
+ signal?: AbortSignal;
180
+ createSession?: PiSessionFactory;
181
+ };
182
+ export type PiRunSummary = {
183
+ runId: string;
184
+ attemptId: string;
185
+ bundlePath: string;
186
+ terminal: TerminalRecord;
187
+ classification: RunAttemptResult["classification"]["kind"];
188
+ captureQualification: CaptureQualificationStatus;
189
+ assessmentMode: TaskPacket["assessmentMode"];
190
+ sessionId?: string;
191
+ retainedWorkspacePath?: string;
192
+ };
193
+ /** Execute exactly one frozen queue entry through the pinned direct Pi SDK integration. */
194
+ export declare function runPiQueueEntry(options: RunPiQueueEntryOptions): Promise<PiRunSummary>;
195
+ type PiConfigurationByKind = {
196
+ model: PiModelConfiguration;
197
+ harness: PiHarnessConfiguration;
198
+ "native-limits": PiNativeLimitsConfiguration;
199
+ "native-tool-policy": PiNativeToolPolicyConfiguration;
200
+ "capture-profile": PiCaptureProfileConfiguration;
201
+ };
202
+ export declare function resolvePiConfigurationRecord<Kind extends PiConfigurationKind>(bundleRoot: string, reference: ArtifactReference, kind: Kind): PiConfigurationByKind[Kind];
203
+ export {};