engineering-behavior-observatory 0.2.1

package/dist/src/retained-evidence.d.ts

@@ -0,0 +1,12 @@
+import { type AgentSdkNativeRecord } from "./agent-sdk-normalizer.js";
+import { type AdapterCoverageReport, type NormalizedDataset } from "./normalization-integrity.js";
+import type { NormalizationInput, NativeEvidenceResolver } from "./uniform-events.js";
+export type RetainedBehaviorEvidence = {
+    capture: NormalizationInput<unknown>;
+    /** Verified bundle metadata for outcome import; never replaces native session records. */
+    outcomeCapture: NormalizationInput<AgentSdkNativeRecord>;
+    dataset: NormalizedDataset;
+    resolver: NativeEvidenceResolver;
+    coverage: AdapterCoverageReport;
+};
+export declare function createRetainedBehaviorEvidence(bundleRoot: string): Promise<RetainedBehaviorEvidence>;

package/dist/src/retained-evidence.js

@@ -0,0 +1,201 @@
+import { join } from "node:path";
+import { CLAUDE_AGENT_SDK_HARNESS, readQualifiedRunCapture, createAgentSdkNativeEvidenceResolver } from "./agent-sdk-normalizer.js";
+import { createAgentSdkBehaviorEvidence } from "./behavior-assertions.js";
+import { describeAndValidateCodexDataset, CODEX_HARNESS } from "./codex.js";
+import { normalizeOpenHandsCapture, openHandsCapabilityProfile } from "./openhands.js";
+import { createDeepSeekHarnessAdapter, DEEPSEEK_HARNESS_ID, DEEPSEEK_SDK_VERSION, normalizeDeepSeekCapture, qualifyRetainedDeepSeekCapture } from "./deepseek-adapter.js";
+import { createCursorSdkBehaviorEvidence, CURSOR_SDK_HARNESS } from "./cursor-sdk.js";
+import { createCapturedNativeEvidenceResolver, describeNormalizedDataset, validateNormalizedDataset } from "./normalization-integrity.js";
+import { readBoundedFile } from "./scheduler.js";
+import { assertProtocolObservation } from "./process-protocol.js";
+import { assertPiNativeRecord, normalizePiCapture, piCapabilityProfile, piNativeType, qualifyRetainedPiCapture, PI_ADAPTER_VERSION, PI_HARNESS, PINNED_PI_SDK_VERSION, } from "./pi.js";
+export async function createRetainedBehaviorEvidence(bundleRoot) {
+    const manifest = JSON.parse(readBoundedFile(join(bundleRoot, "manifest.json"), "Run manifest").toString("utf8"));
+    const harness = manifest.run.harness.id;
+    if (harness === CURSOR_SDK_HARNESS)
+        return createCursorSdkBehaviorEvidence(bundleRoot);
+    const isAgentSdk = [CLAUDE_AGENT_SDK_HARNESS, "agent-sdk"].includes(harness)
+        || manifest.run.runtime.some(({ source, name }) => source === "anthropic" && [CLAUDE_AGENT_SDK_HARNESS, "agent-sdk"].includes(name));
+    if (isAgentSdk) {
+        const evidence = await createAgentSdkBehaviorEvidence(bundleRoot);
+        return { ...evidence, outcomeCapture: evidence.capture };
+    }
+    if (![CODEX_HARNESS, "openhands-agent-server", DEEPSEEK_HARNESS_ID, PI_HARNESS].includes(harness)) {
+        throw new Error(`Unsupported retained harness ${harness}; refusing Agent SDK fallback normalization.`);
+    }
+    if (harness === "openhands-agent-server")
+        openHandsCapabilityProfile(manifest.run.harness.version);
+    if (harness === DEEPSEEK_HARNESS_ID && manifest.run.harness.version !== DEEPSEEK_SDK_VERSION) {
+        throw new Error(`Unsupported retained DeepSeek runtime ${manifest.run.harness.version}.`);
+    }
+    if (harness === PI_HARNESS && manifest.run.harness.version !== PINNED_PI_SDK_VERSION) {
+        throw new Error(`Unsupported retained Pi runtime ${manifest.run.harness.version}.`);
+    }
+    if (harness === PI_HARNESS) {
+        const expected = new Map([
+            ["pi-coding-agent", { source: "earendil-works", version: PINNED_PI_SDK_VERSION }],
+            [PI_HARNESS, { source: "EBO", version: PINNED_PI_SDK_VERSION }],
+            ["pi-sdk-adapter", { source: "EBO", version: PI_ADAPTER_VERSION }],
+        ]);
+        for (const [name, identity] of expected) {
+            const matches = manifest.run.runtime.filter((runtime) => runtime.name === name);
+            if (matches.length !== 1 || matches[0].source !== identity.source || matches[0].version !== identity.version) {
+                throw new Error(`Retained Pi runtime identity ${name} differs from the pinned adapter manifest.`);
+            }
+        }
+    }
+    const outcomeCapture = await readQualifiedRunCapture(bundleRoot);
+    // A verifier task failure also presupposes a normally completed native run.
+    const expectsCompletion = manifest.terminal.state === "completed" || manifest.terminal.failureClass === "task";
+    const capture = {
+        ...outcomeCapture,
+        records: outcomeCapture.records.filter(({ record }) => record.kind === "session")
+            .map(({ reference, record }) => {
+            const line = Number(reference.recordLocator.match(/^line:(\d+)$/u)?.[1]);
+            if (harness === PI_HARNESS)
+                assertPiNativeRecord(record.document, line);
+            else
+                assertNativeEnvelope(harness, record.document, line);
+            return { reference, record: record.document };
+        }),
+    };
+    const resolver = createCapturedNativeEvidenceResolver(capture, createAgentSdkNativeEvidenceResolver({ ...outcomeCapture,
+        records: outcomeCapture.records.filter(({ record }) => record.kind !== "session" && record.kind !== "hook"),
+    }));
+    let dataset;
+    if (harness === PI_HARNESS) {
+        const native = qualifyRetainedPiCapture(capture, manifest.run.native?.sessionId, expectsCompletion);
+        capture.qualification = native.qualification;
+        outcomeCapture.qualification = native.qualification;
+        dataset = describeNormalizedDataset({
+            capture: native,
+            normalization: await normalizePiCapture(native),
+            capabilityProfile: piCapabilityProfile,
+            adapterVersion: PI_ADAPTER_VERSION,
+            nativeType: piNativeType,
+        });
+    }
+    else if (harness === CODEX_HARNESS) {
+        const native = capture;
+        const handshakeVersions = native.records.filter(({ record }) => record.kind === "response" && record.source === CODEX_HARNESS && record.method === "initialize")
+            .map(({ record }) => String(record.payload?.userAgent ?? "").match(/^[^\s/]+\/([^\s]+)/u)?.[1]);
+        const telemetryVersions = outcomeCapture.records.flatMap(({ record }) => {
+            const document = record.document;
+            return document?.schemaVersion === "ebo.codex-telemetry/v1" ? [document.runtime?.version] : [];
+        });
+        if (handshakeVersions.length === 0 || [...handshakeVersions, ...telemetryVersions,
+            ...manifest.run.runtime.filter(({ name }) => name === CODEX_HARNESS).map(({ version }) => version),
+        ].some((version) => version !== manifest.run.harness.version)) {
+            throw new Error("Retained Codex native runtime version differs from the run manifest.");
+        }
+        let acceptedStartSequence = 0;
+        const identities = (method, key) => {
+            const accepted = native.records.filter(({ record }) => {
+                const payload = record.payload;
+                return record.kind === "response" && record.source === CODEX_HARNESS && record.method === method
+                    && typeof payload?.[key]?.id === "string" && payload[key].id.trim() !== "";
+            });
+            const optionalPartialTurn = key === "turn" && !expectsCompletion;
+            if (accepted.length === 0 && optionalPartialTurn)
+                return undefined;
+            if (accepted.length !== 1)
+                throw new Error(`Retained Codex ${method} requires one owned identity.`);
+            const response = accepted[0].record;
+            const requests = native.records.filter(({ record }) => record.kind === "request" && record.source === "ebo-codex-client" && record.id === response.id);
+            const responses = native.records.filter(({ record }) => record.kind === "response" && record.source === CODEX_HARNESS && record.id === response.id);
+            const request = requests[0]?.record;
+            const payload = response.payload;
+            if (response.id === undefined || response.id === null || requests.length !== 1 || responses.length !== 1
+                || request?.method !== method || request.sequence <= acceptedStartSequence || request.sequence >= response.sequence
+                || key === "turn" && (request.payload?.threadId !== native.threadId
+                    || payload.turn.threadId !== undefined && payload.turn.threadId !== native.threadId)) {
+                throw new Error(`Retained Codex ${method} requires a unique ordered owned request/response pair.`);
+            }
+            acceptedStartSequence = response.sequence;
+            return payload[key].id;
+        };
+        native.threadId = identities("thread/start", "thread");
+        native.turnId = identities("turn/start", "turn");
+        if (native.threadId !== manifest.run.native?.sessionId)
+            throw new Error("Retained Codex thread identity differs from the run manifest.");
+        const terminals = native.records.filter(({ record }) => {
+            const payload = record.payload;
+            return record.kind === "notification" && record.source === CODEX_HARNESS && record.method === "turn/completed"
+                && payload?.threadId === native.threadId && payload?.turn?.id === native.turnId;
+        });
+        if (terminals.length > 1 || terminals.some(({ record }) => record.sequence <= acceptedStartSequence)
+            || expectsCompletion && terminals[0]?.record.payload?.turn?.status !== "completed") {
+            throw new Error("Retained Codex capture requires unambiguous matching owned terminal evidence.");
+        }
+        dataset = (await describeAndValidateCodexDataset(native, manifest.run.harness.version)).dataset;
+    }
+    else if (harness === "openhands-agent-server") {
+        const native = capture;
+        const sessionId = manifest.run.native?.sessionId;
+        const serverInfo = native.records.filter(({ record }) => record.channel === "server-info");
+        if (serverInfo.length !== 1)
+            throw new Error("Retained OpenHands capture requires exactly one native server-info record.");
+        for (const { record } of native.records) {
+            if (record.channel === "server-info" && record.payload.version !== manifest.run.harness.version) {
+                throw new Error("Retained OpenHands native server version differs from the run manifest.");
+            }
+            if (record.session_id !== undefined && record.session_id !== sessionId
+                || ["conversation-created", "websocket-status", "websocket-event", "rest-event", "conversation-final"].includes(record.channel)
+                    && (sessionId === undefined || record.session_id !== sessionId)
+                || ["conversation-created", "conversation-final"].includes(record.channel) && record.payload.id !== sessionId) {
+                throw new Error("Retained OpenHands conversation identity differs from the run manifest.");
+            }
+        }
+        const finals = native.records.filter(({ record }) => record.channel === "conversation-final");
+        if (expectsCompletion && (finals.length !== 1 || finals[0].record.payload.execution_status !== "finished")) {
+            throw new Error("Completed retained OpenHands capture lacks owned finished conversation evidence.");
+        }
+        dataset = describeNormalizedDataset({ capture: native, normalization: await normalizeOpenHandsCapture(native),
+            capabilityProfile: openHandsCapabilityProfile(manifest.run.harness.version), adapterVersion: manifest.run.harness.version,
+            nativeType: (record) => typeof record.payload.kind === "string" ? record.payload.kind : record.channel });
+    }
+    else {
+        const native = qualifyRetainedDeepSeekCapture(capture, manifest.run.native?.sessionId, expectsCompletion);
+        capture.qualification = native.qualification;
+        outcomeCapture.qualification = native.qualification;
+        dataset = describeNormalizedDataset({ capture: native, normalization: normalizeDeepSeekCapture(native),
+            capabilityProfile: createDeepSeekHarnessAdapter().normalization.capabilityProfile, adapterVersion: DEEPSEEK_SDK_VERSION,
+            nativeType: (record) => record.method ?? record.kind });
+    }
+    const coverage = await validateNormalizedDataset(dataset, resolver);
+    return { capture, outcomeCapture, dataset, resolver, coverage };
+}
+function assertNativeEnvelope(harness, value, line) {
+    const fail = () => { throw new Error(`Invalid retained ${harness} native envelope at line ${line}.`); };
+    if (value === null || typeof value !== "object" || Array.isArray(value))
+        fail();
+    const record = value;
+    if (!Number.isSafeInteger(line) || line < 1 || record.sequence !== line)
+        fail();
+    if (harness === CODEX_HARNESS) {
+        assertProtocolObservation(value, line);
+        if (value.source !== CODEX_HARNESS && value.source !== "ebo-codex-client")
+            fail();
+        if (value.source === "ebo-codex-client" && (!["request", "response", "notification"].includes(value.kind)
+            || value.kind === "notification" && value.method !== "initialized"))
+            fail();
+        return;
+    }
+    const optionalText = (fields) => fields.every((field) => record[field] === undefined
+        || typeof record[field] === "string" && record[field].trim() !== "");
+    if (harness === "openhands-agent-server") {
+        if (record.schemaVersion !== "ebo.openhands-native-record/v1"
+            || !["server-info", "conversation-create-response", "conversation-created", "websocket-status", "websocket-event", "conversation-final", "rest-event", "capture-error", "cleanup"].includes(String(record.channel))
+            || record.payload === null || typeof record.payload !== "object" || Array.isArray(record.payload)
+            || !optionalText(["session_id"]) || record.channelSequence !== undefined
+            && (typeof record.channelSequence !== "number" || !Number.isSafeInteger(record.channelSequence) || record.channelSequence < 0))
+            fail();
+        return;
+    }
+    if (record.schemaVersion !== "ebo.deepseek-native-observation/v1"
+        || !["composition", "capability", "request", "response", "notification", "diagnostic", "error"].includes(String(record.kind))
+        || typeof record.observedAt !== "string" || record.observedAt.trim() === ""
+        || !optionalText(["method", "sessionId", "sourceIdentity"]) || record.stream !== undefined && record.stream !== "stderr"
+        || ["request", "response", "notification"].includes(String(record.kind)) && typeof record.method !== "string")
+        fail();
+}

package/dist/src/run-bundles.d.ts

@@ -0,0 +1,193 @@
+import { type AssessmentMode } from "./contracts.js";
+import type { ClaudeAgentSdkAttemptEvidence, ClaudeAgentSdkCapabilities } from "./agent-sdk.js";
+import type { AttemptIdentity, TerminalRecord } from "./lifecycle.js";
+type DigestString = `sha256:${string}`;
+type SharingClass = "unknown" | "restricted" | "internal" | "partner" | "public";
+type EvidenceKind = "session" | "hook" | "telemetry" | "workspace" | "verifier";
+type EvidenceAuthority = "semantic" | "timing-resource" | "outcome";
+export type RunBundleRuntimeComponent = {
+    source: string;
+    name: string;
+    version: string;
+};
+export type RunBundleRun = {
+    id: string;
+    trial?: {
+        index: number;
+    };
+    assessmentMode: AssessmentMode;
+    task: {
+        id: string;
+        digest?: DigestString;
+    };
+    fixture: {
+        id: string;
+        digest: DigestString;
+    };
+    model: {
+        provider: string;
+        id: string;
+        configurationDigest?: DigestString;
+    };
+    harness: {
+        id: string;
+        version: string;
+        configurationDigest?: DigestString;
+    };
+    runtime: RunBundleRuntimeComponent[];
+    verifier?: {
+        locator: string;
+        digest: DigestString;
+        format?: "commonjs" | "module";
+    };
+    native?: {
+        sessionId?: string;
+        traceId?: string;
+    };
+};
+export type RunBundleConfiguration = {
+    digest: DigestString;
+    captureProfileDigest?: DigestString;
+    budgetDigest: DigestString;
+    toolPolicyDigest: DigestString;
+};
+export type RunBundleEvidenceDescriptor = {
+    id: string;
+    source: string;
+    kind: EvidenceKind | "diagnostic" | "capture-report" | "export-manifest";
+    authority: EvidenceAuthority | "capture" | "export";
+    mediaType: string;
+    digest: DigestString;
+    sizeBytes: number;
+    sharingClass: SharingClass;
+    relativePath: string;
+    fingerprint?: DigestString;
+    nativeReference?: {
+        type: string;
+        id: string;
+    };
+    sanitizedFrom?: {
+        artifactId: string;
+        digest: DigestString;
+    };
+};
+export type RunManifest = {
+    schemaVersion: "run-manifest/v1";
+    bundleId: string;
+    run: RunBundleRun;
+    attempt: AttemptIdentity;
+    configuration: RunBundleConfiguration;
+    terminal: TerminalRecord;
+    evidence: RunBundleEvidenceDescriptor[];
+};
+export type CaptureMissingEvidence = {
+    kind: string;
+    reason: "not-emitted" | "not-collected" | "optional-beta-unavailable" | "process-interrupted" | "policy-restricted" | "unsupported" | "not-checked";
+    affects: Array<"semantic" | "timing-resource" | "outcome">;
+    detail?: string;
+};
+export type RunBundleDefinition = {
+    bundleRoot: string;
+    bundleId: string;
+    run: RunBundleRun;
+    attempt: AttemptIdentity;
+    configuration: RunBundleConfiguration;
+};
+export type RegisterRunBundleArtifact = {
+    id: string;
+    source: string;
+    kind: EvidenceKind;
+    mediaType: string;
+    sharingClass: Exclude<SharingClass, "partner" | "public">;
+    relativePath: string;
+    fingerprint?: DigestString;
+    nativeReference?: {
+        type: string;
+        id: string;
+    };
+};
+export type CaptureWorkspaceOutcomeOptions = {
+    startPath: string;
+    finalPath: string;
+    excludeDirectoryNames?: readonly string[];
+    respectGitignore?: boolean;
+    omitEmptyDirectories?: boolean;
+    id?: string;
+    source?: string;
+    relativePath?: string;
+    snapshotRelativePath?: string;
+};
+export type CapturedWorkspaceOutcome = {
+    descriptor: RunBundleEvidenceDescriptor;
+    fingerprint: DigestString;
+    treeDigest: DigestString;
+    format: "patch" | "snapshot";
+};
+export type CaptureQualificationStatus = "qualified" | "qualified-with-gaps" | "unqualified";
+export type CaptureQualificationDimension = "attemptIdentity" | "semanticEvidence" | "hooks" | "telemetry" | "workspace" | "verifier" | "terminal" | "sharing";
+export type CaptureQualificationReasonCode = "MANIFEST_INVALID" | "ATTEMPT_IDENTITY_INVALID" | "ARTIFACT_INTEGRITY_INVALID" | "ARTIFACT_TOO_LARGE" | "NATIVE_JSON_EMPTY" | "NATIVE_JSON_MALFORMED" | "NATIVE_JSONL_EMPTY" | "NATIVE_JSONL_MALFORMED" | "SESSION_EVIDENCE_MISSING" | "SESSION_IDENTITY_MISSING" | "SESSION_RECORD_IDENTITY_MISMATCH" | "HOOK_EVIDENCE_MISSING" | "HOOK_RECORDS_MISSING" | "HOOK_CAPABILITY_NOT_CHECKED" | "HOOK_CAPABILITY_VERSION_MISMATCH" | "HOOK_SUPPORTED_BUT_MISSING" | "HOOK_UNSUPPORTED_BY_PINNED_SDK" | "HOOK_CAPABILITY_CONTRADICTION" | "HOOK_CAPTURE_WARNING" | "TELEMETRY_EVIDENCE_MISSING" | "TELEMETRY_RECEIPT_MISSING" | "TELEMETRY_UNSUPPORTED_BY_PINNED_RUNTIME" | "OPTIONAL_BETA_TIMING_UNAVAILABLE" | "WORKSPACE_EVIDENCE_MISSING" | "WORKSPACE_PATCH_NOT_CHECKED" | "WORKSPACE_PATCH_UNUSABLE" | "VERIFIER_EVIDENCE_MISSING" | "VERIFIER_RESULT_ERROR" | "VERIFIER_RESULT_NOT_RUN" | "TERMINAL_CLASSIFICATION_INVALID" | "INFRASTRUCTURE_FAILURE_MISCLASSIFIED_AS_TASK" | "SHARING_CLASSIFICATION_UNKNOWN" | "CAPTURE_REPORT_MISSING" | "CAPTURE_REPORT_INVALID" | "CAPTURE_REPORT_CONTRADICTS_SOURCE" | "EVENT_LOG_COMPLETENESS_UNPROVEN" | "EXPORT_MANIFEST_INVALID";
+export type CaptureQualificationReason = {
+    code: CaptureQualificationReasonCode;
+    dimension: CaptureQualificationDimension;
+    evidenceId?: string;
+    detail: string;
+};
+export type CaptureQualificationReport = {
+    status: CaptureQualificationStatus;
+    semanticAnalysisUsable: boolean;
+    assessmentMode?: AssessmentMode;
+    attempt?: AttemptIdentity;
+    terminal?: TerminalRecord;
+    dimensions: Record<CaptureQualificationDimension, {
+        status: "qualified" | "unsupported" | "gap" | "unqualified";
+        reasonCodes: CaptureQualificationReasonCode[];
+    }>;
+    reasons: CaptureQualificationReason[];
+};
+export type AgentSdkQualificationEvidence = Pick<ClaudeAgentSdkAttemptEvidence, "capabilities" | "effectiveConfiguration" | "captureWarnings">;
+export type CaptureQualificationOptions = {
+    startingWorkspacePath?: string;
+    workspaceOutcomeExcludedDirectoryNames?: readonly string[];
+    workspaceOutcomeRespectsGitignore?: boolean;
+    workspaceOutcomeOmitsEmptyDirectories?: boolean;
+    hookCapabilities?: Pick<ClaudeAgentSdkCapabilities, "sdkVersion" | "hooks" | "unsupportedHooks">;
+    agentSdkEvidence?: AgentSdkQualificationEvidence;
+    expectedHooks?: readonly string[];
+    semanticEvidenceKinds?: readonly ("session" | "hook")[];
+    relatedSessionIds?: readonly string[];
+};
+export declare function createRunBundleAssembler(definition: RunBundleDefinition): Promise<RunBundleAssembler>;
+export declare class RunBundleAssembler {
+    readonly bundleRoot: string;
+    readonly bundleId: string;
+    readonly attempt: AttemptIdentity;
+    private run;
+    private readonly configuration;
+    private evidence;
+    private manifest?;
+    private revision;
+    private finalized;
+    private captureMissing;
+    constructor(definition: RunBundleDefinition);
+    initialize(): Promise<void>;
+    currentManifest(): RunManifest;
+    registerArtifact(input: RegisterRunBundleArtifact): Promise<RunBundleEvidenceDescriptor>;
+    writeJsonArtifact(input: RegisterRunBundleArtifact, document: unknown): Promise<RunBundleEvidenceDescriptor>;
+    writeAgentSdkTelemetry(input: {
+        evidence: Pick<ClaudeAgentSdkAttemptEvidence, "telemetry" | "usage">;
+        traceId?: string;
+        id?: string;
+        relativePath?: string;
+    }): Promise<RunBundleEvidenceDescriptor>;
+    captureWorkspaceOutcome(options: CaptureWorkspaceOutcomeOptions, whileProjected?: (projectedPath: string, outcome: CapturedWorkspaceOutcome) => Promise<void>): Promise<CapturedWorkspaceOutcome>;
+    finalize(input: {
+        terminal: TerminalRecord;
+        missingEvidence?: CaptureMissingEvidence[];
+        qualification?: CaptureQualificationOptions;
+    }): Promise<RunManifest>;
+    private assertOpen;
+    private checkpoint;
+}
+/** Structurally qualify one retained run bundle without normalizing native evidence. */
+export declare function qualifyRunBundle(bundleRoot: string, options?: CaptureQualificationOptions): Promise<CaptureQualificationReport>;
+export {};