engineering-behavior-observatory 0.2.1

package/dist/src/openhands-run.js

@@ -0,0 +1,415 @@
+import { writeArtifactAtomically } from "./artifacts.js";
+import { captureOpenHandsAgentServer, normalizeOpenHandsCapture, OPENHANDS_AGENT_SERVER_VERSION, OPENHANDS_DEFAULT_MAX_RESPONSE_BYTES, OPENHANDS_MAX_RESPONSE_BYTES, } from "./openhands.js";
+import { createAttemptIdentity, createRunIdentity, executeRunAttempt, } from "./lifecycle.js";
+import { createRunBundleAssembler, qualifyRunBundle, } from "./run-bundles.js";
+import { validateUniformEvents } from "./uniform-events.js";
+import { openJsonlEvidenceWriter } from "./process-protocol.js";
+/** Execute and retain one caller-configured OpenHands Agent Server attempt. */
+export async function captureOpenHandsAgentServerRun(options) {
+    if (options.definition.run.harness.id !== "openhands-agent-server") {
+        throw new Error(`OpenHands run harness must be openhands-agent-server; received ${options.definition.run.harness.id}.`);
+    }
+    if (options.definition.run.model.id !== options.configuration.model) {
+        throw new Error("The declared model must match the OpenHands run configuration.");
+    }
+    const requestModel = openHandsRequestModel(options.configuration.startConversation);
+    if (requestModel !== options.configuration.model) {
+        throw new Error(`The actual conversation request model must match ${options.configuration.model}; received ${String(requestModel)}.`);
+    }
+    const definition = withPinnedRuntime(options.definition);
+    const assembler = await createRunBundleAssembler(definition);
+    const sessionWriter = await openJsonlEvidenceWriter(`${assembler.bundleRoot}/session.jsonl`, {
+        exclusive: true,
+        maxLineBytes: Math.min(Math.max(1, options.configuration.maxResponseBytes ?? OPENHANDS_DEFAULT_MAX_RESPONSE_BYTES), OPENHANDS_MAX_RESPONSE_BYTES) + 64 * 1024,
+    });
+    let workspace;
+    let workspaceOutcome;
+    let workspaceOutcomePromise;
+    let workspaceCaptureError;
+    let verifierResult;
+    let capture;
+    let nativeEvidenceWritten = false;
+    let captureSettled = false;
+    let progressiveSessionId;
+    let progressiveWriteError;
+    let acceptingRecords = true;
+    const progressiveRecords = [];
+    const workspaceOutcomeExcludedDirectoryNames = [...new Set([
+            ".git",
+            ...(options.workspaceOutcomeExcludedDirectoryNames ?? []).filter((name) => name !== ".git"),
+        ])];
+    const qualificationOptions = {
+        startingWorkspacePath: options.startingWorkspacePath,
+        workspaceOutcomeExcludedDirectoryNames,
+        ...(options.workspaceOutcomeRespectsGitignore === undefined ? {} : {
+            workspaceOutcomeRespectsGitignore: options.workspaceOutcomeRespectsGitignore,
+        }),
+        ...(options.workspaceOutcomeOmitsEmptyDirectories === undefined ? {} : {
+            workspaceOutcomeOmitsEmptyDirectories: options.workspaceOutcomeOmitsEmptyDirectories,
+        }),
+    };
+    const captureWorkspace = async (verifierContext) => {
+        if (workspaceOutcome !== undefined)
+            return workspaceOutcome;
+        if (workspace?.status !== "ready" || workspace.path === undefined || workspace.artifactId === undefined) {
+            throw new Error("OpenHands capture requires a retained ready workspace before outcome packaging.");
+        }
+        workspaceOutcomePromise ??= assembler.captureWorkspaceOutcome({
+            startPath: options.startingWorkspacePath,
+            finalPath: workspace.path,
+            id: workspace.artifactId,
+            excludeDirectoryNames: workspaceOutcomeExcludedDirectoryNames,
+            ...(options.workspaceOutcomeRespectsGitignore === undefined ? {} : {
+                respectGitignore: options.workspaceOutcomeRespectsGitignore,
+            }),
+            ...(options.workspaceOutcomeOmitsEmptyDirectories === undefined ? {} : {
+                omitEmptyDirectories: options.workspaceOutcomeOmitsEmptyDirectories,
+            }),
+        }, verifierContext === undefined || options.verifier === undefined
+            ? undefined
+            : async (projectedPath, outcome) => {
+                workspaceOutcome = outcome;
+                verifierResult = await options.verifier(verifierContext, outcome, projectedPath);
+            });
+        workspaceOutcome = await workspaceOutcomePromise;
+        return workspaceOutcome;
+    };
+    const writeNativeEvidence = async () => {
+        if (nativeEvidenceWritten)
+            return;
+        await sessionWriter.flush();
+        if (progressiveWriteError !== undefined)
+            throw new Error(progressiveWriteError);
+        if (!captureSettled || sessionWriter.count === 0)
+            return;
+        await sessionWriter.close();
+        await assembler.registerArtifact({
+            id: "session",
+            source: "openhands-agent-server",
+            kind: "session",
+            mediaType: "application/x-ndjson",
+            sharingClass: "restricted",
+            relativePath: "session.jsonl",
+            ...(progressiveSessionId === undefined ? {} : {
+                nativeReference: { type: "session", id: progressiveSessionId },
+            }),
+        });
+        if (capture === undefined) {
+            nativeEvidenceWritten = true;
+            return;
+        }
+        const hooks = capture.records.filter(({ record }) => record.payload.kind === "HookExecutionEvent");
+        if (hooks.length > 0) {
+            const hookBytes = Buffer.from(`${hooks.map(({ record }) => JSON.stringify({
+                session_id: capture.conversationId,
+                hook_event_name: record.payload.hook_event_type,
+                native_record: record,
+            })).join("\n")}\n`);
+            await writeArtifactAtomically(assembler.bundleRoot, "hooks.jsonl", hookBytes, undefined, { overwrite: false });
+            await assembler.registerArtifact({
+                id: "hooks",
+                source: "openhands-agent-server",
+                kind: "hook",
+                mediaType: "application/x-ndjson",
+                sharingClass: "restricted",
+                relativePath: "hooks.jsonl",
+            });
+        }
+        nativeEvidenceWritten = true;
+    };
+    const coordinatedWorkspace = {
+        setup: async (context) => {
+            workspace = await options.workspace.setup(context);
+            return workspace;
+        },
+        cleanup: async (context) => {
+            try {
+                if (workspace?.status === "ready")
+                    await captureWorkspace();
+            }
+            catch (error) {
+                if (workspaceOutcome === undefined) {
+                    workspaceCaptureError = error instanceof Error ? error.message : String(error);
+                    throw error;
+                }
+            }
+            await options.workspace.cleanup?.(context);
+        },
+    };
+    const run = createRunIdentity({
+        id: definition.run.id,
+        taskId: definition.run.task.id,
+        modelId: definition.run.model.id,
+        harnessId: definition.run.harness.id,
+    });
+    const attemptIdentity = createAttemptIdentity(definition.run.id, definition.attempt.number, definition.attempt.id, definition.attempt.retryOf);
+    const attempt = await executeRunAttempt({
+        run,
+        assessmentMode: definition.run.assessmentMode,
+        attempt: attemptIdentity,
+        workspace: coordinatedWorkspace,
+        harness: async ({ signal, registerShutdown }) => {
+            if (workspace?.status !== "ready" || workspace.path === undefined) {
+                throw new Error("OpenHands run requires a ready workspace path.");
+            }
+            const { serverWorkspacePath, ...configuration } = options.configuration;
+            registerShutdown(async () => {
+                if (progressiveSessionId === undefined)
+                    return;
+                await cleanupOpenHandsConversation(configuration, progressiveSessionId, options.shutdownGraceMs ?? 250);
+            });
+            try {
+                capture = await captureOpenHandsAgentServer({
+                    ...configuration,
+                    runId: definition.run.id,
+                    attemptId: definition.attempt.id,
+                    signal,
+                    onRecord: (captured) => {
+                        if (!acceptingRecords)
+                            return;
+                        if (captured.record.session_id !== undefined)
+                            progressiveSessionId = captured.record.session_id;
+                        progressiveRecords.push(captured);
+                        void sessionWriter.append(captured.record).catch((error) => {
+                            progressiveWriteError ??= error instanceof Error ? error.message : String(error);
+                        });
+                    },
+                    startConversation: {
+                        ...options.configuration.startConversation,
+                        workspace: { type: "local", working_dir: serverWorkspacePath ?? workspace.path },
+                    },
+                });
+            }
+            finally {
+                captureSettled = true;
+            }
+            if (capture.captureError !== undefined) {
+                return {
+                    status: "failed",
+                    failureClass: "infrastructure",
+                    reason: capture.captureError,
+                    captureError: capture.captureError,
+                    evidence: capture,
+                };
+            }
+            const status = capture.finalConversation?.execution_status;
+            return status === "finished"
+                ? {
+                    status: "completed",
+                    completionEvidence: capture.finalConversation,
+                    evidence: capture,
+                    ...(capture.reconciliation.finalReadError === undefined ? {} : {
+                        captureError: capture.reconciliation.finalReadError,
+                    }),
+                }
+                : { status: "failed", failureClass: "infrastructure", reason: `OpenHands conversation ended as ${String(status)}.`, evidence: capture };
+        },
+        ...(options.verifier === undefined ? {} : {
+            verifier: async (context) => {
+                await captureWorkspace(context);
+                if (verifierResult === undefined)
+                    throw new Error("OpenHands verifier did not return a result.");
+                await assembler.writeJsonArtifact({
+                    id: "verifier",
+                    source: "ebo-verifier",
+                    kind: "verifier",
+                    mediaType: "application/json",
+                    sharingClass: "restricted",
+                    relativePath: "verifier.json",
+                }, verifierResult);
+                return {
+                    status: verifierResult.status,
+                    ...(verifierResult.error === undefined ? {} : { error: verifierResult.error }),
+                    evidence: verifierResult,
+                };
+            },
+        }),
+        evidence: { flush: writeNativeEvidence },
+        ...(options.signal === undefined ? {} : { signal: options.signal }),
+        ...(options.maxWallClockMs === undefined ? {} : { maxWallClockMs: options.maxWallClockMs }),
+        ...(options.harnessBudgetMs === undefined ? {} : { harnessBudgetMs: options.harnessBudgetMs }),
+        ...(options.shutdownGraceMs === undefined ? {} : { shutdownGraceMs: options.shutdownGraceMs }),
+    });
+    if (capture === undefined) {
+        acceptingRecords = false;
+        await sessionWriter.close();
+        captureSettled = true;
+        if (progressiveSessionId !== undefined && sessionWriter.count > 0) {
+            capture = interruptedCapture(definition.run.id, definition.attempt.id, progressiveSessionId, progressiveRecords.slice(0, sessionWriter.count));
+        }
+    }
+    if (capture === undefined) {
+        await writeNativeEvidence();
+        const terminal = structuredClone(attempt.terminal);
+        if (workspaceOutcome === undefined)
+            delete terminal.workspaceArtifactId;
+        const manifest = await assembler.finalize({
+            terminal,
+            missingEvidence: [
+                {
+                    kind: sessionWriter.count === 0 ? "session" : "session-identity",
+                    reason: "not-collected",
+                    affects: ["semantic"],
+                    detail: "The attempt ended before the OpenHands harness produced an identified native session.",
+                },
+                {
+                    kind: "telemetry",
+                    reason: "unsupported",
+                    affects: ["timing-resource"],
+                    detail: "The pinned Agent Server REST/WebSocket boundary does not expose native OpenTelemetry records.",
+                },
+            ],
+            qualification: qualificationOptions,
+        });
+        const qualification = await qualifyRunBundle(assembler.bundleRoot, qualificationOptions);
+        return {
+            attempt,
+            manifest,
+            qualification,
+            ...(workspace?.status === "ready" && workspaceOutcome === undefined
+                ? { retainedWorkspacePath: workspace.path }
+                : {}),
+        };
+    }
+    await writeNativeEvidence();
+    const terminal = structuredClone(attempt.terminal);
+    if (workspaceOutcome === undefined)
+        delete terminal.workspaceArtifactId;
+    const missingEvidence = [
+        {
+            kind: "telemetry",
+            reason: "unsupported",
+            affects: ["timing-resource"],
+            detail: "The pinned Agent Server REST/WebSocket boundary does not expose native OpenTelemetry records.",
+        },
+        {
+            kind: "event-log-completeness",
+            reason: "not-checked",
+            affects: ["semantic"],
+            detail: capture.eventLogCompleteness.reason,
+        },
+        ...(capture.captureError === undefined ? [] : [{
+                kind: "agent-server-control",
+                reason: "not-collected",
+                affects: ["semantic"],
+                detail: capture.captureError,
+            }]),
+        ...(capture.reconciliation.finalReadError === undefined ? [] : [{
+                kind: "final-rest-events",
+                reason: "not-collected",
+                affects: ["semantic"],
+                detail: capture.reconciliation.finalReadError,
+            }]),
+        ...(capture.reconciliation.transportGaps.length === 0 ? [] : [{
+                kind: "websocket-events",
+                reason: "not-collected",
+                affects: ["semantic"],
+                detail: `Transport gaps: ${capture.reconciliation.transportGaps.join(", ")}`,
+            }]),
+        ...(workspaceOutcome === undefined && workspaceCaptureError !== undefined ? [{
+                kind: "workspace",
+                reason: "not-collected",
+                affects: ["outcome"],
+                detail: workspaceCaptureError.slice(0, 4096),
+            }] : []),
+    ];
+    const manifest = await assembler.finalize({ terminal, missingEvidence, qualification: qualificationOptions });
+    const qualification = await qualifyRunBundle(assembler.bundleRoot, qualificationOptions);
+    const retainedWorkspacePath = workspace?.status === "ready" && workspaceOutcome === undefined
+        ? workspace.path
+        : undefined;
+    if (!["qualified", "qualified-with-gaps"].includes(qualification.status)) {
+        return {
+            attempt,
+            manifest,
+            qualification,
+            capture,
+            ...(retainedWorkspacePath === undefined ? {} : { retainedWorkspacePath }),
+        };
+    }
+    try {
+        const normalized = await normalizeOpenHandsCapture(capture);
+        await validateUniformEvents(normalized.events, {
+            resolve: (reference) => resolvesCaptureReference(capture, reference),
+        });
+        return { attempt, manifest, qualification, capture, normalized };
+    }
+    catch (error) {
+        return {
+            attempt,
+            manifest,
+            qualification,
+            capture,
+            normalizationError: (error instanceof Error ? error.message : String(error)).slice(0, 4096),
+        };
+    }
+}
+function resolvesCaptureReference(capture, reference) {
+    return capture.records.some(({ reference: base }) => base.artifactId === reference.artifactId
+        && (reference.recordLocator === base.recordLocator || reference.recordLocator.startsWith(`${base.recordLocator}#`)));
+}
+function interruptedCapture(runId, attemptId, conversationId, records) {
+    const events = (channel) => records
+        .filter(({ record }) => record.channel === channel)
+        .map(({ record }) => record.payload);
+    const ids = (values) => [...new Set(values.flatMap(({ id }) => typeof id === "string" && id !== "" ? [id] : []))];
+    const streamedEventIds = ids(events("websocket-event"));
+    const finalEventIds = ids(events("rest-event"));
+    const streamedSet = new Set(streamedEventIds);
+    const finalSet = new Set(finalEventIds);
+    return {
+        runId,
+        attemptId,
+        qualification: "qualified-with-gaps",
+        records: structuredClone(records),
+        conversationId,
+        serverInfo: structuredClone(events("server-info")[0] ?? {}),
+        captureError: "OpenHands capture was aborted before returning.",
+        reconciliation: {
+            status: "partial",
+            streamedEventIds,
+            finalEventIds,
+            streamedOnlyEventIds: streamedEventIds.filter((id) => !finalSet.has(id)),
+            finalOnlyEventIds: finalEventIds.filter((id) => !streamedSet.has(id)),
+            transportGaps: ["capture-aborted"],
+        },
+        eventLogCompleteness: {
+            status: "unknown",
+            reason: "REST/WebSocket reconciliation cannot prove complete in-process EventLog delivery.",
+        },
+    };
+}
+function withPinnedRuntime(definition) {
+    const runtime = definition.run.runtime.filter(({ name }) => name !== "openhands-agent-server");
+    runtime.push({ source: "OpenHands", name: "openhands-agent-server", version: OPENHANDS_AGENT_SERVER_VERSION });
+    return {
+        ...structuredClone(definition),
+        run: {
+            ...structuredClone(definition.run),
+            harness: { ...structuredClone(definition.run.harness), version: OPENHANDS_AGENT_SERVER_VERSION },
+            runtime,
+        },
+    };
+}
+function openHandsRequestModel(startConversation) {
+    for (const candidate of [startConversation.agent, startConversation.agent_settings]) {
+        if (typeof candidate !== "object" || candidate === null || Array.isArray(candidate))
+            continue;
+        const llm = candidate.llm;
+        if (typeof llm !== "object" || llm === null || Array.isArray(llm))
+            continue;
+        const model = llm.model;
+        if (typeof model === "string" && model !== "")
+            return model;
+    }
+    return undefined;
+}
+async function cleanupOpenHandsConversation(configuration, conversationId, timeoutMs) {
+    const headers = new Headers();
+    if (configuration.sessionApiKey !== undefined)
+        headers.set("x-session-api-key", configuration.sessionApiKey);
+    const response = await (configuration.fetch ?? globalThis.fetch)(new URL(`/api/conversations/${encodeURIComponent(conversationId)}`, configuration.baseUrl), { method: "DELETE", headers, signal: AbortSignal.timeout(Math.max(1, timeoutMs)), redirect: "manual" });
+    if (!response.ok && response.status !== 404) {
+        throw new Error(`OpenHands conversation cleanup failed with ${response.status}.`);
+    }
+}

package/dist/src/openhands.d.ts

@@ -0,0 +1,130 @@
+import type { AdapterCapabilityProfile, CapturedNativeRecord, HarnessAdapter, NormalizationInput, NormalizationResult, QualifiedNativeCapture } from "./uniform-events.js";
+export declare const OPENHANDS_AGENT_SERVER_VERSION = "1.46.0";
+export declare const OPENHANDS_TYPESCRIPT_CLIENT_VERSION = "1.39.0";
+export declare const OPENHANDS_DEFAULT_MAX_RESPONSE_BYTES: number;
+export declare const OPENHANDS_MAX_RESPONSE_BYTES: number;
+export declare const OPENHANDS_DEFAULT_MAX_CAPTURE_BYTES: number;
+export declare const OPENHANDS_MAX_CAPTURE_BYTES: number;
+export declare const OPENHANDS_AGENT_SERVER_CAPABILITIES: {
+    readonly schemaVersion: "ebo.adapter-capability-profile/v1";
+    readonly adapterId: "openhands-agent-server-v1.46.0";
+    readonly harness: "openhands-agent-server";
+    readonly nativeTypes: readonly ["MessageEvent", "SystemPromptEvent", "ActionEvent", "ObservationEvent", "AgentErrorEvent", "UserRejectObservation", "CondensationRequest", "Condensation", "CondensationSummaryEvent", "ConversationErrorEvent", "ServerErrorEvent", "HookExecutionEvent", "ConversationStateUpdateEvent", "InterruptEvent", "PauseEvent", "conversation-final"];
+    readonly families: {
+        readonly message: {
+            readonly status: "available";
+        };
+        readonly "model-request": {
+            readonly status: "unsupported";
+            readonly detail: "The pinned public event boundary does not expose model requests.";
+        };
+        readonly tool: {
+            readonly status: "available";
+        };
+        readonly context: {
+            readonly status: "available";
+        };
+        readonly permission: {
+            readonly status: "unsupported";
+            readonly detail: "Hook blocking is retained natively but is not projected as a permission event.";
+        };
+        readonly delegation: {
+            readonly status: "unsupported";
+            readonly detail: "Nested delegate actions remain source-specific tool evidence.";
+        };
+        readonly artifact: {
+            readonly status: "unsupported";
+            readonly detail: "Workspace outcome is packaged outside the Agent Server event normalizer.";
+        };
+        readonly validation: {
+            readonly status: "unsupported";
+            readonly detail: "Verifier evidence is packaged outside the Agent Server event normalizer.";
+        };
+        readonly runtime: {
+            readonly status: "available";
+        };
+        readonly outcome: {
+            readonly status: "partial";
+            readonly detail: "Final conversation state is available when the run reaches the final REST read.";
+        };
+    };
+    readonly evidence: {
+        readonly nativeOrder: {
+            readonly status: "partial";
+            readonly detail: "REST page and WebSocket receipt order remain separate source domains.";
+        };
+        readonly nativeTime: {
+            readonly status: "partial";
+            readonly detail: "Event timestamps are available; control records may omit native time.";
+        };
+        readonly parentage: {
+            readonly status: "partial";
+            readonly detail: "Event parent_id and selected action associations are projected when their targets are captured.";
+        };
+        readonly content: {
+            readonly status: "partial";
+            readonly detail: "Supported content stays in native records and is referenced rather than copied.";
+        };
+    };
+};
+/** Retained evidence keeps the identity of the runtime that produced it. */
+export declare function openHandsCapabilityProfile(version: string): AdapterCapabilityProfile;
+export interface OpenHandsWebSocket {
+    readonly readyState: number;
+    addEventListener(type: string, listener: (event: {
+        data?: unknown;
+    }) => void): void;
+    removeEventListener(type: string, listener: (event: {
+        data?: unknown;
+    }) => void): void;
+    send(data: string): void;
+    close(): void;
+}
+export type OpenHandsNativeRecord = {
+    schemaVersion: "ebo.openhands-native-record/v1";
+    session_id?: string;
+    channel: "server-info" | "conversation-create-response" | "conversation-created" | "websocket-status" | "websocket-event" | "conversation-final" | "rest-event" | "capture-error" | "cleanup";
+    sequence: number;
+    channelSequence?: number;
+    payload: Record<string, unknown>;
+};
+export type OpenHandsCaptureRequest = {
+    runId: string;
+    attemptId: string;
+    baseUrl: string;
+    startConversation: Record<string, unknown>;
+    message: Record<string, unknown>;
+    sessionApiKey?: string;
+    fetch?: typeof globalThis.fetch;
+    webSocket?: (url: string) => OpenHandsWebSocket;
+    pollIntervalMs?: number;
+    timeoutMs?: number;
+    maxReconnects?: number;
+    reconnectDelayMs?: number;
+    maxResponseBytes?: number;
+    maxCaptureBytes?: number;
+    signal?: AbortSignal;
+    onRecord?: (record: CapturedNativeRecord<OpenHandsNativeRecord>) => void;
+};
+export type OpenHandsCapture = QualifiedNativeCapture<OpenHandsNativeRecord> & {
+    conversationId?: string;
+    serverInfo: Record<string, unknown>;
+    finalConversation?: Record<string, unknown>;
+    captureError?: string;
+    reconciliation: {
+        status: "matched" | "mismatch" | "partial";
+        streamedEventIds: string[];
+        finalEventIds: string[];
+        streamedOnlyEventIds: string[];
+        finalOnlyEventIds: string[];
+        transportGaps: string[];
+        finalReadError?: string;
+    };
+    eventLogCompleteness: {
+        status: "unknown";
+        reason: string;
+    };
+};
+export declare function createOpenHandsHarnessAdapter(): HarnessAdapter<OpenHandsCaptureRequest, OpenHandsNativeRecord>;
+export declare function captureOpenHandsAgentServer(request: OpenHandsCaptureRequest): Promise<OpenHandsCapture>;
+export declare function normalizeOpenHandsCapture(capture: NormalizationInput<OpenHandsNativeRecord>): Promise<NormalizationResult>;