@neta-art/cohub-cli 7.0.0 → 7.1.0

package/dist/runtime/connection.js

@@ -1,31 +1,63 @@
+import { randomUUID } from "node:crypto";
 import { setTimeout as delay } from "node:timers/promises";
-import { RUNTIME_MAX_FRAME_BYTES, RUNTIME_PROTOCOL_VERSION, runtimeCommandSchema, runtimeReadySchema } from "@neta-art/cohub";
+import { RUNTIME_MAX_FRAME_BYTES, RUNTIME_PROTOCOL_VERSION, runtimeCommandSchema, runtimeReadySchema, } from "@neta-art/cohub";
 import { executeCodex, executePi } from "./harness.js";
 import { ProcessCleanupUncertainError } from "./process-group.js";
 import { ContextRequiredError } from "./session-store.js";
+import { serializeDiagnosticError, } from "./diagnostics.js";
 export async function serveRuntime(options) {
+    const runtimeId = options.runtimeId ?? options.diagnostics?.runtimeId ?? randomUUID();
     let backoff = 500;
+    let attempt = 0;
     let conflictSince = null;
     const uploads = new AbortController();
     const uploadSignal = AbortSignal.any([options.signal, uploads.signal]);
+    const log = (level, event, data, context) => options.diagnostics?.log(level, event, data, context);
     const flush = () => options.store.flushArchives(uploadSignal).catch((error) => {
-        if (!uploadSignal.aborted)
-            console.error("Archive pending / 归档待重试:", error);
+        if (!uploadSignal.aborted) {
+            log("warn", "archive.flush_failed", { error: serializeDiagnosticError(error) });
+            console.error("Archive pending:", error);
+        }
     });
-    const timer = setInterval(() => { void flush(); }, 10_000);
+    const timer = setInterval(() => {
+        void flush();
+    }, 10_000);
     void flush();
+    log("info", "runtime.started", {
+        runtimeId,
+        workspace: options.cwd,
+        harnesses: options.capabilities.harnesses,
+    });
     try {
         while (!options.signal.aborted) {
-            const outcome = await connect({ ...options, onReady: () => { backoff = 500; conflictSince = null; options.onReady(); void flush(); } });
+            attempt += 1;
+            const outcome = await connect({
+                ...options,
+                runtimeId,
+                attempt,
+                onReady: () => {
+                    backoff = 500;
+                    attempt = 0;
+                    conflictSince = null;
+                    options.onReady();
+                    void flush();
+                },
+            });
             if (options.signal.aborted)
                 return;
             if (outcome === "fatal")
-                throw new Error("Runtime connection rejected / Runtime 连接被拒绝");
+                throw new Error("Runtime connection rejected");
             if (outcome === "conflict") {
                 conflictSince ??= Date.now();
-                if (Date.now() - conflictSince >= (options.leaseConflictTimeoutMs ?? 90_000))
-                    throw new Error("Space is already connected to another Runtime / Space 已连接其他 Runtime");
+                if (Date.now() - conflictSince >= (options.leaseConflictTimeoutMs ?? 90_000)) {
+                    throw new Error("Space is already connected to another Runtime");
+                }
             }
+            log("debug", "runtime.reconnect_scheduled", {
+                attempt: attempt || 1,
+                delayMs: backoff,
+                outcome,
+            });
             await delay(backoff, undefined, { signal: options.signal }).catch(() => undefined);
             backoff = Math.min(10_000, backoff * 2);
         }
@@ -34,11 +66,37 @@ export async function serveRuntime(options) {
         clearInterval(timer);
         uploads.abort();
         await flush();
+        log("info", "runtime.stopped", { runtimeId });
     }
 }
+function executionContext(connectionId, input) {
+    return {
+        connectionId,
+        sessionId: input.sessionId,
+        turnId: input.turnId,
+        harness: input.harness,
+        requestId: input.traceContext?.requestId ?? input.requestId ?? null,
+        traceContext: input.traceContext,
+    };
+}
 async function connect(options) {
-    let token = await options.token();
-    const socket = new WebSocket(options.url);
+    const log = (level, event, data, context) => options.diagnostics?.log(level, event, data, context);
+    const connectedAt = Date.now();
+    log("debug", "runtime.websocket.connecting", {
+        attempt: options.attempt,
+        url: options.url,
+    });
+    let currentToken;
+    try {
+        currentToken = await options.token();
+    }
+    catch (error) {
+        log("error", "runtime.auth_token_failed", { error: serializeDiagnosticError(error) });
+        throw error;
+    }
+    const runtimeUrl = new URL(options.url);
+    runtimeUrl.searchParams.set("runtimeId", options.runtimeId);
+    const socket = new WebSocket(runtimeUrl.toString());
     const active = new Map();
     const seen = new Set();
     const disconnected = new AbortController();
@@ -49,30 +107,69 @@ async function connect(options) {
     let conflict = false;
     let readyTimer;
     const send = (frame) => {
-        if (socket.readyState !== WebSocket.OPEN || socket.bufferedAmount > RUNTIME_MAX_FRAME_BYTES)
+        if (socket.readyState !== WebSocket.OPEN || socket.bufferedAmount > RUNTIME_MAX_FRAME_BYTES) {
+            log("warn", "runtime.frame_send_unavailable", {
+                readyState: socket.readyState,
+                bufferedBytes: socket.bufferedAmount,
+            }, { connectionId });
             throw new Error("Runtime connection unavailable");
+        }
         const data = JSON.stringify(frame);
-        if (Buffer.byteLength(data) > RUNTIME_MAX_FRAME_BYTES)
+        const bytes = Buffer.byteLength(data);
+        if (bytes > RUNTIME_MAX_FRAME_BYTES) {
+            log("error", "runtime.frame_too_large", { bytes, maxBytes: RUNTIME_MAX_FRAME_BYTES }, { connectionId });
             throw new Error("Runtime frame exceeds transfer limit");
+        }
         socket.send(data);
     };
-    const stop = () => { for (const execution of active.values())
-        execution.controller.abort(); socket.close(); };
+    const stop = () => {
+        for (const execution of active.values())
+            execution.controller.abort();
+        socket.close();
+    };
     options.signal.addEventListener("abort", stop, { once: true });
     const heartbeat = setInterval(() => {
-        if (Date.now() - lastHeartbeat > 30_000)
+        const heartbeatAgeMs = Date.now() - lastHeartbeat;
+        if (heartbeatAgeMs > 30_000) {
+            log("error", "runtime.heartbeat_timeout", {
+                ageMs: heartbeatAgeMs,
+                activeExecutions: active.size,
+            }, { connectionId });
+            for (const execution of active.values()) {
+                log("error", "runtime.execution_heartbeat_timeout", {
+                    ageMs: heartbeatAgeMs,
+                }, executionContext(connectionId, execution));
+            }
             stop();
+        }
         else if (connectionId) {
             try {
                 send({ type: "runtime.heartbeat" });
             }
-            catch {
+            catch (error) {
+                log("warn", "runtime.heartbeat_send_failed", {
+                    error: serializeDiagnosticError(error),
+                }, { connectionId });
                 stop();
             }
-            void options.token().then((next) => { if (next !== token) {
-                send({ type: "runtime.auth", token: next });
-                token = next;
-            } }).catch(stop);
+            void options.token().then((next) => {
+                if (next !== currentToken) {
+                    try {
+                        send({ type: "runtime.auth", token: next });
+                        currentToken = next;
+                        log("debug", "runtime.auth_refreshed", undefined, { connectionId });
+                    }
+                    catch (error) {
+                        log("warn", "runtime.auth_refresh_send_failed", {
+                            error: serializeDiagnosticError(error),
+                        }, { connectionId });
+                        stop();
+                    }
+                }
+            }).catch((error) => {
+                log("warn", "runtime.auth_refresh_failed", { error: serializeDiagnosticError(error) }, { connectionId });
+                stop();
+            });
         }
     }, 10_000);
     const closed = new Promise((resolve) => {
@@ -80,25 +177,63 @@ async function connect(options) {
             fatal = [4400, 4401, 4403].includes(event.code);
             conflict = event.code === 4409;
             disconnected.abort();
+            log(fatal || conflict ? "error" : "warn", "runtime.websocket.closed", {
+                code: event.code,
+                reason: event.reason,
+                durationMs: Date.now() - connectedAt,
+                activeExecutions: active.size,
+                fatal,
+                conflict,
+            }, { connectionId });
             if (!options.signal.aborted)
                 console.error(`Runtime disconnected (${event.code}): ${event.reason}`);
-            for (const execution of active.values())
+            for (const execution of active.values()) {
+                log("error", "runtime.execution_transport_lost", {
+                    code: event.code,
+                    reason: event.reason,
+                }, executionContext(connectionId, execution));
                 execution.controller.abort();
+            }
             resolve();
         }, { once: true });
     });
-    socket.addEventListener("error", () => socket.close());
+    socket.addEventListener("error", (event) => {
+        const detail = event.error;
+        log("warn", "runtime.websocket.error", {
+            type: event.type,
+            error: serializeDiagnosticError(detail ?? new Error("WebSocket error")),
+        }, { connectionId });
+        socket.close();
+    });
     socket.addEventListener("open", () => {
         try {
-            send({ type: "runtime.hello", version: RUNTIME_PROTOCOL_VERSION, spaceId: options.spaceId, token, capabilities: options.capabilities });
+            log("info", "runtime.websocket.open", {
+                attempt: options.attempt,
+                durationMs: Date.now() - connectedAt,
+            });
+            send({
+                type: "runtime.hello",
+                version: RUNTIME_PROTOCOL_VERSION,
+                spaceId: options.spaceId,
+                token: currentToken,
+                capabilities: options.capabilities,
+            });
         }
-        catch {
+        catch (error) {
+            log("error", "runtime.hello_failed", { error: serializeDiagnosticError(error) });
             stop();
         }
     });
     socket.addEventListener("message", (event) => {
         void (async () => {
-            const raw = JSON.parse(String(event.data));
+            let raw;
+            try {
+                raw = JSON.parse(String(event.data));
+            }
+            catch (error) {
+                log("error", "runtime.protocol.invalid_json", { error: serializeDiagnosticError(error) }, { connectionId });
+                throw error;
+            }
             if (raw.type === "runtime.ready") {
                 const frame = runtimeReadySchema.parse(raw);
                 if (connectionId === frame.connectionId)
@@ -107,7 +242,20 @@ async function connect(options) {
                     throw new Error("Runtime connection identity changed");
                 connectionId = frame.connectionId;
                 clearTimeout(readyTimer);
+                log("info", "runtime.ready", {
+                    connectionId,
+                    durationMs: Date.now() - connectedAt,
+                }, { connectionId });
                 options.onReady();
+                let recoveryBatch = 0;
+                for await (const executions of options.store.pendingExecutionBatches()) {
+                    recoveryBatch += 1;
+                    log("info", "runtime.recovery_batch_sent", {
+                        batch: recoveryBatch,
+                        count: executions.length,
+                    }, { connectionId });
+                    send({ type: "runtime.recovery", executions });
+                }
                 return;
             }
             if (!connectionId)
@@ -118,11 +266,15 @@ async function connect(options) {
             }
             const frame = runtimeCommandSchema.parse(raw);
             if (frame.type === "session.context") {
+                log("debug", "runtime.context_received", {
+                    messageCount: frame.context.messages.length,
+                }, { connectionId });
                 contexts.get(frame.requestId)?.(frame.context);
                 contexts.delete(frame.requestId);
                 return;
             }
             if (frame.type === "turn.abort") {
+                log("info", "runtime.turn_abort_received", undefined, { connectionId, requestId: frame.requestId });
                 active.get(frame.requestId)?.controller.abort();
                 return;
             }
@@ -130,6 +282,7 @@ async function connect(options) {
                 const execution = active.get(frame.requestId);
                 if (!execution)
                     return;
+                const context = executionContext(connectionId, execution);
                 try {
                     await execution.promise;
                     if (!execution.result)
@@ -137,11 +290,16 @@ async function connect(options) {
                     await options.store.acknowledge(execution.result.state, frame.turnId, frame.revision);
                 }
                 catch (error) {
+                    log("error", "runtime.turn_ack_failed", {
+                        revision: frame.revision,
+                        error: serializeDiagnosticError(error),
+                    }, context);
                     console.error("Runtime acknowledgement failed; result retained:", error);
                     active.delete(frame.requestId);
-                    send({ type: "runtime.event", requestId: frame.requestId, event: { type: "turn.error", message: "Local acknowledgement failed; result retained / 本地确认失败，结果已保留" } });
+                    send({ type: "runtime.event", requestId: frame.requestId, event: { type: "turn.error", message: "Local acknowledgement failed; result retained" } });
                     return;
                 }
+                log("info", "runtime.turn_acknowledged", { revision: frame.revision }, context);
                 active.delete(frame.requestId);
                 send({ type: "runtime.event", requestId: frame.requestId, event: { type: "turn.acknowledged" } });
                 return;
@@ -150,6 +308,12 @@ async function connect(options) {
                 const identity = frame.execution;
                 if (identity.spaceId !== options.spaceId)
                     throw new Error("Invalid Runtime target");
+                const context = executionContext(connectionId, {
+                    ...identity,
+                    requestId: frame.traceContext?.requestId,
+                    traceContext: frame.traceContext,
+                });
+                log("info", "runtime.recovery_requested", undefined, context);
                 const previous = active.get(frame.requestId);
                 if (previous) {
                     if (previous.sessionId !== identity.sessionId || previous.turnId !== identity.turnId || previous.harness !== identity.harness)
@@ -162,7 +326,13 @@ async function connect(options) {
                     return;
                 }
                 const running = [...active].find(([, entry]) => entry.turnId === identity.turnId && entry.sessionId === identity.sessionId && entry.harness === identity.harness);
-                const recovery = { ...identity, controller: new AbortController(), promise: Promise.resolve() };
+                const recovery = {
+                    ...identity,
+                    requestId: frame.traceContext?.requestId,
+                    traceContext: frame.traceContext,
+                    controller: new AbortController(),
+                    promise: Promise.resolve(),
+                };
                 active.set(frame.requestId, recovery);
                 recovery.promise = (async () => {
                     try {
@@ -182,11 +352,14 @@ async function connect(options) {
                     }
                     catch (error) {
                         if (!recovery.controller.signal.aborted) {
+                            log("error", "runtime.recovery_failed", { error: serializeDiagnosticError(error) }, context);
                             console.error("Runtime recovery failed; original files retained:", error);
                             try {
-                                send({ type: "runtime.event", requestId: frame.requestId, event: { type: "turn.error", uncertain: true, message: "Result unavailable; files retained / 结果不可用，原始文件已保留" } });
+                                send({ type: "runtime.event", requestId: frame.requestId, event: { type: "turn.error", uncertain: true, message: "Result unavailable; files retained" } });
+                            }
+                            catch {
+                                // The next connection can read the same result.
                             }
-                            catch { /* The next connection can read the same result. */ }
                         }
                         active.delete(frame.requestId);
                     }
@@ -195,6 +368,11 @@ async function connect(options) {
             }
             if (frame.input.spaceId !== options.spaceId || !options.capabilities.harnesses.includes(frame.input.harness))
                 throw new Error("Invalid Runtime target");
+            const context = executionContext(connectionId, frame.input);
+            log("info", "runtime.turn_received", {
+                requestId: frame.requestId,
+                resumeOnly: frame.resumeOnly === true,
+            }, context);
             const previous = active.get(frame.requestId);
             if (previous) {
                 if (previous.sessionId !== frame.input.sessionId || previous.turnId !== frame.input.turnId || previous.harness !== frame.input.harness)
@@ -206,17 +384,27 @@ async function connect(options) {
                         send({ type: "runtime.event", requestId: frame.requestId, event });
                 return;
             }
-            const requestContext = async (executionSignal, historyOnly = false) => {
+            const requestContext = async (executionSignal) => {
                 const signal = AbortSignal.any([executionSignal, disconnected.signal]);
                 signal.throwIfAborted();
                 const pendingTurnIds = await options.store.pendingTurnIds(frame.input.sessionId);
+                log("debug", "runtime.context_requested", { pendingTurnCount: pendingTurnIds.length }, context);
                 return new Promise((resolve, reject) => {
-                    const abort = () => { clearTimeout(timeout); contexts.delete(frame.requestId); signal.removeEventListener("abort", abort); reject(new Error("Runtime context request aborted")); };
+                    const abort = () => {
+                        clearTimeout(timeout);
+                        contexts.delete(frame.requestId);
+                        signal.removeEventListener("abort", abort);
+                        reject(new Error("Runtime context request aborted"));
+                    };
                     const timeout = setTimeout(abort, 60_000);
                     signal.addEventListener("abort", abort, { once: true });
-                    contexts.set(frame.requestId, (context) => { clearTimeout(timeout); signal.removeEventListener("abort", abort); resolve(context); });
+                    contexts.set(frame.requestId, (nextContext) => {
+                        clearTimeout(timeout);
+                        signal.removeEventListener("abort", abort);
+                        resolve(nextContext);
+                    });
                     try {
-                        send({ type: "runtime.event", requestId: frame.requestId, event: { type: "context.required", pendingTurnIds, ...(historyOnly ? { historyOnly: true } : {}) } });
+                        send({ type: "runtime.event", requestId: frame.requestId, event: { type: "context.required", pendingTurnIds } });
                     }
                     catch {
                         abort();
@@ -240,7 +428,8 @@ async function connect(options) {
                 active.delete(requestId);
             }
             if ([...active.values()].some((entry) => entry.sessionId === frame.input.sessionId) || active.size >= 8) {
-                send({ type: "runtime.event", requestId: frame.requestId, event: { type: "turn.error", message: "Local Runtime is busy / 本地 Runtime 繁忙" } });
+                log("warn", "runtime.turn_rejected_busy", { activeExecutions: active.size }, context);
+                send({ type: "runtime.event", requestId: frame.requestId, event: { type: "turn.error", message: "Local Runtime is busy" } });
                 return;
             }
             seen.add(frame.requestId);
@@ -250,7 +439,15 @@ async function connect(options) {
                     seen.delete(oldest);
             }
             const controller = new AbortController();
-            const execution = { controller, sessionId: frame.input.sessionId, turnId: frame.input.turnId, harness: frame.input.harness, promise: Promise.resolve() };
+            const execution = {
+                controller,
+                sessionId: frame.input.sessionId,
+                turnId: frame.input.turnId,
+                harness: frame.input.harness,
+                requestId: frame.input.requestId,
+                traceContext: frame.input.traceContext,
+                promise: Promise.resolve(),
+            };
             active.set(frame.requestId, execution);
             const durableEvents = [];
             const emit = (value) => {
@@ -275,34 +472,46 @@ async function connect(options) {
                         active.delete(frame.requestId);
                         return;
                     }
-                    const run = () => (frame.input.harness === "pi" ? executePi : executeCodex)(frame.input, options.harnesses, options.cwd, options.store, emit, controller.signal);
+                    const run = () => (frame.input.harness === "pi" ? executePi : executeCodex)(frame.input, options.harnesses, options.cwd, options.store, emit, controller.signal, options.diagnostics, context);
                     // Preparation can request context, then fall back once from native archive to DB.
                     // These retries precede started(), so they never replay model or tool work.
-                    for (let attempt = 0;; attempt++) {
+                    for (let retry = 0;; retry += 1) {
                         try {
                             execution.result = await run();
                             break;
                         }
                         catch (error) {
-                            if (!(error instanceof ContextRequiredError) || attempt >= 2)
+                            if (!(error instanceof ContextRequiredError) || retry >= 2)
                                 throw error;
-                            frame.input.context = await requestContext(controller.signal, error.historyOnly);
+                            frame.input.context = await requestContext(controller.signal);
                         }
                     }
                     await options.store.recordResult(execution.result.state, frame.requestId, [...durableEvents, execution.result.event]);
                     emit(execution.result.event);
                 }
                 catch (error) {
+                    log("error", "runtime.turn_failed", {
+                        uncertain: Boolean(execution.result) || frame.resumeOnly === true || error instanceof ProcessCleanupUncertainError,
+                        error: serializeDiagnosticError(error),
+                    }, context);
                     try {
                         emit({ type: "turn.error", message: error instanceof Error ? error.message : String(error), uncertain: !!execution.result || frame.resumeOnly === true || error instanceof ProcessCleanupUncertainError });
                     }
-                    catch { /* Native files remain for recovery. */ }
+                    catch {
+                        // Native files remain for recovery.
+                    }
                     active.delete(frame.requestId);
                 }
             })();
-        })().catch((error) => { console.error("Runtime protocol error:", error); stop(); });
+        })().catch((error) => {
+            log("error", "runtime.protocol_error", { error: serializeDiagnosticError(error) }, { connectionId });
+            stop();
+        });
     });
-    readyTimer = setTimeout(() => socket.close(4408, "Runtime handshake timed out"), 15_000);
+    readyTimer = setTimeout(() => {
+        log("error", "runtime.handshake_timeout", { timeoutMs: 15_000 }, { connectionId });
+        socket.close(4408, "Runtime handshake timed out");
+    }, 15_000);
     if (options.signal.aborted)
         stop();
     try {

package/dist/runtime/diagnostics.d.ts

@@ -0,0 +1,104 @@
+import type { RuntimeTraceContext } from "@neta-art/cohub";
+export type RuntimeDiagnosticLevel = "debug" | "info" | "warn" | "error";
+export type RuntimeDiagnosticError = {
+    name?: string;
+    message: string;
+    code?: string;
+    status?: number;
+    syscall?: string;
+    hostname?: string;
+    causeCode?: string;
+    causeMessage?: string;
+    causeSyscall?: string;
+    causeHostname?: string;
+    traceContext?: RuntimeTraceContext;
+    stack?: string;
+};
+export type RuntimeDiagnostic = {
+    schemaVersion: 1;
+    sequence: number;
+    timestamp: string;
+    elapsedMs: number;
+    level: RuntimeDiagnosticLevel;
+    component: string;
+    event: string;
+    runtimeId: string;
+    connectionId?: string | null;
+    spaceId: string;
+    sessionId?: string | null;
+    turnId?: string | null;
+    harness?: "pi" | "codex" | null;
+    traceContext?: RuntimeTraceContext;
+    data?: Record<string, unknown>;
+    error?: RuntimeDiagnosticError;
+};
+export type RuntimeDiagnosticContext = {
+    component?: string;
+    connectionId?: string | null;
+    requestId?: string | null;
+    spaceId?: string;
+    sessionId?: string | null;
+    turnId?: string | null;
+    harness?: "pi" | "codex" | null;
+    traceContext?: RuntimeTraceContext;
+};
+export type RuntimeDiagnosticsOptions = {
+    root: string;
+    runtimeId?: string;
+    spaceId: string;
+    component?: string;
+    logFlushIntervalMs?: number;
+    maxLogFileBytes?: number;
+    maxTotalLogBytes?: number;
+};
+export type ReadRuntimeDiagnosticsOptions = {
+    limit?: number;
+};
+export declare const runtimeDiagnosticsDirectory: (root: string) => string;
+export declare const runtimeDiagnosticsPath: (root: string, runtimeId: string) => string;
+export declare function redactDiagnosticString(value: string): string;
+export declare function serializeDiagnosticError(error: unknown): RuntimeDiagnosticError;
+export declare class RuntimeDiagnostics {
+    readonly runtimeId: string;
+    readonly directory: string;
+    readonly logPath: string;
+    private readonly spaceId;
+    private readonly component;
+    private readonly logFlushIntervalMs;
+    private readonly maxLogFileBytes;
+    private readonly maxTotalLogBytes;
+    private readonly startedAtMs;
+    private readonly ready;
+    private logFile;
+    private logFileSize;
+    private logFileStartedAt;
+    private writeChain;
+    private writeBuffer;
+    private writeBufferBytes;
+    private droppedWriteEvents;
+    private writeTimer;
+    private sequence;
+    private closed;
+    constructor(options: RuntimeDiagnosticsOptions);
+    log(level: RuntimeDiagnosticLevel, event: string, data?: Record<string, unknown>, context?: RuntimeDiagnosticContext): void;
+    close(): Promise<void>;
+    private enqueueWrite;
+    private scheduleLogFlush;
+    private flushLogBuffer;
+    private writeEntries;
+    private ensureLogFile;
+    private rotateLog;
+    private enforceLogCapacity;
+}
+export declare class RuntimeDiagnosticReader {
+    readonly root: string;
+    private readonly files;
+    private readonly lastSequenceByRuntime;
+    private initialized;
+    constructor(root: string);
+    read(options?: {
+        limit?: number;
+    }): Promise<RuntimeDiagnostic[]>;
+}
+export declare function readRuntimeDiagnosticEvents(root: string, options?: ReadRuntimeDiagnosticsOptions): Promise<RuntimeDiagnostic[]>;
+export declare function isValidRuntimeTraceparent(value: string | null | undefined): value is string;