@moxt-ai/mobius 0.0.25 → 0.0.27

package/dist/run-daemon.js

@@ -1,7 +1,7 @@
 import { randomUUID } from "node:crypto";
 import { basename, join } from "node:path";
 import { styleText } from "node:util";
-import { AGENT_ACTIVITY_KIND, AGENT_INSTALL_CANCEL_KIND, AGENT_INSTALL_CANCELLED_KIND, AGENT_INSTALL_COMPLETED_KIND, AGENT_INSTALL_FAILED_KIND, AGENT_INSTALL_PROGRESS_KIND, AGENT_INSTALL_REQUEST_KIND, AGENT_INSTALL_STARTED_KIND, AGENT_MESSAGE_DELTA_KIND, BOT_BINDING_APPROVAL_REQUEST_KIND, BOT_BINDING_APPROVED_KIND, BOT_BINDING_UNAVAILABLE_KIND, BOT_SESSION_PROMPT_KIND, BotExecutionBindingUnavailableEvent, decodeAgentInstallCancelCommand, decodeAgentInstallRequestCommand, decodeBotExecutionBindingApprovalRequest, decodeBotExecutionBindingApprovedEvent, decodeBotExecutionBindingUnavailableEvent, decodeBotSessionPromptCommand, decodeMessageKind, decodeNativeSessionCancel, decodeNativeSessionCommand, decodeNativeSessionTurnForkCommand, decodeRuntimeHeartbeatAckEvent, decodeRuntimeLivenessReadyEvent, decodeSessionCancelCommand, decodeSessionConfigureCommand, decodeSessionElicitationRespondCommand, decodeSessionEventAcknowledgement, decodeSessionEventRejection, decodeSessionPermissionRespondCommand, decodeSessionPrepareCommand, decodeSessionPromptCommand, decodeWorkspaceListCommand, encodeAgentActivityEvent, encodeAgentContentEvent, encodeAgentInstallCancelledEvent, encodeAgentInstallCompletedEvent, encodeAgentInstallFailedEvent, encodeAgentInstallProgressEvent, encodeAgentInstallStartedEvent, encodeAgentMessageDeltaEvent, encodeBotExecutionBindingApprovedEvent, encodeBotExecutionBindingUnavailableEvent, encodeNativeSessionResult, encodeSessionCommandsUpdatedEvent, encodeSessionCompletedEvent, encodeSessionConfigUpdatedEvent, encodeSessionElicitationRequestedEvent, encodeSessionElicitationResolvedEvent, encodeSessionFailedEvent, encodeSessionInfoUpdatedEvent, encodeSessionOperationFailedEvent, encodeSessionPermissionRequestedEvent, encodeSessionPermissionResolvedEvent, encodeSessionReadyEvent, encodeSessionStartedEvent, encodeSessionUsageUpdatedEvent, encodeWorkspaceEntriesEvent, NATIVE_SESSION_CANCEL_KIND, NATIVE_SESSION_REQUEST_KIND, NATIVE_SESSION_TURN_FORK_KIND, OwnerBotReplyRoute, RUNTIME_HEARTBEAT_ACK_KIND, RUNTIME_LIVENESS_READY_KIND, SESSION_CANCEL_KIND, SESSION_COMPLETED_KIND, SESSION_CONFIGURE_KIND, SESSION_ELICITATION_RESPOND_KIND, SESSION_EVENT_ACK_KIND, SESSION_EVENT_REJECTED_KIND, SESSION_FAILED_KIND, SESSION_PERMISSION_RESPOND_KIND, SESSION_PREPARE_KIND, SESSION_PROMPT_KIND, SESSION_STARTED_KIND, SessionPromptDisposition, WORKSPACE_LIST_KIND, } from "@moxt-ai/mobius-protocol";
+import { AGENT_ACTIVITY_KIND, AGENT_INSTALL_CANCEL_KIND, AGENT_INSTALL_CANCELLED_KIND, AGENT_INSTALL_COMPLETED_KIND, AGENT_INSTALL_FAILED_KIND, AGENT_INSTALL_PROGRESS_KIND, AGENT_INSTALL_REQUEST_KIND, AGENT_INSTALL_STARTED_KIND, AGENT_MESSAGE_DELTA_KIND, BOT_BINDING_APPROVAL_REQUEST_KIND, BOT_BINDING_APPROVED_KIND, BOT_BINDING_UNAVAILABLE_KIND, BOT_SESSION_PROMPT_KIND, BotExecutionBindingUnavailableEvent, decodeAgentInstallCancelCommand, decodeAgentInstallRequestCommand, decodeBotExecutionBindingApprovalRequest, decodeBotExecutionBindingApprovedEvent, decodeBotExecutionBindingUnavailableEvent, decodeBotSessionPromptCommand, decodeMessageKind, decodeNativeSessionCancel, decodeNativeSessionCommand, decodeNativeSessionTurnForkCommand, decodeRuntimeDelivery, decodeRuntimeDeliveryCommitted, decodeRuntimeHeartbeatAckEvent, decodeRuntimeLivenessReadyEvent, decodeSessionCancelCommand, decodeSessionConfigureCommand, decodeSessionElicitationRespondCommand, decodeSessionEventAcknowledgement, decodeSessionEventEnvelope, decodeSessionEventRejection, decodeSessionPermissionRespondCommand, decodeSessionPrepareCommand, decodeSessionPromptCommand, decodeWorkspaceListCommand, encodeAgentActivityEvent, encodeAgentContentEvent, encodeAgentInstallCancelledEvent, encodeAgentInstallCompletedEvent, encodeAgentInstallFailedEvent, encodeAgentInstallProgressEvent, encodeAgentInstallStartedEvent, encodeAgentMessageDeltaEvent, encodeBotExecutionBindingApprovedEvent, encodeBotExecutionBindingUnavailableEvent, encodeNativeSessionResult, encodeRuntimeDeliveryReceived, encodeSessionCommandsUpdatedEvent, encodeSessionCompletedEvent, encodeSessionConfigUpdatedEvent, encodeSessionElicitationRequestedEvent, encodeSessionElicitationResolvedEvent, encodeSessionFailedEvent, encodeSessionInfoUpdatedEvent, encodeSessionOperationFailedEvent, encodeSessionPermissionRequestedEvent, encodeSessionPermissionResolvedEvent, encodeSessionReadyEvent, encodeSessionStartedEvent, encodeSessionUsageUpdatedEvent, encodeWorkspaceEntriesEvent, NATIVE_SESSION_CANCEL_KIND, NATIVE_SESSION_REQUEST_KIND, NATIVE_SESSION_TURN_FORK_KIND, OwnerBotReplyRoute, RUNTIME_DELIVERY_COMMITTED_KIND, RUNTIME_DELIVERY_KIND, RUNTIME_DELIVERY_RECEIVED_KIND, RUNTIME_HEARTBEAT_ACK_KIND, RUNTIME_LIVENESS_READY_KIND, SESSION_CANCEL_KIND, SESSION_COMPLETED_KIND, SESSION_CONFIGURE_KIND, SESSION_ELICITATION_RESPOND_KIND, SESSION_EVENT_ACK_KIND, SESSION_EVENT_REJECTED_KIND, SESSION_FAILED_KIND, SESSION_PERMISSION_RESPOND_KIND, SESSION_PREPARE_KIND, SESSION_PROMPT_KIND, SESSION_STARTED_KIND, SessionPromptDisposition, WORKSPACE_LIST_KIND, } from "@moxt-ai/mobius-protocol";
 import { fetch, WebSocket } from "undici";
 import { AcpExecutionError, AcpSessionManager } from "./acp/session-manager.js";
 import { SqliteAcpSessionStore } from "./acp/session-store.js";
@@ -20,12 +20,14 @@ import { NativeSessionService } from "./native-sessions/service.js";
 import { environmentProxyDispatcher } from "./network/environment-proxy.js";
 import { PromptArtifactError, resolvePromptArtifacts } from "./prompt-artifact-resolver.js";
 import { connectRelay } from "./relay-connection/connect.js";
+import { RelayConnectionDiagnostics } from "./relay-connection/diagnostics.js";
 import { RelayConnectionLiveness } from "./relay-connection/liveness.js";
-import { SessionEventOutbox } from "./relay-connection/session-event-outbox.js";
+import { relayHttpEndpoint, runtimeAuthorization } from "./relay-connection/runtime-authentication.js";
+import { RuntimeDeliveryInboxCapacityError, SessionEventOutbox } from "./relay-connection/session-event-outbox.js";
 import { systemDaemonTimer } from "./relay-connection/timer.js";
 import { readStateDirectory } from "./storage/state-directory.js";
 const RELAY_RECONNECT_DELAY_MILLISECONDS = 500;
-const RELAY_CONNECTION_REFRESH_MILLISECONDS = 20 * 60 * 1_000;
+const AGENT_ACTIVITY_COALESCING_MILLISECONDS = 100;
 const ACTIVE_TURN_RECONCILIATION_TIMEOUT_MILLISECONDS = 20_000;
 const ACTIVE_TURN_RECONCILIATION_CLOSE_CODE = 4009;
 function botIpcSocketPath() {
@@ -34,56 +36,6 @@ function botIpcSocketPath() {
     }
     return join("/tmp", `mobius-bot-${randomUUID()}.sock`);
 }
-class RelayConnectionRefreshState {
-}
-class ScheduledRelayConnectionRefreshState extends RelayConnectionRefreshState {
-}
-class DueRelayConnectionRefreshState extends RelayConnectionRefreshState {
-}
-class FinishedRelayConnectionRefreshState extends RelayConnectionRefreshState {
-}
-class RelayConnectionRefresh {
-    #isSafe;
-    #refresh;
-    #scheduledTask;
-    #state = new ScheduledRelayConnectionRefreshState();
-    constructor(timer, isSafe, refresh) {
-        this.#isSafe = isSafe;
-        this.#refresh = refresh;
-        this.#scheduledTask = timer.schedule(this.#request, RELAY_CONNECTION_REFRESH_MILLISECONDS);
-    }
-    afterTurnFinished = () => {
-        if (this.#state instanceof DueRelayConnectionRefreshState) {
-            this.#request();
-            return;
-        }
-        if (this.#state instanceof ScheduledRelayConnectionRefreshState ||
-            this.#state instanceof FinishedRelayConnectionRefreshState) {
-            return;
-        }
-        throw new Error("Unknown relay connection refresh state");
-    };
-    close = () => {
-        this.#scheduledTask.cancel();
-        this.#state = new FinishedRelayConnectionRefreshState();
-    };
-    #request = () => {
-        if (this.#state instanceof FinishedRelayConnectionRefreshState) {
-            return;
-        }
-        if (!(this.#state instanceof ScheduledRelayConnectionRefreshState) &&
-            !(this.#state instanceof DueRelayConnectionRefreshState)) {
-            throw new Error("Unknown relay connection refresh state");
-        }
-        if (!this.#isSafe()) {
-            this.#state = new DueRelayConnectionRefreshState();
-            return;
-        }
-        this.#state = new FinishedRelayConnectionRefreshState();
-        this.#scheduledTask.cancel();
-        this.#refresh();
-    };
-}
 class DaemonAcpObserver {
     #record;
     constructor(record) {
@@ -264,8 +216,7 @@ class ActiveTurnReconciliationError extends Error {
     }
 }
 function activeTurnReconciliationEndpoint(config, turn) {
-    const endpoint = new URL(config.relayUrl);
-    endpoint.protocol = endpoint.protocol === "wss:" ? "https:" : "http:";
+    const endpoint = relayHttpEndpoint(config.relayUrl);
     let channelPath = "";
     if (endpoint.pathname.endsWith("/daemon")) {
         channelPath = endpoint.pathname.slice(0, -"/daemon".length);
@@ -278,7 +229,6 @@ function activeTurnReconciliationEndpoint(config, turn) {
         channelPath = `${endpoint.pathname.slice(0, -runtimePath.length)}/channels/${encodeURIComponent(config.channelId)}`;
     }
     endpoint.pathname = `${channelPath}/sessions/${encodeURIComponent(turn.command.sessionId)}/attempts/${encodeURIComponent(turn.attemptId)}/reconciliation`;
-    endpoint.searchParams.delete("liveness");
     return endpoint;
 }
 async function reconcileActiveTurn(config, turn, signal) {
@@ -290,7 +240,7 @@ async function reconcileActiveTurn(config, turn, signal) {
         response = await fetch(activeTurnReconciliationEndpoint(config, turn), {
             body: JSON.stringify({ outcome: { kind: "running" } }),
             dispatcher: environmentProxyDispatcher,
-            headers: { "content-type": "application/json" },
+            headers: { ...runtimeAuthorization(config.relayUrl), "content-type": "application/json" },
             method: "POST",
             signal: requestSignal,
         });
@@ -335,7 +285,8 @@ class DaemonSessionEventConnectionState {
 }
 class DisconnectedDaemonSessionEventState extends DaemonSessionEventConnectionState {
     disconnect = () => this;
-    send = () => undefined;
+    matches = () => false;
+    send = () => false;
 }
 class ConnectedDaemonSessionEventState extends DaemonSessionEventConnectionState {
     #socket;
@@ -344,40 +295,207 @@ class ConnectedDaemonSessionEventState extends DaemonSessionEventConnectionState
         this.#socket = socket;
     }
     disconnect = (socket) => socket === this.#socket ? new DisconnectedDaemonSessionEventState() : this;
+    matches = (socket) => socket === this.#socket;
     send = (message) => {
-        if (this.#socket.readyState === WebSocket.OPEN) {
-            try {
-                this.#socket.send(message);
-            }
-            catch {
-                // The durable outbox retains the event for the replacement connection.
-            }
+        if (this.#socket.readyState !== WebSocket.OPEN) {
+            return false;
+        }
+        try {
+            this.#socket.send(message);
+            return true;
+        }
+        catch {
+            // The durable outbox retains the event for the replacement connection.
+            return false;
         }
     };
 }
+const SESSION_EVENT_REPLAY_WINDOW = 16;
+const SESSION_EVENT_SOURCE_REPLAY_WINDOW = 4;
+const SESSION_EVENT_REPLAY_BYTE_LIMIT = 32 * 1024 * 1024;
+class InFlightSessionEvent {
+    byteLength;
+    sourceKey;
+    constructor(sourceKey, byteLength) {
+        this.byteLength = byteLength;
+        this.sourceKey = sourceKey;
+    }
+}
+function sessionEventSourceKey(envelope) {
+    return envelope.source.kind === "attempt"
+        ? `attempt:${envelope.sessionId}:${envelope.source.attemptId}`
+        : `runtime:${envelope.sessionId}`;
+}
 class DaemonSessionEventPublisher {
     #outbox;
+    #inFlightEvents = new Map();
+    #inFlightBytes = 0;
     #connection = new DisconnectedDaemonSessionEventState();
     constructor(outbox) {
         this.#outbox = outbox;
     }
     connect = (socket) => {
-        const connection = new ConnectedDaemonSessionEventState(socket);
-        this.#connection = connection;
-        for (const pending of this.#outbox.pending()) {
-            connection.send(pending);
-        }
+        this.#connection = new ConnectedDaemonSessionEventState(socket);
+        this.#clearInFlight();
+        this.#publishNextBatch();
     };
     disconnect = (socket) => {
+        if (!this.#connection.matches(socket)) {
+            return;
+        }
         this.#connection = this.#connection.disconnect(socket);
+        this.#clearInFlight();
     };
     recordAttempt = (sessionId, payload, attemptId) => {
-        this.#connection.send(this.#outbox.enqueue(sessionId, payload, attemptId));
+        const kind = decodeMessageKind(payload);
+        if (kind === SESSION_COMPLETED_KIND || kind === SESSION_FAILED_KIND) {
+            this.#outbox.enqueueCritical(sessionId, payload, attemptId);
+        }
+        else {
+            this.#outbox.enqueue(sessionId, payload, attemptId);
+        }
+        this.#publishNextBatch();
     };
     recordRuntime = (sessionId, payload) => {
-        this.#connection.send(this.#outbox.enqueue(sessionId, payload));
+        this.#outbox.enqueue(sessionId, payload);
+        this.#publishNextBatch();
+    };
+    acknowledge = (acknowledgement) => {
+        if (!this.#outbox.acknowledge(acknowledgement)) {
+            return;
+        }
+        this.#removeInFlight(acknowledgement.eventId);
+        this.#publishNextBatch();
+    };
+    quarantine = (rejection) => {
+        const quarantined = this.#outbox.quarantine(rejection);
+        if (!quarantined) {
+            return false;
+        }
+        this.#removeInFlight(rejection.eventId);
+        this.#publishNextBatch();
+        return true;
+    };
+    #publishNextBatch = () => {
+        while (this.#inFlightEvents.size < SESSION_EVENT_REPLAY_WINDOW) {
+            const sourceCounts = new Map();
+            for (const inFlight of this.#inFlightEvents.values()) {
+                sourceCounts.set(inFlight.sourceKey, (sourceCounts.get(inFlight.sourceKey) ?? 0) + 1);
+            }
+            const saturatedSources = [...sourceCounts.entries()]
+                .filter(([, count]) => count >= SESSION_EVENT_SOURCE_REPLAY_WINDOW)
+                .map(([sourceKey]) => sourceKey);
+            const batch = this.#outbox.pendingSourceHeads(SESSION_EVENT_REPLAY_WINDOW - this.#inFlightEvents.size, [...this.#inFlightEvents.keys()], saturatedSources);
+            let sent = 0;
+            for (const encoded of batch) {
+                const envelope = decodeSessionEventEnvelope(encoded);
+                const sourceKey = sessionEventSourceKey(envelope);
+                const sourceCount = sourceCounts.get(sourceKey) ?? 0;
+                if (sourceCount >= SESSION_EVENT_SOURCE_REPLAY_WINDOW) {
+                    continue;
+                }
+                const byteLength = new TextEncoder().encode(encoded).byteLength;
+                if (this.#inFlightBytes + byteLength > SESSION_EVENT_REPLAY_BYTE_LIMIT) {
+                    continue;
+                }
+                if (!this.#connection.send(encoded)) {
+                    this.#clearInFlight();
+                    return;
+                }
+                this.#inFlightEvents.set(envelope.eventId, new InFlightSessionEvent(sourceKey, byteLength));
+                this.#inFlightBytes += byteLength;
+                sourceCounts.set(sourceKey, sourceCount + 1);
+                sent += 1;
+            }
+            if (sent === 0) {
+                return;
+            }
+        }
+    };
+    #removeInFlight = (eventId) => {
+        const inFlight = this.#inFlightEvents.get(eventId);
+        if (inFlight === undefined) {
+            return;
+        }
+        this.#inFlightEvents.delete(eventId);
+        this.#inFlightBytes -= inFlight.byteLength;
+    };
+    #clearInFlight = () => {
+        this.#inFlightEvents.clear();
+        this.#inFlightBytes = 0;
     };
 }
+class CoalescingDaemonActivityPublisher {
+    #pending = new Map();
+    #publish;
+    #publishedActivityIds = new Set();
+    #scheduled = new Map();
+    #timer;
+    constructor(timer, publish) {
+        this.#publish = publish;
+        this.#timer = timer;
+    }
+    flush = () => {
+        const activities = [...this.#pending.values()].sort((left, right) => left.sequence - right.sequence);
+        for (const task of this.#scheduled.values()) {
+            task.cancel();
+        }
+        this.#pending.clear();
+        this.#scheduled.clear();
+        for (const activity of activities) {
+            this.#publish(activity);
+        }
+    };
+    record = (activity) => {
+        if (activity.status !== "pending" && activity.status !== "running") {
+            this.#discardPending(activity.activityId);
+            this.#publishedActivityIds.add(activity.activityId);
+            this.#publish(activity);
+            return Promise.resolve();
+        }
+        if (!this.#publishedActivityIds.has(activity.activityId)) {
+            this.#publishedActivityIds.add(activity.activityId);
+            this.#publish(activity);
+            return Promise.resolve();
+        }
+        this.#pending.set(activity.activityId, activity);
+        if (!this.#scheduled.has(activity.activityId)) {
+            this.#scheduled.set(activity.activityId, this.#timer.schedule(() => this.#flushActivity(activity.activityId), AGENT_ACTIVITY_COALESCING_MILLISECONDS));
+        }
+        return Promise.resolve();
+    };
+    #discardPending = (activityId) => {
+        this.#scheduled.get(activityId)?.cancel();
+        this.#scheduled.delete(activityId);
+        this.#pending.delete(activityId);
+    };
+    #flushActivity = (activityId) => {
+        const activity = this.#pending.get(activityId);
+        this.#discardPending(activityId);
+        if (activity !== undefined) {
+            this.#publish(activity);
+        }
+    };
+}
+function encodeDaemonActivity(command, activity) {
+    return encodeAgentActivityEvent({
+        activityId: activity.activityId,
+        activityType: activity.activityType,
+        commandId: command.commandId,
+        content: activity.content,
+        detail: activity.detail,
+        input: activity.input,
+        kind: AGENT_ACTIVITY_KIND,
+        locations: activity.locations,
+        name: activity.name,
+        output: activity.output,
+        sequence: activity.sequence,
+        sessionId: command.sessionId,
+        status: activity.status,
+        title: activity.title,
+        toolKind: activity.toolKind,
+    });
+}
 function reportFailure(publisher, command, code, message, attemptId) {
     publisher.recordAttempt(command.sessionId, encodeSessionFailedEvent({
         code,
@@ -534,10 +652,16 @@ async function executeSessionConfigure(publisher, socket, command, config, direc
         }));
     }
     catch (error) {
+        if (signal.aborted) {
+            signal.throwIfAborted();
+        }
         reportOperationFailure(socket, command.requestId, command.sessionId, error);
     }
 }
-async function executePrompt(outbox, publisher, command, config, directories, audit, sessions, attemptId, signal) {
+async function executePrompt(outbox, publisher, command, config, directories, audit, sessions, attemptId, timer, signal) {
+    const activities = new CoalescingDaemonActivityPublisher(timer, (activity) => {
+        publisher.recordAttempt(command.sessionId, encodeDaemonActivity(command, activity), attemptId);
+    });
     try {
         const resolved = await resolveSessionDirectories(directories, config.bindingId, command.directoryId, command.additionalDirectoryIds, signal);
         const agent = config.catalog.find(command.agentId);
@@ -582,26 +706,7 @@ async function executePrompt(outbox, publisher, command, config, directories, au
                 cwd: resolved.cwd,
                 disposition: command.disposition,
                 images: attachments.images,
-                onActivity: (activity) => {
-                    publisher.recordAttempt(command.sessionId, encodeAgentActivityEvent({
-                        activityId: activity.activityId,
-                        activityType: activity.activityType,
-                        commandId: command.commandId,
-                        content: activity.content,
-                        detail: activity.detail,
-                        input: activity.input,
-                        kind: AGENT_ACTIVITY_KIND,
-                        locations: activity.locations,
-                        name: activity.name,
-                        output: activity.output,
-                        sequence: activity.sequence,
-                        sessionId: command.sessionId,
-                        status: activity.status,
-                        title: activity.title,
-                        toolKind: activity.toolKind,
-                    }), attemptId);
-                    return Promise.resolve();
-                },
+                onActivity: activities.record,
                 onText: (text, messageId) => {
                     publisher.recordAttempt(command.sessionId, encodeAgentMessageDeltaEvent({
                         commandId: command.commandId,
@@ -620,6 +725,7 @@ async function executePrompt(outbox, publisher, command, config, directories, au
             });
             stopReason = result.stopReason;
         }
+        activities.flush();
         publisher.recordAttempt(command.sessionId, encodeSessionCompletedEvent({
             commandId: command.commandId,
             kind: SESSION_COMPLETED_KIND,
@@ -634,6 +740,7 @@ async function executePrompt(outbox, publisher, command, config, directories, au
         });
     }
     catch (error) {
+        activities.flush();
         audit.record({
             commandId: command.commandId,
             event: "session_failed",
@@ -656,7 +763,10 @@ function botExecutionAgent(agent, contextHandle, socketPath) {
         MOBIUS_RELAY_URL: "",
     }, agent.adapterVersion, agent.agentVersion, agent.adapterIdentity, agent.agentIdentity, agent.promptTimeoutMilliseconds, agent.capabilities);
 }
-async function executeBotPrompt(outbox, publisher, command, config, directories, audit, sessions, grants, contexts, applicator, ipc, attemptId, signal) {
+async function executeBotPrompt(outbox, publisher, command, config, directories, audit, sessions, grants, contexts, applicator, ipc, attemptId, timer, signal) {
+    const activities = new CoalescingDaemonActivityPublisher(timer, (activity) => {
+        publisher.recordAttempt(command.sessionId, encodeDaemonActivity(command, activity), attemptId);
+    });
     try {
         const deadline = Date.parse(command.executionDeadlineAt);
         const remaining = deadline - Date.now();
@@ -713,26 +823,7 @@ async function executeBotPrompt(outbox, publisher, command, config, directories,
                 commandId: command.commandId,
                 cwd: resolved.cwd,
                 images: attachments.images,
-                onActivity: (activity) => {
-                    publisher.recordAttempt(command.sessionId, encodeAgentActivityEvent({
-                        activityId: activity.activityId,
-                        activityType: activity.activityType,
-                        commandId: command.commandId,
-                        content: activity.content,
-                        detail: activity.detail,
-                        input: activity.input,
-                        kind: AGENT_ACTIVITY_KIND,
-                        locations: activity.locations,
-                        name: activity.name,
-                        output: activity.output,
-                        sequence: activity.sequence,
-                        sessionId: command.sessionId,
-                        status: activity.status,
-                        title: activity.title,
-                        toolKind: activity.toolKind,
-                    }), attemptId);
-                    return Promise.resolve();
-                },
+                onActivity: activities.record,
                 onText: (text, messageId) => {
                     publisher.recordAttempt(command.sessionId, encodeAgentMessageDeltaEvent({
                         commandId: command.commandId,
@@ -748,6 +839,7 @@ async function executeBotPrompt(outbox, publisher, command, config, directories,
                 sessionId: command.sessionId,
                 signal: executionSignal,
             });
+            activities.flush();
             publisher.recordAttempt(command.sessionId, encodeSessionCompletedEvent({
                 commandId: command.commandId,
                 kind: SESSION_COMPLETED_KIND,
@@ -767,6 +859,7 @@ async function executeBotPrompt(outbox, publisher, command, config, directories,
         }
     }
     catch (error) {
+        activities.flush();
         const failure = Date.parse(command.executionDeadlineAt) <= Date.now()
             ? new LocalBotAuthorizationError("turn_expired", "The Bot execution deadline has expired", {
                 cause: error,
@@ -804,16 +897,16 @@ function waitForRelayRetry(signal, timer) {
         signal.addEventListener("abort", finish, { once: true });
     });
 }
-async function publishAgentCatalogUntilAvailable(config, signal, timer) {
+async function publishAgentCatalogUntilAvailable(config, diagnostics, signal, timer) {
     try {
         await publishAgentCatalog(config, signal);
         return;
     }
-    catch {
+    catch (error) {
         if (signal.aborted) {
             return;
         }
-        console.warn("Mobius daemon is waiting to publish its Agent catalog");
+        diagnostics.catalogFailed(error);
     }
     while (!signal.aborted) {
         await waitForRelayRetry(signal, timer);
@@ -828,11 +921,13 @@ async function publishAgentCatalogUntilAvailable(config, signal, timer) {
         }
     }
 }
-async function runRelayConnection(config, directories, audit, sessions, installations, installationObserver, nativeSessions, outbox, publisher, grants, contexts, applicator, ipc, activeTurns, activeAttemptIds, onConnected, signal, timer) {
+async function runRelayConnection(config, directories, audit, sessions, installations, installationObserver, nativeSessions, outbox, publisher, grants, contexts, applicator, ipc, activeTurns, activeAttemptIds, connectionEpoch, diagnostics, onConnected, signal, timer) {
     const relayUrl = new URL(config.relayUrl);
+    relayUrl.searchParams.set("deliveryReceipts", "1");
     relayUrl.searchParams.set("liveness", "1");
     const socket = await connectRelay(relayUrl, signal);
     const reconnectingTurns = [...activeTurns.values()];
+    diagnostics.connectionOpened(connectionEpoch);
     onConnected();
     const connectionController = new AbortController();
     const nativeRequests = new Map();
@@ -841,7 +936,6 @@ async function runRelayConnection(config, directories, audit, sessions, installa
         await new Promise((resolve, reject) => {
             let finished = false;
             const cleanup = () => {
-                connectionRefresh.close();
                 connectionLiveness.close();
                 signal.removeEventListener("abort", stop);
                 socket.removeEventListener("close", handleClose);
@@ -859,57 +953,251 @@ async function runRelayConnection(config, directories, audit, sessions, installa
                 cleanup();
                 resolve();
             };
-            const rejectConnection = () => {
+            const rejectConnection = (failure) => {
                 if (finished) {
                     return;
                 }
                 finished = true;
                 cleanup();
-                reject(new Error("Relay WebSocket failed"));
+                reject(failure);
             };
             const stop = () => {
+                diagnostics.connectionClosed(connectionEpoch, 1000, "Daemon stopped", "daemon_stop");
                 closeRelaySocket(socket, 1000, "Daemon stopped");
                 resolveConnection();
             };
-            const refreshConnection = () => {
-                closeRelaySocket(socket, 1000, "Refreshing idle relay connection");
-                resolveConnection();
+            const failAndClose = (failure, code, reason, source) => {
+                diagnostics.connectionFailed(connectionEpoch, failure, source);
+                diagnostics.connectionClosed(connectionEpoch, code, reason, source);
+                closeRelaySocket(socket, code, reason);
+                rejectConnection(failure);
+            };
+            const processingDeliveryIds = new Set();
+            const processDeliveredPrompt = async (delivery) => {
+                const command = decodeSessionPromptCommand(delivery.payload);
+                if (command.sessionId !== delivery.sessionId) {
+                    throw new Error("Runtime delivery Session identity does not match its payload");
+                }
+                const active = activeTurns.get(command.commandId);
+                if (active !== undefined) {
+                    await active.completion;
+                    return;
+                }
+                const receipt = outbox.claimExecution(command.commandId, command.sessionId, delivery.payload);
+                if (receipt.kind === "duplicate") {
+                    if (receipt.state === "running") {
+                        reportFailure(publisher, command, "agent_execution_interrupted", "The local Agent execution was interrupted before it completed", receipt.attemptId);
+                        outbox.transitionExecution(command.commandId, "failed");
+                        return;
+                    }
+                    if (receipt.state !== "dispatching") {
+                        return;
+                    }
+                }
+                const turn = new ActiveDaemonTurn(command, receipt.attemptId);
+                activeTurns.set(command.commandId, turn);
+                if (command.disposition !== SessionPromptDisposition.Steer) {
+                    activeAttemptIds.set(command.sessionId, receipt.attemptId);
+                }
+                try {
+                    await executePrompt(outbox, publisher, command, config, directories, audit, sessions, receipt.attemptId, timer, turn.signal);
+                }
+                finally {
+                    activeTurns.delete(command.commandId);
+                    if (activeAttemptIds.get(command.sessionId) === receipt.attemptId) {
+                        activeAttemptIds.delete(command.sessionId);
+                    }
+                    turn.finish();
+                }
+            };
+            const processDeliveredPayload = async (delivery) => {
+                const kind = decodeMessageKind(delivery.payload);
+                if (kind === SESSION_CANCEL_KIND) {
+                    const cancellation = decodeSessionCancelCommand(delivery.payload);
+                    if (cancellation.sessionId !== delivery.sessionId) {
+                        throw new Error("Runtime delivery Session identity does not match its payload");
+                    }
+                    const turn = activeTurns.get(cancellation.commandId);
+                    if (turn === undefined || !turn.matches(cancellation) || !turn.requestCancellation()) {
+                        return;
+                    }
+                    audit.record({
+                        commandId: cancellation.commandId,
+                        event: "session_cancellation_requested",
+                        sessionId: cancellation.sessionId,
+                    });
+                    await sessions.cancelPrompt(cancellation.sessionId);
+                    return;
+                }
+                if (kind === SESSION_CONFIGURE_KIND) {
+                    const command = decodeSessionConfigureCommand(delivery.payload);
+                    if (command.sessionId !== delivery.sessionId) {
+                        throw new Error("Runtime delivery Session identity does not match its payload");
+                    }
+                    await executeSessionConfigure(publisher, socket, command, config, directories, audit, sessions, connectionController.signal);
+                    return;
+                }
+                if (kind === SESSION_PERMISSION_RESPOND_KIND) {
+                    const command = decodeSessionPermissionRespondCommand(delivery.payload);
+                    if (command.sessionId !== delivery.sessionId) {
+                        throw new Error("Runtime delivery Session identity does not match its payload");
+                    }
+                    try {
+                        await sessions.respondPermission(command);
+                        audit.record({
+                            commandId: command.interactionId,
+                            event: "session_permission_resolved",
+                            sessionId: command.sessionId,
+                        });
+                    }
+                    catch (error) {
+                        reportOperationFailure(socket, command.interactionId, command.sessionId, error);
+                    }
+                    return;
+                }
+                if (kind === SESSION_ELICITATION_RESPOND_KIND) {
+                    const command = decodeSessionElicitationRespondCommand(delivery.payload);
+                    if (command.sessionId !== delivery.sessionId) {
+                        throw new Error("Runtime delivery Session identity does not match its payload");
+                    }
+                    try {
+                        await sessions.respondElicitation(command);
+                        audit.record({
+                            commandId: command.interactionId,
+                            event: "session_elicitation_resolved",
+                            sessionId: command.sessionId,
+                        });
+                    }
+                    catch (error) {
+                        reportOperationFailure(socket, command.interactionId, command.sessionId, error);
+                    }
+                    return;
+                }
+                if (kind === SESSION_PROMPT_KIND) {
+                    await processDeliveredPrompt(delivery);
+                    return;
+                }
+                throw new Error("Runtime delivery contains an unsupported protocol message");
+            };
+            const processRuntimeDelivery = (delivery, state) => {
+                if (state === "processed" || processingDeliveryIds.has(delivery.deliveryId)) {
+                    return;
+                }
+                processingDeliveryIds.add(delivery.deliveryId);
+                void processDeliveredPayload(delivery)
+                    .then(() => {
+                    outbox.markRuntimeDeliveryProcessed(delivery.deliveryId, delivery.sessionId);
+                })
+                    .catch((error) => {
+                    if (connectionController.signal.aborted) {
+                        return;
+                    }
+                    console.error("Mobius daemon could not process a durable Runtime delivery", error);
+                    failAndClose(new Error("Runtime delivery processing failed", { cause: error }), 4008, "Invalid Runtime delivery", "delivery");
+                })
+                    .finally(() => processingDeliveryIds.delete(delivery.deliveryId));
+            };
+            const receiveRuntimeDelivery = (delivery) => {
+                const payloadKind = decodeMessageKind(delivery.payload);
+                let payloadSessionId = "";
+                if (payloadKind === SESSION_CANCEL_KIND) {
+                    payloadSessionId = decodeSessionCancelCommand(delivery.payload).sessionId;
+                }
+                else if (payloadKind === SESSION_CONFIGURE_KIND) {
+                    payloadSessionId = decodeSessionConfigureCommand(delivery.payload).sessionId;
+                }
+                else if (payloadKind === SESSION_PERMISSION_RESPOND_KIND) {
+                    payloadSessionId = decodeSessionPermissionRespondCommand(delivery.payload).sessionId;
+                }
+                else if (payloadKind === SESSION_ELICITATION_RESPOND_KIND) {
+                    payloadSessionId = decodeSessionElicitationRespondCommand(delivery.payload).sessionId;
+                }
+                else if (payloadKind === SESSION_PROMPT_KIND) {
+                    payloadSessionId = decodeSessionPromptCommand(delivery.payload).sessionId;
+                }
+                else {
+                    throw new Error("Runtime delivery contains an unsupported protocol message");
+                }
+                if (payloadSessionId !== delivery.sessionId) {
+                    throw new Error("Runtime delivery Session identity does not match its payload");
+                }
+                const stored = outbox.claimRuntimeDelivery(delivery);
+                send(socket, encodeRuntimeDeliveryReceived({
+                    deliveryId: delivery.deliveryId,
+                    kind: RUNTIME_DELIVERY_RECEIVED_KIND,
+                    sessionId: delivery.sessionId,
+                }));
+                processRuntimeDelivery(delivery, stored.state);
+            };
+            const resumeRuntimeDeliveries = () => {
+                for (const stored of outbox.runtimeDeliveries()) {
+                    const delivery = {
+                        deliveryId: stored.deliveryId,
+                        kind: RUNTIME_DELIVERY_KIND,
+                        payload: stored.payload,
+                        sessionId: stored.sessionId,
+                    };
+                    if (stored.state !== "relay_committed") {
+                        send(socket, encodeRuntimeDeliveryReceived({
+                            deliveryId: stored.deliveryId,
+                            kind: RUNTIME_DELIVERY_RECEIVED_KIND,
+                            sessionId: stored.sessionId,
+                        }));
+                    }
+                    processRuntimeDelivery(delivery, stored.state);
+                }
             };
             const connectionLiveness = new RelayConnectionLiveness(timer, {
                 send: (message) => send(socket, message),
-                unresponsive: () => {
+                unresponsive: (heartbeatId, timeoutMilliseconds) => {
+                    diagnostics.heartbeatTimedOut(connectionEpoch, heartbeatId, timeoutMilliseconds);
+                    diagnostics.connectionClosed(connectionEpoch, 4001, "Relay liveness check timed out", "heartbeat_timeout");
                     closeRelaySocket(socket, 4001, "Relay liveness check timed out");
                     resolveConnection();
                 },
             });
-            const connectionRefresh = new RelayConnectionRefresh(timer, () => activeTurns.size === 0, refreshConnection);
             signal.addEventListener("abort", stop, { once: true });
-            const handleClose = () => {
+            const handleClose = (event) => {
+                diagnostics.connectionClosed(connectionEpoch, event.code, event.reason, "relay");
                 resolveConnection();
             };
-            const handleError = () => {
+            const handleError = (event) => {
                 if (signal.aborted) {
                     resolveConnection();
                     return;
                 }
-                rejectConnection();
+                const cause = Reflect.get(event, "error");
+                const failure = new Error("Relay WebSocket emitted an error", { cause: cause ?? event });
+                diagnostics.connectionFailed(connectionEpoch, failure, "socket_error");
+                diagnostics.connectionClosed(connectionEpoch, 1006, "WebSocket error", "socket_error");
+                rejectConnection(failure);
             };
             const handleMessage = (event) => {
                 const encoded = event.data;
                 if (typeof encoded !== "string") {
-                    closeRelaySocket(socket, 4003, "Text message required");
-                    rejectConnection();
+                    failAndClose(new Error("Relay sent a non-text message"), 4003, "Text message required", "protocol");
                     return;
                 }
                 try {
                     const kind = decodeMessageKind(encoded);
+                    if (kind === RUNTIME_DELIVERY_COMMITTED_KIND) {
+                        const committed = decodeRuntimeDeliveryCommitted(encoded);
+                        if (!outbox.markRuntimeDeliveryCommitted(committed.deliveryId, committed.sessionId)) {
+                            console.warn("Mobius daemon ignored confirmation for an unknown Runtime delivery");
+                        }
+                        return;
+                    }
+                    if (kind === RUNTIME_DELIVERY_KIND) {
+                        receiveRuntimeDelivery(decodeRuntimeDelivery(encoded));
+                        return;
+                    }
                     if (kind === SESSION_EVENT_ACK_KIND) {
-                        outbox.acknowledge(decodeSessionEventAcknowledgement(encoded));
+                        publisher.acknowledge(decodeSessionEventAcknowledgement(encoded));
                         return;
                     }
                     if (kind === SESSION_EVENT_REJECTED_KIND) {
                         const rejection = decodeSessionEventRejection(encoded);
-                        if (outbox.quarantine(rejection)) {
+                        if (publisher.quarantine(rejection)) {
                             audit.record({
                                 commandId: rejection.eventId,
                                 details: { code: rejection.code },
@@ -1205,7 +1493,7 @@ async function runRelayConnection(config, directories, audit, sessions, installa
                         const turn = new ActiveDaemonTurn(command, receipt.attemptId);
                         activeTurns.set(command.commandId, turn);
                         activeAttemptIds.set(command.sessionId, receipt.attemptId);
-                        void executeBotPrompt(outbox, publisher, command, config, directories, audit, sessions, grants, contexts, applicator, ipc, receipt.attemptId, turn.signal)
+                        void executeBotPrompt(outbox, publisher, command, config, directories, audit, sessions, grants, contexts, applicator, ipc, receipt.attemptId, timer, turn.signal)
                             .catch((error) => {
                             console.error("Mobius daemon Bot turn failed", error);
                         })
@@ -1215,7 +1503,6 @@ async function runRelayConnection(config, directories, audit, sessions, installa
                                 activeAttemptIds.delete(command.sessionId);
                             }
                             turn.finish();
-                            connectionRefresh.afterTurnFinished();
                         });
                         return;
                     }
@@ -1235,7 +1522,7 @@ async function runRelayConnection(config, directories, audit, sessions, installa
                     if (command.disposition !== SessionPromptDisposition.Steer) {
                         activeAttemptIds.set(command.sessionId, receipt.attemptId);
                     }
-                    void executePrompt(outbox, publisher, command, config, directories, audit, sessions, receipt.attemptId, turn.signal)
+                    void executePrompt(outbox, publisher, command, config, directories, audit, sessions, receipt.attemptId, timer, turn.signal)
                         .catch((error) => {
                         console.error("Mobius daemon turn failed", error);
                     })
@@ -1245,13 +1532,15 @@ async function runRelayConnection(config, directories, audit, sessions, installa
                             activeAttemptIds.delete(command.sessionId);
                         }
                         turn.finish();
-                        connectionRefresh.afterTurnFinished();
                     });
                 }
                 catch (error) {
                     console.error("Mobius daemon rejected a message", error);
-                    closeRelaySocket(socket, 4008, "Invalid protocol message");
-                    rejectConnection();
+                    if (error instanceof RuntimeDeliveryInboxCapacityError) {
+                        failAndClose(error, 4013, "Runtime delivery queue full", "delivery_backpressure");
+                        return;
+                    }
+                    failAndClose(new Error("Relay sent an invalid protocol message", { cause: error }), 4008, "Invalid protocol message", "protocol");
                 }
             };
             socket.addEventListener("close", handleClose);
@@ -1262,14 +1551,13 @@ async function runRelayConnection(config, directories, audit, sessions, installa
                     return;
                 }
                 publisher.connect(socket);
-                void Promise.all(reconnectingTurns.map((turn) => turn.completion)).then(connectionRefresh.afterTurnFinished);
+                resumeRuntimeDeliveries();
             }, (error) => {
                 if (finished) {
                     return;
                 }
                 console.error("Mobius daemon could not reconcile active Agent turns", error);
-                closeRelaySocket(socket, ACTIVE_TURN_RECONCILIATION_CLOSE_CODE, "Active Agent turn reconciliation failed");
-                rejectConnection();
+                failAndClose(new Error("Active Agent turn reconciliation failed", { cause: error }), ACTIVE_TURN_RECONCILIATION_CLOSE_CODE, "Active Agent turn reconciliation failed", "reconciliation");
             });
             if (signal.aborted) {
                 stop();
@@ -1313,9 +1601,10 @@ export async function runDaemon(config, audit, signal, timer = systemDaemonTimer
     const sessions = new AcpSessionManager(new DaemonAcpObserver(recordSessionEvent), sessionStore);
     const directories = new WorkspaceDirectoryService(config.bindingId, config.defaultWorkspace);
     const installationObserver = new DaemonInstallationObserver();
+    const diagnostics = new RelayConnectionDiagnostics(audit, config.channelId, config.relayUrl);
     const nativeSessions = new NativeSessionService(config, directories, sessions, audit);
     const installations = new AgentInstallationManager(specifications, installationExecutor, installationObserver, installationAudit);
-    let retryAnnounced = false;
+    let connectionEpoch = 0;
     let connectionAnnounced = false;
     const announceConnection = () => {
         if (connectionAnnounced) {
@@ -1326,22 +1615,19 @@ export async function runDaemon(config, audit, signal, timer = systemDaemonTimer
         process.stdout.write("Mobius daemon connected to the relay.\n\n");
         process.stdout.write(`${styleText(["bold", "yellow"], `Reconnect this channel later:\n  ${reconnectCommand}`)}\n\n`);
     };
-    const catalogPublication = publishAgentCatalogUntilAvailable(config, signal, timer);
+    const catalogPublication = publishAgentCatalogUntilAvailable(config, diagnostics, signal, timer);
     try {
         await ipc.listen(signal);
         while (!signal.aborted) {
+            connectionEpoch += 1;
             try {
-                await runRelayConnection(config, directories, audit, sessions, installations, installationObserver, nativeSessions, outbox, publisher, grants, contexts, applicator, ipc, activeTurns, activeAttemptIds, announceConnection, signal, timer);
-                retryAnnounced = false;
+                await runRelayConnection(config, directories, audit, sessions, installations, installationObserver, nativeSessions, outbox, publisher, grants, contexts, applicator, ipc, activeTurns, activeAttemptIds, connectionEpoch, diagnostics, announceConnection, signal, timer);
             }
-            catch {
+            catch (error) {
                 if (signal.aborted) {
                     return;
                 }
-                if (!retryAnnounced) {
-                    console.warn("Mobius daemon is waiting for the relay");
-                    retryAnnounced = true;
-                }
+                diagnostics.connectionFailed(connectionEpoch, error, "connection");
             }
             await waitForRelayRetry(signal, timer);
         }