@masons/runtime-broker 0.2.1 → 0.2.5

package/dist/broker/broker-daemon.js

@@ -1,6 +1,7 @@
 import { spawn as spawnChild } from "node:child_process";
 import { randomUUID } from "node:crypto";
 import { basename } from "node:path";
+import { ConnectorClientUnavailableError, ConnectorSendAckTimeoutError, } from "../connector-client.js";
 import { createControlEventDispatcher, } from "./control-event-dispatcher.js";
 import { readDeliveryCursorFile, writeDeliveryCursorFile, } from "./delivery-cursor-file.js";
 import { deleteDiscoveryFile, mintBearerToken, writeDiscoveryFile, } from "./discovery-file.js";
@@ -22,6 +23,29 @@ import { CONTENT_PREVIEW_MAX_CODEPOINTS, truncateContentPreview, } from "./undis
 import { createUndispatchedChangedEmitter, postUndispatchedChangedViaPort, } from "./undispatched-emitter.js";
 import { createUndispatchedInbox, } from "./undispatched-inbox.js";
 const REMOTE_SPAWN_CAPABILITY = "remote_spawn_v1";
+const DEFAULT_RUNTIME_ASSIGNMENT_ACCEPTANCE_TIMEOUT_MS = 30_000;
+const RUNTIME_ASSIGNMENT_REPLY_RECORD_TTL_MS = 24 * 60 * 60 * 1000;
+const RUNTIME_PROCESSING_RETRY_INITIAL_MS = 1_000;
+const RUNTIME_PROCESSING_RETRY_MAX_MS = 30_000;
+function isBackgroundExchangeMode(value) {
+    return value === "resident_endpoint" || value === "adapter_managed_turn";
+}
+function buildLocalDispatchTargetRef(targetEndpointId, target) {
+    if (target?.background_exchange_mode === "adapter_managed_turn" &&
+        target.runtime_session_id) {
+        return {
+            kind: "projection_endpoint",
+            projection_endpoint_id: targetEndpointId,
+            runtime_session_id: target.runtime_session_id,
+            background_exchange_mode: "adapter_managed_turn",
+        };
+    }
+    return {
+        kind: "projection_endpoint",
+        projection_endpoint_id: targetEndpointId,
+        background_exchange_mode: "resident_endpoint",
+    };
+}
 function derivePublicDisplayLabel(body) {
     const parts = [
         body.kind,
@@ -39,6 +63,8 @@ export async function startBrokerDaemon(opts) {
     const { paths, asNodeId: _asNodeIdReservedForA3, connector, apiPort, logger, closedEndpointLookup, livenessProbe = defaultIsPluginAlive, spawnDriverRegistry, spawnFn = spawnChild, servicesEventClientFactory, } = opts;
     const graceMs = opts.graceMs ?? DEFAULT_GRACE_MS;
     const presenceGraceMs = opts.presenceGraceMs;
+    const runtimeAssignmentAcceptanceTimeoutMs = opts.runtimeAssignmentAcceptanceTimeoutMs ??
+        DEFAULT_RUNTIME_ASSIGNMENT_ACCEPTANCE_TIMEOUT_MS;
     const postControlAck = opts.postControlAck ??
         (async () => {
         });
@@ -63,6 +89,10 @@ export async function startBrokerDaemon(opts) {
     const reconnectingBuffers = createReconnectingBufferManager(paths.buffersDir);
     const graceTimers = createGraceTimerManager();
     const controlEventDispatcher = createControlEventDispatcher();
+    const runtimeAssignments = new Map();
+    const runtimeAssignmentByEndpointCorrelation = new Map();
+    const runtimeAssignmentReplyRecords = new Map();
+    const runtimeAssignmentReplyInFlight = new Map();
     const spawnCorrelation = createSpawnCorrelationManager({
         timeoutMs: opts.spawnCorrelationTimeoutMs,
     });
@@ -121,6 +151,91 @@ export async function startBrokerDaemon(opts) {
         backoffMaxMs: opts.runtimeInboundRoutedBackoffMaxMs,
         maxRetries: opts.runtimeInboundRoutedMaxRetries,
     });
+    const runtimeAssignmentKey = (sourceMessageId, assignmentId) => `${sourceMessageId}\u0000${assignmentId}`;
+    const runtimeAssignmentReplyKey = (sourceMessageId, assignmentId, replyRequestId) => `${sourceMessageId}\u0000${assignmentId}\u0000${replyRequestId}`;
+    const runtimeAssignmentReplyFingerprint = (content, contentType) => JSON.stringify([content, contentType ?? "text"]);
+    const pruneRuntimeAssignmentReplyRecords = (now = Date.now()) => {
+        for (const [key, record] of runtimeAssignmentReplyRecords.entries()) {
+            const [sourceMessageId, assignmentId] = key.split("\u0000");
+            if (sourceMessageId !== undefined &&
+                assignmentId !== undefined &&
+                runtimeAssignments.has(runtimeAssignmentKey(sourceMessageId, assignmentId))) {
+                continue;
+            }
+            if (record.expiresAt <= now) {
+                runtimeAssignmentReplyRecords.delete(key);
+            }
+        }
+    };
+    const endpointCorrelationKey = (endpointId, correlationId) => `${endpointId}\u0000${correlationId}`;
+    const isBackgroundExchangeTarget = (entry) => Boolean(entry.runtime_capabilities?.includes("background_exchange_v1") &&
+        isBackgroundExchangeMode(entry.background_exchange_mode));
+    const rememberRuntimeAssignmentCorrelation = (endpointId, correlationId, assignmentKey) => {
+        if (typeof correlationId !== "string" || correlationId.length === 0) {
+            return;
+        }
+        runtimeAssignmentByEndpointCorrelation.set(endpointCorrelationKey(endpointId, correlationId), assignmentKey);
+    };
+    const forgetRuntimeAssignment = (key) => {
+        const assignment = runtimeAssignments.get(key);
+        if (assignment?.acceptanceTimer) {
+            clearTimeout(assignment.acceptanceTimer);
+        }
+        if (assignment?.replyEmittedRetryTimer) {
+            clearTimeout(assignment.replyEmittedRetryTimer);
+        }
+        runtimeAssignments.delete(key);
+        for (const [correlationKey, assignmentKey] of Array.from(runtimeAssignmentByEndpointCorrelation.entries())) {
+            if (assignmentKey === key) {
+                runtimeAssignmentByEndpointCorrelation.delete(correlationKey);
+            }
+        }
+        for (const replyKey of Array.from(runtimeAssignmentReplyInFlight.keys())) {
+            if (replyKey.startsWith(`${key}\u0000`)) {
+                runtimeAssignmentReplyInFlight.delete(replyKey);
+            }
+        }
+    };
+    const scheduleAcceptanceTimeout = (key) => {
+        if (runtimeAssignmentAcceptanceTimeoutMs <= 0)
+            return;
+        const assignment = runtimeAssignments.get(key);
+        if (!assignment || assignment.accepted || assignment.acceptanceTimer) {
+            return;
+        }
+        const timer = setTimeout(() => {
+            const current = runtimeAssignments.get(key);
+            if (!current || current.accepted)
+                return;
+            logger.warn("runtime_assignment_acceptance_timeout", {
+                undispatched_id: current.context.undispatchedId,
+                source_message_id: current.context.sourceMessageId,
+                target_endpoint_id: current.context.targetRef.projection_endpoint_id,
+            });
+            void restorePendingAssignmentsForEndpoint(current.context.targetRef.projection_endpoint_id, "adapter_acceptance_timeout");
+        }, runtimeAssignmentAcceptanceTimeoutMs);
+        timer.unref?.();
+        assignment.acceptanceTimer = timer;
+    };
+    const emitRuntimeProcessingState = async (event) => {
+        if (!apiPort.emitRuntimeProcessingState) {
+            return {
+                ok: false,
+                terminal: true,
+                detail: "runtime_processing_state_port_not_configured",
+            };
+        }
+        try {
+            return await apiPort.emitRuntimeProcessingState(event);
+        }
+        catch (err) {
+            return {
+                ok: false,
+                terminal: false,
+                detail: err instanceof Error ? err.message : String(err),
+            };
+        }
+    };
     const directPostUndispatched = async (event, opts = {}) => {
         let outcome;
         try {
@@ -195,6 +310,153 @@ export async function startBrokerDaemon(opts) {
             return;
         replyCorrelationCache.record(endpoint_id, c);
     };
+    const restorePendingAssignmentsForEndpoint = (endpoint_id, reason) => {
+        const tasks = [];
+        for (const [key, assignment] of Array.from(runtimeAssignments.entries())) {
+            if (assignment.context.targetRef.projection_endpoint_id !== endpoint_id ||
+                assignment.accepted) {
+                continue;
+            }
+            if (assignment.acceptanceTimer) {
+                clearTimeout(assignment.acceptanceTimer);
+                assignment.acceptanceTimer = undefined;
+            }
+            const readded = undispatchedInbox.tryAdd(assignment.message);
+            if (!readded) {
+                logger.error("runtime_assignment_reinsert_failed_at_capacity", {
+                    undispatched_id: assignment.context.undispatchedId,
+                    source_message_id: assignment.context.sourceMessageId,
+                    target_endpoint_id: endpoint_id,
+                    inbox_size: undispatchedInbox.size(),
+                    inbox_capacity: undispatchedInbox.capacity(),
+                });
+                scheduleAcceptanceTimeout(key);
+                continue;
+            }
+            const task = emitRuntimeProcessingState({
+                type: "runtime_processing_state",
+                version: 1,
+                source_message_id: assignment.context.sourceMessageId,
+                assignment_id: assignment.context.assignmentId,
+                state_event_id: `${assignment.context.assignmentId}:target_lost`,
+                state_sequence: 2,
+                target_ref: assignment.context.targetRef,
+                state: "target_lost_before_acceptance",
+                occurred_at: assignment.assignedAt + 1,
+                undispatched_id: assignment.context.undispatchedId,
+                reason_code: reason,
+            }).then((outcome) => {
+                if (outcome.ok) {
+                    forgetRuntimeAssignment(key);
+                    return;
+                }
+                logger.warn("runtime_assignment_target_lost_emit_failed", {
+                    undispatched_id: assignment.context.undispatchedId,
+                    source_message_id: assignment.context.sourceMessageId,
+                    target_endpoint_id: endpoint_id,
+                    status: outcome.status,
+                    terminal: outcome.terminal,
+                    detail: outcome.detail,
+                });
+                scheduleAcceptanceTimeout(key);
+            });
+            tasks.push(task);
+        }
+        return Promise.allSettled(tasks).then(() => undefined);
+    };
+    const restorePendingAssignmentBeforeAcceptance = (key, assignment, reason) => {
+        if (assignment.acceptanceTimer) {
+            clearTimeout(assignment.acceptanceTimer);
+            assignment.acceptanceTimer = undefined;
+        }
+        const readded = undispatchedInbox.tryAdd(assignment.message);
+        if (!readded) {
+            logger.error("runtime_assignment_reinsert_failed_at_capacity", {
+                undispatched_id: assignment.context.undispatchedId,
+                source_message_id: assignment.context.sourceMessageId,
+                target_endpoint_id: assignment.context.targetRef.projection_endpoint_id,
+                inbox_size: undispatchedInbox.size(),
+                inbox_capacity: undispatchedInbox.capacity(),
+                reason,
+            });
+            scheduleAcceptanceTimeout(key);
+            return false;
+        }
+        forgetRuntimeAssignment(key);
+        return true;
+    };
+    const emitReplyEmittedWithRetry = (assignmentKey, event, attempt = 0) => {
+        const assignment = runtimeAssignments.get(assignmentKey);
+        if (!assignment)
+            return;
+        assignment.replyEmittedEvent = event;
+        if (assignment.replyEmittedRetryTimer) {
+            clearTimeout(assignment.replyEmittedRetryTimer);
+            assignment.replyEmittedRetryTimer = undefined;
+        }
+        void emitRuntimeProcessingState(event).then((outcome) => {
+            const current = runtimeAssignments.get(assignmentKey);
+            if (!current)
+                return;
+            if (outcome.ok) {
+                forgetRuntimeAssignment(assignmentKey);
+                return;
+            }
+            logger.warn("runtime_reply_emitted_projection_failed", {
+                source_message_id: current.context.sourceMessageId,
+                assignment_id: current.context.assignmentId,
+                status: outcome.status,
+                terminal: outcome.terminal,
+                detail: outcome.detail,
+            });
+            if (outcome.terminal) {
+                forgetRuntimeAssignment(assignmentKey);
+                return;
+            }
+            const delay = Math.min(RUNTIME_PROCESSING_RETRY_INITIAL_MS * 2 ** attempt, RUNTIME_PROCESSING_RETRY_MAX_MS);
+            const timer = setTimeout(() => {
+                emitReplyEmittedWithRetry(assignmentKey, event, attempt + 1);
+            }, delay);
+            timer.unref?.();
+            current.replyEmittedRetryTimer = timer;
+        });
+    };
+    const flushRuntimeAssignmentsOnShutdown = async () => {
+        const endpointIds = new Set();
+        for (const assignment of runtimeAssignments.values()) {
+            endpointIds.add(assignment.context.targetRef.projection_endpoint_id);
+        }
+        for (const [key, assignment] of Array.from(runtimeAssignments.entries())) {
+            if (assignment.replyEmittedEvent) {
+                const outcome = await emitRuntimeProcessingState(assignment.replyEmittedEvent);
+                if (outcome.ok) {
+                    forgetRuntimeAssignment(key);
+                    continue;
+                }
+                logger.warn("runtime_reply_emitted_shutdown_flush_failed", {
+                    source_message_id: assignment.context.sourceMessageId,
+                    assignment_id: assignment.context.assignmentId,
+                    status: outcome.status,
+                    terminal: outcome.terminal,
+                    detail: outcome.detail,
+                });
+                if (outcome.terminal) {
+                    forgetRuntimeAssignment(key);
+                }
+            }
+        }
+        for (const endpointId of endpointIds) {
+            await restorePendingAssignmentsForEndpoint(endpointId, "broker_shutdown");
+        }
+        if (runtimeAssignments.size > 0) {
+            logger.warn("runtime_assignments_dropped_on_shutdown", {
+                size: runtimeAssignments.size,
+            });
+        }
+        for (const key of Array.from(runtimeAssignments.keys())) {
+            forgetRuntimeAssignment(key);
+        }
+    };
     const channels = new Map();
     let networkPresence = "offline";
     let presenceGraceTimer;
@@ -357,11 +619,12 @@ export async function startBrokerDaemon(opts) {
                     break;
                 case "drain_buffer": {
                     const buf = reconnectingBuffers.forEndpoint(endpoint_id);
-                    const drained = buf.drain();
+                    const drained = buf.read();
                     const e = registry.get(endpoint_id);
+                    let allPushed = true;
                     if (e) {
                         for (const msg of drained) {
-                            pushToPlugin(e.ipc_ws, {
+                            const pushed = pushToPlugin(e.ipc_ws, {
                                 event: "message_received",
                                 from: msg.envelope.from,
                                 content: msg.envelope.content,
@@ -370,15 +633,32 @@ export async function startBrokerDaemon(opts) {
                                 ...(msg.envelope.sourceMessageId !== undefined && {
                                     sourceMessageId: msg.envelope.sourceMessageId,
                                 }),
+                                ...(msg.envelope.runtimeAssignment !== undefined && {
+                                    runtimeAssignment: msg.envelope.runtimeAssignment,
+                                }),
                             });
+                            if (!pushed) {
+                                allPushed = false;
+                                break;
+                            }
                         }
                     }
+                    if (allPushed) {
+                        buf.delete();
+                    }
+                    else {
+                        logger.warn("reconnecting_buffer_drain_failed", {
+                            endpoint_id,
+                            buffered_count: drained.length,
+                        });
+                    }
                     break;
                 }
                 case "delete_buffer":
                     reconnectingBuffers.cleanup(endpoint_id);
                     break;
                 case "remove_from_registry":
+                    void restorePendingAssignmentsForEndpoint(endpoint_id, "endpoint_lost");
                     clearHeartbeat(endpoint_id);
                     registry.unregister(endpoint_id);
                     replyCorrelationCache.forgetEndpoint(endpoint_id);
@@ -389,20 +669,49 @@ export async function startBrokerDaemon(opts) {
         }
     };
     controlEventDispatcher.on("dispatch_undispatched", async (event) => {
-        const taken = undispatchedInbox.take(event.undispatched_id);
+        const targetEndpointId = event.target_ref?.projection_endpoint_id ?? event.target_endpoint_id;
+        if (!targetEndpointId) {
+            return {
+                ok: false,
+                detail: "dispatch target_ref missing",
+            };
+        }
+        const assignmentKey = runtimeAssignmentKey(event.source_message_id, event.idempotency_key);
+        const existingAssignment = runtimeAssignments.get(assignmentKey);
+        const taken = undispatchedInbox.take(event.undispatched_id) ??
+            (!existingAssignment?.accepted &&
+                existingAssignment?.context.undispatchedId === event.undispatched_id &&
+                existingAssignment.context.targetRef.projection_endpoint_id ===
+                    targetEndpointId
+                ? existingAssignment.message
+                : undefined);
         if (!taken) {
             return {
                 ok: false,
                 detail: `unknown undispatched_id: ${event.undispatched_id}`,
             };
         }
-        const target = registry.get(event.target_endpoint_id);
+        if (taken.source_message_id === undefined) {
+            undispatchedInbox.tryAdd(taken);
+            return {
+                ok: false,
+                detail: "dispatch source_message_id missing from inbox row",
+            };
+        }
+        if (event.source_message_id !== taken.source_message_id) {
+            undispatchedInbox.tryAdd(taken);
+            return {
+                ok: false,
+                detail: `source_message_id mismatch: ${event.source_message_id}`,
+            };
+        }
+        const target = registry.get(targetEndpointId);
         if (!target) {
             const readded = undispatchedInbox.tryAdd(taken);
             if (!readded) {
                 logger.error("dispatch_reinsert_failed_at_capacity", {
                     undispatched_id: event.undispatched_id,
-                    target_endpoint_id: event.target_endpoint_id,
+                    target_endpoint_id: targetEndpointId,
                     inbox_size: undispatchedInbox.size(),
                     inbox_capacity: undispatchedInbox.capacity(),
                 });
@@ -419,9 +728,94 @@ export async function startBrokerDaemon(opts) {
             }
             return {
                 ok: false,
-                detail: `target endpoint not found: ${event.target_endpoint_id}`,
+                detail: `target endpoint not found: ${targetEndpointId}`,
+            };
+        }
+        let targetRef = event.target_ref;
+        if (!targetRef) {
+            if (target.background_exchange_mode === "resident_endpoint") {
+                targetRef = {
+                    kind: "projection_endpoint",
+                    projection_endpoint_id: targetEndpointId,
+                    background_exchange_mode: "resident_endpoint",
+                };
+            }
+            else if (target.background_exchange_mode === "adapter_managed_turn" &&
+                target.runtime_session_id) {
+                targetRef = {
+                    kind: "projection_endpoint",
+                    projection_endpoint_id: targetEndpointId,
+                    runtime_session_id: target.runtime_session_id,
+                    background_exchange_mode: "adapter_managed_turn",
+                };
+            }
+        }
+        if (!targetRef ||
+            !target.runtime_capabilities?.includes("background_exchange_v1") ||
+            target.background_exchange_mode !==
+                targetRef.background_exchange_mode ||
+            (targetRef.background_exchange_mode === "adapter_managed_turn" &&
+                target.runtime_session_id !== targetRef.runtime_session_id)) {
+            undispatchedInbox.tryAdd(taken);
+            return {
+                ok: false,
+                detail: `target endpoint is not background-exchange capable: ${targetEndpointId}`,
             };
         }
+        const assignedAt = event.emitted_at;
+        const processingEvent = {
+            type: "runtime_processing_state",
+            version: 1,
+            source_message_id: event.source_message_id,
+            assignment_id: event.idempotency_key,
+            state_event_id: `${event.idempotency_key}:assigned`,
+            state_sequence: 1,
+            target_ref: targetRef,
+            state: "assigned_to_runtime_target",
+            occurred_at: assignedAt,
+            undispatched_id: event.undispatched_id,
+        };
+        const processingResult = await emitRuntimeProcessingState(processingEvent);
+        if (!processingResult.ok) {
+            undispatchedInbox.tryAdd(taken);
+            logger.warn("runtime_processing_state_emit_failed", {
+                undispatched_id: event.undispatched_id,
+                source_message_id: event.source_message_id,
+                target_endpoint_id: targetEndpointId,
+                status: processingResult.status,
+                terminal: processingResult.terminal,
+                detail: processingResult.detail,
+            });
+            return {
+                ok: false,
+                detail: processingResult.detail ??
+                    "runtime_processing_state_assignment_failed",
+            };
+        }
+        const runtimeAssignment = {
+            undispatchedId: event.undispatched_id,
+            sourceMessageId: event.source_message_id,
+            assignmentId: event.idempotency_key,
+            targetRef,
+            deliveryIntent: "external_handoff",
+            ...(typeof taken.metadata.correlation_id === "string" && {
+                replyCorrelationId: taken.metadata.correlation_id,
+            }),
+        };
+        const storedAssignment = existingAssignment ?? {
+            context: runtimeAssignment,
+            message: taken,
+            accepted: false,
+            assignedAt,
+            lastStateSequence: processingEvent.state_sequence,
+        };
+        storedAssignment.context = runtimeAssignment;
+        storedAssignment.message = taken;
+        storedAssignment.assignedAt = assignedAt;
+        storedAssignment.lastStateSequence = Math.max(storedAssignment.lastStateSequence, processingEvent.state_sequence);
+        runtimeAssignments.set(assignmentKey, storedAssignment);
+        scheduleAcceptanceTimeout(assignmentKey);
+        rememberRuntimeAssignmentCorrelation(target.endpoint_id, taken.metadata.correlation_id, assignmentKey);
         const stamped = {
             event: "message_received",
             from: taken.sender_address,
@@ -430,47 +824,50 @@ export async function startBrokerDaemon(opts) {
             metadata: {
                 ...taken.metadata,
                 dispatched_from: event.undispatched_id,
-                dispatched_by: "passport",
+                delivery_intent: "external_handoff",
             },
-            ...(taken.source_message_id !== undefined && {
-                sourceMessageId: taken.source_message_id,
-            }),
+            sourceMessageId: taken.source_message_id,
+            runtimeAssignment,
         };
         recordInboundCorrelation(target.endpoint_id, stamped.metadata);
         if (target.state === "active") {
-            pushToPlugin(target.ipc_ws, stamped);
+            if (!pushToPlugin(target.ipc_ws, stamped)) {
+                await restorePendingAssignmentsForEndpoint(target.endpoint_id, "ipc_push_failed");
+                return {
+                    ok: false,
+                    detail: `target endpoint not writable: ${targetEndpointId}`,
+                };
+            }
         }
         else {
-            const buf = reconnectingBuffers.forEndpoint(target.endpoint_id);
-            buf.append({
-                id: randomUUID(),
-                arrived_at: new Date().toISOString(),
-                envelope: {
-                    from: taken.sender_address,
-                    content: taken.content,
-                    contentType: taken.content_type,
-                    metadata: stamped.metadata,
-                    ...(stamped.sourceMessageId !== undefined && {
+            try {
+                const buf = reconnectingBuffers.forEndpoint(target.endpoint_id);
+                buf.append({
+                    id: randomUUID(),
+                    arrived_at: new Date().toISOString(),
+                    envelope: {
+                        from: taken.sender_address,
+                        content: taken.content,
+                        contentType: taken.content_type,
+                        metadata: stamped.metadata,
                         sourceMessageId: stamped.sourceMessageId,
-                    }),
-                },
-            });
+                        runtimeAssignment,
+                    },
+                });
+            }
+            catch (err) {
+                await restorePendingAssignmentsForEndpoint(target.endpoint_id, "reconnecting_buffer_failed");
+                return {
+                    ok: false,
+                    detail: err instanceof Error ? err.message : String(err),
+                };
+            }
         }
         logger.info("undispatched_dispatched", {
             undispatched_id: event.undispatched_id,
-            target_endpoint_id: event.target_endpoint_id,
+            target_endpoint_id: targetEndpointId,
             target_state: target.state,
         });
-        emitUndispatched({
-            type: "undispatched_changed",
-            version: 1,
-            action: "dispatch",
-            undispatched_id: event.undispatched_id,
-            ...(taken.source_message_id !== undefined && {
-                source_message_id: taken.source_message_id,
-            }),
-            dispatched_to_endpoint_id: event.target_endpoint_id,
-        });
         return { ok: true };
     });
     controlEventDispatcher.on("spawn_request", async (event) => {
@@ -562,9 +959,14 @@ export async function startBrokerDaemon(opts) {
             const spawnAvailability = spawnDriver
                 ? await spawnDriver.isAvailable().catch(() => ({ available: false }))
                 : null;
-            const runtimeCapabilities = spawnAvailability?.available
-                ? [REMOTE_SPAWN_CAPABILITY]
+            const runtimeCapabilitiesSet = new Set(body.runtime_capabilities?.filter((capability) => capability !== REMOTE_SPAWN_CAPABILITY) ?? []);
+            if (spawnAvailability?.available) {
+                runtimeCapabilitiesSet.add(REMOTE_SPAWN_CAPABILITY);
+            }
+            const runtimeCapabilities = runtimeCapabilitiesSet.size > 0
+                ? Array.from(runtimeCapabilitiesSet)
                 : undefined;
+            const runtimeSessionId = body.runtime_session_id?.trim() || undefined;
             const apiResp = await apiPort.register({
                 runtime_kind: body.kind,
                 endpoint_nonce: randomUUID(),
@@ -576,12 +978,19 @@ export async function startBrokerDaemon(opts) {
                 session_name: body.session_name,
                 task_hint: body.task_hint,
                 runtime_capabilities: runtimeCapabilities,
+                execution_surface: body.execution_surface,
+                runtime_session_id: runtimeSessionId,
+                background_exchange_mode: body.background_exchange_mode,
             });
             registry.register({
                 endpoint_id: apiResp.endpoint_id,
                 agent_id: body.agent_id,
                 plugin_pid: body.plugin_pid,
                 ipc_ws: boundWs,
+                runtime_capabilities: runtimeCapabilities,
+                execution_surface: body.execution_surface,
+                runtime_session_id: runtimeSessionId,
+                background_exchange_mode: body.background_exchange_mode,
                 display_metadata: {
                     session_name: body.session_name,
                     kind: body.kind,
@@ -691,15 +1100,262 @@ export async function startBrokerDaemon(opts) {
             }
             catch (err) {
                 const msg = err instanceof Error ? err.message : String(err);
-                if (msg === "WebSocket not connected" || msg === "SEND_ACK timeout") {
+                if (err instanceof ConnectorClientUnavailableError) {
                     throw new BrokerHttpError(503, "connector_unavailable", msg, {
                         retryable: true,
                     });
                 }
+                if (err instanceof ConnectorSendAckTimeoutError) {
+                    throw new BrokerHttpError(503, "connector_send_outcome_unknown", msg, { retryable: false });
+                }
                 throw err;
             }
             return { messageId: ack.messageId, status: ack.status };
         },
+        async sendRuntimeAssignmentReply(body) {
+            const entry = registry.get(body.endpoint_id);
+            if (!entry) {
+                throw new BrokerHttpError(404, "unknown_endpoint", `unknown endpoint_id: ${body.endpoint_id}`);
+            }
+            if (entry.plugin_pid !== body.plugin_pid) {
+                throw new BrokerHttpError(403, "endpoint_pid_mismatch", "plugin_pid does not own endpoint_id");
+            }
+            if (entry.state !== "active") {
+                throw new BrokerHttpError(409, "endpoint_not_active", `endpoint ${body.endpoint_id} is ${entry.state}; assignment reply is only allowed from Active`);
+            }
+            if (!body.reply_request_id) {
+                throw new BrokerHttpError(400, "reply_request_id_required", "reply_request_id is required for assignment replies");
+            }
+            const assignmentKey = runtimeAssignmentKey(body.source_message_id, body.assignment_id);
+            const replyKey = runtimeAssignmentReplyKey(body.source_message_id, body.assignment_id, body.reply_request_id);
+            pruneRuntimeAssignmentReplyRecords();
+            const requestFingerprint = runtimeAssignmentReplyFingerprint(body.content, body.contentType);
+            const assignment = runtimeAssignments.get(assignmentKey);
+            const existingRecord = runtimeAssignmentReplyRecords.get(replyKey);
+            const inFlight = runtimeAssignmentReplyInFlight.get(replyKey);
+            if (assignment) {
+                if (assignment.context.targetRef.projection_endpoint_id !==
+                    body.endpoint_id) {
+                    throw new BrokerHttpError(403, "runtime_assignment_target_mismatch", "endpoint_id does not match the assigned target");
+                }
+                if (!assignment.accepted) {
+                    throw new BrokerHttpError(409, "runtime_assignment_not_accepted", "assignment replies require adapter_accepted first");
+                }
+            }
+            else if (existingRecord || inFlight) {
+                const owner = existingRecord ?? inFlight;
+                if (owner?.endpointId !== body.endpoint_id ||
+                    owner.pluginPid !== body.plugin_pid) {
+                    throw new BrokerHttpError(403, "runtime_assignment_target_mismatch", "endpoint_id does not match the assigned target");
+                }
+            }
+            else {
+                throw new BrokerHttpError(409, "runtime_assignment_unknown", "source assignment is not known to this broker");
+            }
+            if (existingRecord &&
+                existingRecord.requestFingerprint !== requestFingerprint) {
+                throw new BrokerHttpError(409, "runtime_assignment_reply_idempotency_conflict", "reply_request_id was already used with different reply content");
+            }
+            if (existingRecord?.outcomeUnknown) {
+                throw new BrokerHttpError(503, "connector_send_outcome_unknown", existingRecord.outcomeUnknown.message, { retryable: false });
+            }
+            if (existingRecord?.result)
+                return existingRecord.result;
+            if (inFlight) {
+                if (inFlight.requestFingerprint !== requestFingerprint) {
+                    throw new BrokerHttpError(409, "runtime_assignment_reply_idempotency_conflict", "reply_request_id was already used with different reply content");
+                }
+                return inFlight.task;
+            }
+            if (!assignment) {
+                throw new BrokerHttpError(409, "runtime_assignment_unknown", "source assignment is not known to this broker");
+            }
+            const task = (async () => {
+                let record = runtimeAssignmentReplyRecords.get(replyKey);
+                if (!record) {
+                    const replyRequestedSequence = assignment.lastStateSequence + 1;
+                    const replyRequestedEvent = {
+                        type: "runtime_processing_state",
+                        version: 1,
+                        source_message_id: assignment.context.sourceMessageId,
+                        assignment_id: assignment.context.assignmentId,
+                        state_event_id: `${assignment.context.assignmentId}:reply_requested:${body.reply_request_id}`,
+                        state_sequence: replyRequestedSequence,
+                        target_ref: assignment.context.targetRef,
+                        state: "reply_requested",
+                        occurred_at: Date.now(),
+                        undispatched_id: assignment.context.undispatchedId,
+                        reply_request_id: body.reply_request_id,
+                    };
+                    const replyRequested = await emitRuntimeProcessingState(replyRequestedEvent);
+                    if (!replyRequested.ok) {
+                        if (replyRequested.terminal && replyRequested.status === 403) {
+                            logger.warn("runtime_reply_requested_projection_skipped", {
+                                source_message_id: assignment.context.sourceMessageId,
+                                assignment_id: assignment.context.assignmentId,
+                                status: replyRequested.status,
+                                detail: replyRequested.detail,
+                            });
+                        }
+                        else {
+                            throw new BrokerHttpError(replyRequested.terminal ? 409 : 503, "runtime_reply_request_rejected", replyRequested.detail ??
+                                "runtime reply request was not accepted");
+                        }
+                    }
+                    record = {
+                        replyRequestedEvent,
+                        endpointId: body.endpoint_id,
+                        pluginPid: body.plugin_pid,
+                        requestFingerprint,
+                        wireMessageId: randomUUID(),
+                        expiresAt: Date.now() + RUNTIME_ASSIGNMENT_REPLY_RECORD_TTL_MS,
+                    };
+                    runtimeAssignmentReplyRecords.set(replyKey, record);
+                    assignment.lastStateSequence = replyRequestedSequence;
+                }
+                const correlationId = typeof assignment.context.replyCorrelationId === "string" &&
+                    assignment.context.replyCorrelationId.length > 0
+                    ? assignment.context.replyCorrelationId
+                    : randomUUID();
+                entry.correlation_ring.add(correlationId);
+                const remoteMessageId = assignment.message.metadata.message_id;
+                const metadata = {
+                    message_id: record.wireMessageId,
+                    ...(typeof remoteMessageId === "string" &&
+                        remoteMessageId.length > 0 && { in_reply_to: remoteMessageId }),
+                    correlation_id: correlationId,
+                    source_endpoint_id: body.endpoint_id,
+                    require_live: false,
+                };
+                let ack;
+                try {
+                    ack = await connector.send(assignment.message.sender_address, body.content, body.contentType ?? "text", metadata);
+                }
+                catch (err) {
+                    const msg = err instanceof Error ? err.message : String(err);
+                    if (err instanceof ConnectorClientUnavailableError) {
+                        throw new BrokerHttpError(503, "connector_unavailable", msg, {
+                            retryable: true,
+                        });
+                    }
+                    if (err instanceof ConnectorSendAckTimeoutError) {
+                        record.outcomeUnknown = { message: msg };
+                        record.expiresAt =
+                            Date.now() + RUNTIME_ASSIGNMENT_REPLY_RECORD_TTL_MS;
+                        throw new BrokerHttpError(503, "connector_send_outcome_unknown", msg, { retryable: false });
+                    }
+                    throw err;
+                }
+                const result = {
+                    messageId: record.wireMessageId,
+                    status: ack.status,
+                };
+                record.result = result;
+                record.expiresAt = Date.now() + RUNTIME_ASSIGNMENT_REPLY_RECORD_TTL_MS;
+                const replyEmittedSequence = assignment.lastStateSequence + 1;
+                emitReplyEmittedWithRetry(assignmentKey, {
+                    type: "runtime_processing_state",
+                    version: 1,
+                    source_message_id: assignment.context.sourceMessageId,
+                    assignment_id: assignment.context.assignmentId,
+                    state_event_id: `${assignment.context.assignmentId}:reply_emitted:${body.reply_request_id}`,
+                    state_sequence: replyEmittedSequence,
+                    target_ref: assignment.context.targetRef,
+                    state: "reply_emitted",
+                    occurred_at: Date.now(),
+                    undispatched_id: assignment.context.undispatchedId,
+                    reply_request_id: body.reply_request_id,
+                    outbound_message_id: record.wireMessageId,
+                });
+                assignment.lastStateSequence = replyEmittedSequence;
+                return result;
+            })();
+            runtimeAssignmentReplyInFlight.set(replyKey, {
+                endpointId: body.endpoint_id,
+                pluginPid: body.plugin_pid,
+                requestFingerprint,
+                task,
+            });
+            try {
+                return await task;
+            }
+            finally {
+                runtimeAssignmentReplyInFlight.delete(replyKey);
+            }
+        },
+        async reportProcessingState(body) {
+            const entry = registry.get(body.endpoint_id);
+            if (!entry) {
+                throw new BrokerHttpError(404, "unknown_endpoint", `unknown endpoint_id: ${body.endpoint_id}`);
+            }
+            if (entry.plugin_pid !== body.plugin_pid) {
+                throw new BrokerHttpError(403, "endpoint_pid_mismatch", "plugin_pid does not own endpoint_id");
+            }
+            const key = runtimeAssignmentKey(body.source_message_id, body.assignment_id);
+            const assignment = runtimeAssignments.get(key);
+            if (!assignment) {
+                throw new BrokerHttpError(409, "runtime_assignment_unknown", "source assignment is not known to this broker");
+            }
+            if (assignment.context.targetRef.projection_endpoint_id !== body.endpoint_id) {
+                throw new BrokerHttpError(403, "runtime_assignment_target_mismatch", "endpoint_id does not match the assigned target");
+            }
+            if (body.state === "reply_requested" || body.state === "reply_emitted") {
+                throw new BrokerHttpError(403, `${body.state}_broker_owned`, `${body.state} is emitted by the broker assignment reply path`);
+            }
+            const event = {
+                type: "runtime_processing_state",
+                version: 1,
+                source_message_id: body.source_message_id,
+                assignment_id: body.assignment_id,
+                state_event_id: body.state_event_id,
+                state_sequence: body.state_sequence,
+                target_ref: assignment.context.targetRef,
+                state: body.state,
+                occurred_at: body.occurred_at ?? Date.now(),
+                undispatched_id: body.undispatched_id ?? assignment.context.undispatchedId,
+                ...(body.retryable !== undefined && { retryable: body.retryable }),
+                ...(body.reason_code !== undefined && {
+                    reason_code: body.reason_code,
+                }),
+                ...(body.detail !== undefined && { detail: body.detail }),
+                ...(body.reply_request_id !== undefined && {
+                    reply_request_id: body.reply_request_id,
+                }),
+                ...(body.outbound_message_id !== undefined && {
+                    outbound_message_id: body.outbound_message_id,
+                }),
+            };
+            const outcome = await emitRuntimeProcessingState(event);
+            if (!outcome.ok) {
+                throw new BrokerHttpError(outcome.terminal ? 409 : 503, "runtime_processing_state_rejected", outcome.detail ?? "runtime processing state was not accepted");
+            }
+            assignment.lastStateSequence = Math.max(assignment.lastStateSequence, body.state_sequence);
+            if (body.state === "adapter_accepted") {
+                assignment.accepted = true;
+                if (assignment.acceptanceTimer) {
+                    clearTimeout(assignment.acceptanceTimer);
+                    assignment.acceptanceTimer = undefined;
+                }
+            }
+            if (!assignment.accepted &&
+                (body.state === "processing_failed" ||
+                    body.state === "waiting_for_runtime_capability" ||
+                    body.state === "target_lost_before_acceptance")) {
+                if (body.state === "processing_failed" && body.retryable !== true) {
+                    if (assignment.acceptanceTimer) {
+                        clearTimeout(assignment.acceptanceTimer);
+                        assignment.acceptanceTimer = undefined;
+                    }
+                    forgetRuntimeAssignment(key);
+                    return;
+                }
+                restorePendingAssignmentBeforeAcceptance(key, assignment, body.state);
+                return;
+            }
+            if (body.state === "processing_failed" || body.state === "expired") {
+                forgetRuntimeAssignment(key);
+            }
+        },
         async reattachEndpoint(endpoint_id, plugin_pid, _ipcWsHint) {
             const ipcWs = channels.get(plugin_pid);
             if (!ipcWs) {
@@ -722,11 +1378,12 @@ export async function startBrokerDaemon(opts) {
             }
             bound.add(endpoint_id);
             const buf = reconnectingBuffers.forEndpoint(endpoint_id);
-            const drained = buf.drain();
+            const drained = buf.read();
             graceTimers.cancel(endpoint_id);
             registry.markActive(endpoint_id, ipcWs);
+            let allPushed = true;
             for (const msg of drained) {
-                pushToPlugin(ipcWs, {
+                const pushed = pushToPlugin(ipcWs, {
                     event: "message_received",
                     from: msg.envelope.from,
                     content: msg.envelope.content,
@@ -735,8 +1392,23 @@ export async function startBrokerDaemon(opts) {
                     ...(msg.envelope.sourceMessageId !== undefined && {
                         sourceMessageId: msg.envelope.sourceMessageId,
                     }),
+                    ...(msg.envelope.runtimeAssignment !== undefined && {
+                        runtimeAssignment: msg.envelope.runtimeAssignment,
+                    }),
                 });
+                if (!pushed) {
+                    allPushed = false;
+                    break;
+                }
             }
+            if (!allPushed) {
+                handleTransition(endpoint_id, {
+                    type: "ipc_close",
+                    plugin_alive: livenessProbe(plugin_pid),
+                });
+                throw new BrokerHttpError(503, "reconnecting_buffer_drain_failed", "failed to push buffered messages to reattached endpoint", { retryable: true });
+            }
+            buf.delete();
             emitTransition(endpoint_id, "active", "reattach");
             logger.info("endpoint_reattached", {
                 endpoint_id,
@@ -749,13 +1421,17 @@ export async function startBrokerDaemon(opts) {
             return undispatchedInbox.list();
         },
         async dispatch(undispatched_id, target_endpoint_id) {
+            const pending = undispatchedInbox.get(undispatched_id);
+            const target = registry.get(target_endpoint_id);
             const result = await controlEventDispatcher.dispatch({
                 type: "dispatch_undispatched",
                 version: 1,
                 idempotency_key: `local-${randomUUID()}`,
                 emitted_at: Date.now(),
                 undispatched_id,
-                target_endpoint_id,
+                source_message_id: pending?.source_message_id ?? undispatched_id,
+                target_ref: buildLocalDispatchTargetRef(target_endpoint_id, target),
+                delivery_intent: "external_handoff",
             });
             if (!result.ok) {
                 throw new BrokerHttpError(400, "dispatch_failed", result.detail ?? "unknown");
@@ -841,21 +1517,71 @@ export async function startBrokerDaemon(opts) {
             ? meta.target_endpoint_id
             : undefined;
         const correlation_id = typeof meta.correlation_id === "string" ? meta.correlation_id : undefined;
+        const addUndispatchedFromPayload = async (opts) => {
+            const undispatched = {
+                id: payload.messageId ?? randomUUID(),
+                ...(payload.messageId !== undefined && {
+                    source_message_id: payload.messageId,
+                }),
+                arrived_at: new Date().toISOString(),
+                sender_address: payload.from,
+                content: payload.content,
+                content_type: payload.contentType,
+                metadata: meta,
+                reason: opts.reason,
+                original_target_endpoint_id: opts.originalTargetEndpointId,
+                original_correlation_id: opts.originalCorrelationId,
+            };
+            const added = undispatchedInbox.tryAdd(undispatched);
+            if (!added) {
+                logger.warn("undispatched_inbox_at_capacity", {
+                    from: payload.from,
+                    size: undispatchedInbox.size(),
+                    capacity: undispatchedInbox.capacity(),
+                });
+                return false;
+            }
+            logger.info("undispatched_added", {
+                id: undispatched.id,
+                reason: opts.reason,
+            });
+            const wireReason = undispatched.reason === "unaddressed"
+                ? "no_target_endpoint"
+                : undispatched.reason;
+            const posted = await directPostUndispatched({
+                type: "undispatched_changed",
+                version: 1,
+                action: "add",
+                undispatched_id: undispatched.id,
+                ...(undispatched.source_message_id !== undefined && {
+                    source_message_id: undispatched.source_message_id,
+                }),
+                sender_address: undispatched.sender_address,
+                content_preview: truncateContentPreview(undispatched.content, CONTENT_PREVIEW_MAX_CODEPOINTS),
+                reason: wireReason,
+                ...(opts.originalTargetEndpointId !== undefined && {
+                    target_endpoint_id_hint: opts.originalTargetEndpointId,
+                }),
+                ...(typeof correlation_id === "string" && {
+                    in_reply_to: correlation_id,
+                }),
+                arrived_at: Date.parse(undispatched.arrived_at),
+            }, { queueOnTransientFailure: typeof payload.seq !== "number" });
+            if (!posted)
+                return false;
+            return markDeliverySeqAccepted(payload.seq);
+        };
         const decision = router.route({ target_endpoint_id, correlation_id });
         if (decision.kind === "endpoint") {
-            if (payload.messageId !== undefined) {
-                const posted = await directPostRuntimeInboundRouted({
-                    type: "runtime_inbound_routed",
-                    version: 1,
-                    source_message_id: payload.messageId,
-                    routed_to_endpoint_id: decision.entry.endpoint_id,
-                    route_kind: decision.entry.state === "active"
-                        ? "active_endpoint"
-                        : "reconnecting_endpoint",
-                    routed_at: Date.now(),
-                }, { queueOnTransientFailure: typeof payload.seq !== "number" });
-                if (!posted && typeof payload.seq === "number")
-                    return false;
+            const isCorrelationContinuation = typeof correlation_id === "string" &&
+                decision.entry.correlation_ring.has(correlation_id);
+            if (isBackgroundExchangeTarget(decision.entry) &&
+                !isCorrelationContinuation) {
+                return addUndispatchedFromPayload({
+                    reason: "unaddressed",
+                    originalTargetEndpointId: decision.entry.endpoint_id,
+                    originalCorrelationId: correlation_id,
+                });
             }
             recordInboundCorrelation(decision.entry.endpoint_id, meta);
             if (decision.entry.state === "active") {
@@ -874,24 +1600,47 @@ export async function startBrokerDaemon(opts) {
             }
             else {
                 const buf = reconnectingBuffers.forEndpoint(decision.entry.endpoint_id);
-                buf.append({
-                    id: payload.messageId ?? randomUUID(),
-                    arrived_at: new Date().toISOString(),
-                    envelope: {
-                        from: payload.from,
-                        content: payload.content,
-                        contentType: payload.contentType,
-                        metadata: meta,
-                        ...(payload.messageId !== undefined && {
-                            sourceMessageId: payload.messageId,
-                        }),
-                    },
-                });
+                try {
+                    buf.append({
+                        id: payload.messageId ?? randomUUID(),
+                        arrived_at: new Date().toISOString(),
+                        envelope: {
+                            from: payload.from,
+                            content: payload.content,
+                            contentType: payload.contentType,
+                            metadata: meta,
+                            ...(payload.messageId !== undefined && {
+                                sourceMessageId: payload.messageId,
+                            }),
+                        },
+                    });
+                }
+                catch (err) {
+                    logger.warn("buffer_for_reconnecting_endpoint_failed", {
+                        endpoint_id: decision.entry.endpoint_id,
+                        err: err instanceof Error ? err.message : String(err),
+                    });
+                    return false;
+                }
                 logger.info("buffered_for_reconnecting_endpoint", {
                     endpoint_id: decision.entry.endpoint_id,
                     buffer_size: buf.size(),
                 });
             }
+            if (payload.messageId !== undefined) {
+                const posted = await directPostRuntimeInboundRouted({
+                    type: "runtime_inbound_routed",
+                    version: 1,
+                    source_message_id: payload.messageId,
+                    routed_to_endpoint_id: decision.entry.endpoint_id,
+                    route_kind: decision.entry.state === "active"
+                        ? "active_endpoint"
+                        : "reconnecting_endpoint",
+                    routed_at: Date.now(),
+                }, { queueOnTransientFailure: false });
+                if (!posted)
+                    return false;
+            }
             return markDeliverySeqAccepted(payload.seq);
         }
         let reason = "unaddressed";
@@ -911,55 +1660,11 @@ export async function startBrokerDaemon(opts) {
                 });
             }
         }
-        const undispatched = {
-            id: payload.messageId ?? randomUUID(),
-            ...(payload.messageId !== undefined && {
-                source_message_id: payload.messageId,
-            }),
-            arrived_at: new Date().toISOString(),
-            sender_address: payload.from,
-            content: payload.content,
-            content_type: payload.contentType,
-            metadata: meta,
+        return addUndispatchedFromPayload({
             reason,
-            original_target_endpoint_id: originalTarget,
-            original_correlation_id: correlation_id,
-        };
-        const added = undispatchedInbox.tryAdd(undispatched);
-        if (!added) {
-            logger.warn("undispatched_inbox_at_capacity", {
-                from: payload.from,
-                size: undispatchedInbox.size(),
-                capacity: undispatchedInbox.capacity(),
-            });
-            return false;
-        }
-        logger.info("undispatched_added", { id: undispatched.id, reason });
-        const wireReason = undispatched.reason === "unaddressed"
-            ? "no_target_endpoint"
-            : undispatched.reason;
-        const posted = await directPostUndispatched({
-            type: "undispatched_changed",
-            version: 1,
-            action: "add",
-            undispatched_id: undispatched.id,
-            ...(undispatched.source_message_id !== undefined && {
-                source_message_id: undispatched.source_message_id,
-            }),
-            sender_address: undispatched.sender_address,
-            content_preview: truncateContentPreview(undispatched.content, CONTENT_PREVIEW_MAX_CODEPOINTS),
-            reason: wireReason,
-            ...(undispatched.original_target_endpoint_id !== undefined && {
-                target_endpoint_id_hint: undispatched.original_target_endpoint_id,
-            }),
-            ...(typeof correlation_id === "string" && {
-                in_reply_to: correlation_id,
-            }),
-            arrived_at: Date.parse(undispatched.arrived_at),
-        }, { queueOnTransientFailure: typeof payload.seq !== "number" });
-        if (!posted)
-            return false;
-        return markDeliverySeqAccepted(payload.seq);
+            originalTargetEndpointId: originalTarget,
+            originalCorrelationId: correlation_id,
+        });
     };
     connector.on("message_received", ({ payload }) => {
         let task;
@@ -1081,6 +1786,7 @@ export async function startBrokerDaemon(opts) {
             });
         }
         await runtimeInboundRoutedEmitter.shutdown();
+        await flushRuntimeAssignmentsOnShutdown();
         const endpoints = registry.list();
         await Promise.all(endpoints.map(async (entry) => {
             try {