@masons/runtime-broker 0.2.0 → 0.2.2

package/dist/broker/broker-daemon.js

@@ -2,6 +2,7 @@ import { spawn as spawnChild } from "node:child_process";
 import { randomUUID } from "node:crypto";
 import { basename } from "node:path";
 import { createControlEventDispatcher, } from "./control-event-dispatcher.js";
+import { readDeliveryCursorFile, writeDeliveryCursorFile, } from "./delivery-cursor-file.js";
 import { deleteDiscoveryFile, mintBearerToken, writeDiscoveryFile, } from "./discovery-file.js";
 import { EndpointRegistry } from "./endpoint-registry.js";
 import { DEFAULT_GRACE_MS, transition, } from "./endpoint-state-machine.js";
@@ -13,12 +14,23 @@ import { isPluginAlive as defaultIsPluginAlive } from "./plugin-liveness.js";
 import { createReceivedMessageCorrelationCache, DEFAULT_TTL_MS as DEFAULT_REPLY_CORRELATION_TTL_MS, } from "./received-message-correlation-cache.js";
 import { createReconnectingBufferManager, } from "./reconnecting-buffer.js";
 import { RoutingTable } from "./routing-table.js";
+import { createRuntimeInboundRoutedEmitter, postRuntimeInboundRoutedViaPort, } from "./runtime-inbound-routed-emitter.js";
 import { createSpawnCorrelationManager, createSpawnRateLimiter, } from "./spawn-correlation.js";
 import { TASK_HINT_MAX_LENGTH, updateTaskHint } from "./task-hint-handler.js";
 import { createTransitionStateRetryQueue } from "./transition-state-retry-queue.js";
 import { CONTENT_PREVIEW_MAX_CODEPOINTS, truncateContentPreview, } from "./undispatched-changed-event-types.js";
 import { createUndispatchedChangedEmitter, postUndispatchedChangedViaPort, } from "./undispatched-emitter.js";
 import { createUndispatchedInbox, } from "./undispatched-inbox.js";
+const REMOTE_SPAWN_CAPABILITY = "remote_spawn_v1";
+const DEFAULT_RUNTIME_ASSIGNMENT_ACCEPTANCE_TIMEOUT_MS = 30_000;
+const RUNTIME_PROCESSING_RETRY_INITIAL_MS = 1_000;
+const RUNTIME_PROCESSING_RETRY_MAX_MS = 30_000;
+function isBackgroundExchangeMode(value) {
+    return value === "resident_endpoint" || value === "adapter_managed_turn";
+}
+function normalizeBackgroundExchangeMode(value) {
+    return isBackgroundExchangeMode(value) ? value : "resident_endpoint";
+}
 function derivePublicDisplayLabel(body) {
     const parts = [
         body.kind,
@@ -36,10 +48,17 @@ export async function startBrokerDaemon(opts) {
     const { paths, asNodeId: _asNodeIdReservedForA3, connector, apiPort, logger, closedEndpointLookup, livenessProbe = defaultIsPluginAlive, spawnDriverRegistry, spawnFn = spawnChild, servicesEventClientFactory, } = opts;
     const graceMs = opts.graceMs ?? DEFAULT_GRACE_MS;
     const presenceGraceMs = opts.presenceGraceMs;
+    const runtimeAssignmentAcceptanceTimeoutMs = opts.runtimeAssignmentAcceptanceTimeoutMs ??
+        DEFAULT_RUNTIME_ASSIGNMENT_ACCEPTANCE_TIMEOUT_MS;
     const postControlAck = opts.postControlAck ??
         (async () => {
         });
     void _asNodeIdReservedForA3;
+    const persistedDeliveryCursor = readDeliveryCursorFile(paths.deliveryCursorFile) ?? 0;
+    connector.setDeliveryCursor(persistedDeliveryCursor);
+    logger.info("delivery_cursor_loaded", {
+        lastKnownSeq: persistedDeliveryCursor,
+    });
     logger.info("connecting_to_connector");
     await connector.connect();
     logger.info("connector_connected");
@@ -55,6 +74,8 @@ export async function startBrokerDaemon(opts) {
     const reconnectingBuffers = createReconnectingBufferManager(paths.buffersDir);
     const graceTimers = createGraceTimerManager();
     const controlEventDispatcher = createControlEventDispatcher();
+    const runtimeAssignments = new Map();
+    const runtimeAssignmentByEndpointCorrelation = new Map();
     const spawnCorrelation = createSpawnCorrelationManager({
         timeoutMs: opts.spawnCorrelationTimeoutMs,
     });
@@ -86,7 +107,8 @@ export async function startBrokerDaemon(opts) {
         }
     }, replyCorrelationSweepInterval);
     replyCorrelationSweepTimer.unref?.();
-    const undispatchedEmitter = createUndispatchedChangedEmitter(opts.undispatchedChangedPost ?? postUndispatchedChangedViaPort(apiPort), logger, {
+    const postUndispatchedChanged = opts.undispatchedChangedPost ?? postUndispatchedChangedViaPort(apiPort);
+    const undispatchedEmitter = createUndispatchedChangedEmitter(postUndispatchedChanged, logger, {
         capacity: opts.undispatchedChangedCapacity,
         backoffInitialMs: opts.undispatchedChangedBackoffInitialMs,
         backoffMaxMs: opts.undispatchedChangedBackoffMaxMs,
@@ -105,6 +127,144 @@ export async function startBrokerDaemon(opts) {
     const emitNetworkPresence = (event) => {
         networkPresenceEmitter.enqueue(event);
     };
+    const postRuntimeInboundRouted = opts.runtimeInboundRoutedPost ?? postRuntimeInboundRoutedViaPort(apiPort);
+    const runtimeInboundRoutedEmitter = createRuntimeInboundRoutedEmitter(postRuntimeInboundRouted, logger, {
+        capacity: opts.runtimeInboundRoutedCapacity,
+        backoffInitialMs: opts.runtimeInboundRoutedBackoffInitialMs,
+        backoffMaxMs: opts.runtimeInboundRoutedBackoffMaxMs,
+        maxRetries: opts.runtimeInboundRoutedMaxRetries,
+    });
+    const runtimeAssignmentKey = (sourceMessageId, assignmentId) => `${sourceMessageId}\u0000${assignmentId}`;
+    const endpointCorrelationKey = (endpointId, correlationId) => `${endpointId}\u0000${correlationId}`;
+    const isBackgroundExchangeTarget = (entry) => Boolean(entry.runtime_capabilities?.includes("background_exchange_v1") &&
+        isBackgroundExchangeMode(entry.background_exchange_mode));
+    const rememberRuntimeAssignmentCorrelation = (endpointId, correlationId, assignmentKey) => {
+        if (typeof correlationId !== "string" || correlationId.length === 0) {
+            return;
+        }
+        runtimeAssignmentByEndpointCorrelation.set(endpointCorrelationKey(endpointId, correlationId), assignmentKey);
+    };
+    const forgetRuntimeAssignment = (key) => {
+        const assignment = runtimeAssignments.get(key);
+        if (assignment?.acceptanceTimer) {
+            clearTimeout(assignment.acceptanceTimer);
+        }
+        if (assignment?.replyEmittedRetryTimer) {
+            clearTimeout(assignment.replyEmittedRetryTimer);
+        }
+        runtimeAssignments.delete(key);
+        for (const [correlationKey, assignmentKey] of Array.from(runtimeAssignmentByEndpointCorrelation.entries())) {
+            if (assignmentKey === key) {
+                runtimeAssignmentByEndpointCorrelation.delete(correlationKey);
+            }
+        }
+    };
+    const scheduleAcceptanceTimeout = (key) => {
+        if (runtimeAssignmentAcceptanceTimeoutMs <= 0)
+            return;
+        const assignment = runtimeAssignments.get(key);
+        if (!assignment || assignment.accepted || assignment.acceptanceTimer) {
+            return;
+        }
+        const timer = setTimeout(() => {
+            const current = runtimeAssignments.get(key);
+            if (!current || current.accepted)
+                return;
+            logger.warn("runtime_assignment_acceptance_timeout", {
+                undispatched_id: current.context.undispatchedId,
+                source_message_id: current.context.sourceMessageId,
+                target_endpoint_id: current.context.targetRef.projection_endpoint_id,
+            });
+            void restorePendingAssignmentsForEndpoint(current.context.targetRef.projection_endpoint_id, "adapter_acceptance_timeout");
+        }, runtimeAssignmentAcceptanceTimeoutMs);
+        timer.unref?.();
+        assignment.acceptanceTimer = timer;
+    };
+    const emitRuntimeProcessingState = async (event) => {
+        if (!apiPort.emitRuntimeProcessingState) {
+            return {
+                ok: false,
+                terminal: true,
+                detail: "runtime_processing_state_port_not_configured",
+            };
+        }
+        try {
+            return await apiPort.emitRuntimeProcessingState(event);
+        }
+        catch (err) {
+            return {
+                ok: false,
+                terminal: false,
+                detail: err instanceof Error ? err.message : String(err),
+            };
+        }
+    };
+    const directPostUndispatched = async (event, opts = {}) => {
+        let outcome;
+        try {
+            outcome = await postUndispatchedChanged(event);
+        }
+        catch (err) {
+            outcome = {
+                ok: false,
+                terminal: false,
+                detail: err instanceof Error ? err.message : String(err),
+            };
+        }
+        if (outcome.ok)
+            return true;
+        logger.warn("undispatched_changed_direct_post_failed", {
+            action: event.action,
+            undispatched_id: event.undispatched_id,
+            terminal: outcome.terminal,
+            status: outcome.status,
+            detail: outcome.detail,
+        });
+        if (!outcome.terminal && opts.queueOnTransientFailure !== false) {
+            undispatchedEmitter.enqueue(event);
+        }
+        return false;
+    };
+    const directPostRuntimeInboundRouted = async (event, opts = {}) => {
+        let outcome;
+        try {
+            outcome = await postRuntimeInboundRouted(event);
+        }
+        catch (err) {
+            outcome = {
+                ok: false,
+                terminal: false,
+                detail: err instanceof Error ? err.message : String(err),
+            };
+        }
+        if (outcome.ok)
+            return true;
+        logger.warn("runtime_inbound_routed_direct_post_failed", {
+            source_message_id: event.source_message_id,
+            routed_to_endpoint_id: event.routed_to_endpoint_id,
+            terminal: outcome.terminal,
+            status: outcome.status,
+            detail: outcome.detail,
+        });
+        if (!outcome.terminal && opts.queueOnTransientFailure !== false) {
+            runtimeInboundRoutedEmitter.enqueue(event);
+        }
+        return false;
+    };
+    const markDeliverySeqAccepted = (seq) => {
+        if (typeof seq !== "number")
+            return true;
+        const before = connector.getDeliveryCursor();
+        const prepared = connector.prepareDeliverySeqAccepted(seq);
+        if (typeof prepared === "number" && prepared !== before) {
+            writeDeliveryCursorFile(paths.deliveryCursorFile, prepared);
+        }
+        const after = connector.markDeliverySeqAccepted(seq);
+        if (after !== prepared) {
+            throw new Error("delivery cursor changed during durable commit");
+        }
+        return true;
+    };
     const recordInboundCorrelation = (endpoint_id, metadata) => {
         const c = metadata && typeof metadata.correlation_id === "string"
             ? metadata.correlation_id
@@ -113,6 +273,127 @@ export async function startBrokerDaemon(opts) {
             return;
         replyCorrelationCache.record(endpoint_id, c);
     };
+    const restorePendingAssignmentsForEndpoint = (endpoint_id, reason) => {
+        const tasks = [];
+        for (const [key, assignment] of Array.from(runtimeAssignments.entries())) {
+            if (assignment.context.targetRef.projection_endpoint_id !== endpoint_id ||
+                assignment.accepted) {
+                continue;
+            }
+            if (assignment.acceptanceTimer) {
+                clearTimeout(assignment.acceptanceTimer);
+                assignment.acceptanceTimer = undefined;
+            }
+            const readded = undispatchedInbox.tryAdd(assignment.message);
+            if (!readded) {
+                logger.error("runtime_assignment_reinsert_failed_at_capacity", {
+                    undispatched_id: assignment.context.undispatchedId,
+                    source_message_id: assignment.context.sourceMessageId,
+                    target_endpoint_id: endpoint_id,
+                    inbox_size: undispatchedInbox.size(),
+                    inbox_capacity: undispatchedInbox.capacity(),
+                });
+                scheduleAcceptanceTimeout(key);
+                continue;
+            }
+            const task = emitRuntimeProcessingState({
+                type: "runtime_processing_state",
+                version: 1,
+                source_message_id: assignment.context.sourceMessageId,
+                assignment_id: assignment.context.assignmentId,
+                state_event_id: `${assignment.context.assignmentId}:target_lost`,
+                state_sequence: 2,
+                target_ref: assignment.context.targetRef,
+                state: "target_lost_before_acceptance",
+                occurred_at: assignment.assignedAt + 1,
+                undispatched_id: assignment.context.undispatchedId,
+                reason_code: reason,
+            }).then((outcome) => {
+                if (outcome.ok) {
+                    forgetRuntimeAssignment(key);
+                    return;
+                }
+                logger.warn("runtime_assignment_target_lost_emit_failed", {
+                    undispatched_id: assignment.context.undispatchedId,
+                    source_message_id: assignment.context.sourceMessageId,
+                    target_endpoint_id: endpoint_id,
+                    status: outcome.status,
+                    terminal: outcome.terminal,
+                    detail: outcome.detail,
+                });
+                scheduleAcceptanceTimeout(key);
+            });
+            tasks.push(task);
+        }
+        return Promise.allSettled(tasks).then(() => undefined);
+    };
+    const emitReplyEmittedWithRetry = (assignmentKey, event, attempt = 0) => {
+        const assignment = runtimeAssignments.get(assignmentKey);
+        if (!assignment)
+            return;
+        assignment.replyEmittedEvent = event;
+        if (assignment.replyEmittedRetryTimer) {
+            clearTimeout(assignment.replyEmittedRetryTimer);
+            assignment.replyEmittedRetryTimer = undefined;
+        }
+        void emitRuntimeProcessingState(event).then((outcome) => {
+            const current = runtimeAssignments.get(assignmentKey);
+            if (!current)
+                return;
+            if (outcome.ok) {
+                forgetRuntimeAssignment(assignmentKey);
+                return;
+            }
+            logger.warn("runtime_reply_emitted_projection_failed", {
+                source_message_id: current.context.sourceMessageId,
+                assignment_id: current.context.assignmentId,
+                status: outcome.status,
+                terminal: outcome.terminal,
+                detail: outcome.detail,
+            });
+            if (outcome.terminal)
+                return;
+            const delay = Math.min(RUNTIME_PROCESSING_RETRY_INITIAL_MS * 2 ** attempt, RUNTIME_PROCESSING_RETRY_MAX_MS);
+            const timer = setTimeout(() => {
+                emitReplyEmittedWithRetry(assignmentKey, event, attempt + 1);
+            }, delay);
+            timer.unref?.();
+            current.replyEmittedRetryTimer = timer;
+        });
+    };
+    const flushRuntimeAssignmentsOnShutdown = async () => {
+        const endpointIds = new Set();
+        for (const assignment of runtimeAssignments.values()) {
+            endpointIds.add(assignment.context.targetRef.projection_endpoint_id);
+        }
+        for (const [key, assignment] of Array.from(runtimeAssignments.entries())) {
+            if (assignment.replyEmittedEvent) {
+                const outcome = await emitRuntimeProcessingState(assignment.replyEmittedEvent);
+                if (outcome.ok) {
+                    forgetRuntimeAssignment(key);
+                    continue;
+                }
+                logger.warn("runtime_reply_emitted_shutdown_flush_failed", {
+                    source_message_id: assignment.context.sourceMessageId,
+                    assignment_id: assignment.context.assignmentId,
+                    status: outcome.status,
+                    terminal: outcome.terminal,
+                    detail: outcome.detail,
+                });
+            }
+        }
+        for (const endpointId of endpointIds) {
+            await restorePendingAssignmentsForEndpoint(endpointId, "broker_shutdown");
+        }
+        if (runtimeAssignments.size > 0) {
+            logger.warn("runtime_assignments_dropped_on_shutdown", {
+                size: runtimeAssignments.size,
+            });
+        }
+        for (const key of Array.from(runtimeAssignments.keys())) {
+            forgetRuntimeAssignment(key);
+        }
+    };
     const channels = new Map();
     let networkPresence = "offline";
     let presenceGraceTimer;
@@ -275,25 +556,46 @@ export async function startBrokerDaemon(opts) {
                     break;
                 case "drain_buffer": {
                     const buf = reconnectingBuffers.forEndpoint(endpoint_id);
-                    const drained = buf.drain();
+                    const drained = buf.read();
                     const e = registry.get(endpoint_id);
+                    let allPushed = true;
                     if (e) {
                         for (const msg of drained) {
-                            pushToPlugin(e.ipc_ws, {
+                            const pushed = pushToPlugin(e.ipc_ws, {
                                 event: "message_received",
                                 from: msg.envelope.from,
                                 content: msg.envelope.content,
                                 contentType: msg.envelope.contentType,
                                 metadata: msg.envelope.metadata,
+                                ...(msg.envelope.sourceMessageId !== undefined && {
+                                    sourceMessageId: msg.envelope.sourceMessageId,
+                                }),
+                                ...(msg.envelope.runtimeAssignment !== undefined && {
+                                    runtimeAssignment: msg.envelope.runtimeAssignment,
+                                }),
                             });
+                            if (!pushed) {
+                                allPushed = false;
+                                break;
+                            }
                         }
                     }
+                    if (allPushed) {
+                        buf.delete();
+                    }
+                    else {
+                        logger.warn("reconnecting_buffer_drain_failed", {
+                            endpoint_id,
+                            buffered_count: drained.length,
+                        });
+                    }
                     break;
                 }
                 case "delete_buffer":
                     reconnectingBuffers.cleanup(endpoint_id);
                     break;
                 case "remove_from_registry":
+                    void restorePendingAssignmentsForEndpoint(endpoint_id, "endpoint_lost");
                     clearHeartbeat(endpoint_id);
                     registry.unregister(endpoint_id);
                     replyCorrelationCache.forgetEndpoint(endpoint_id);
@@ -304,20 +606,49 @@ export async function startBrokerDaemon(opts) {
         }
     };
     controlEventDispatcher.on("dispatch_undispatched", async (event) => {
-        const taken = undispatchedInbox.take(event.undispatched_id);
+        const targetEndpointId = event.target_ref?.projection_endpoint_id ?? event.target_endpoint_id;
+        if (!targetEndpointId) {
+            return {
+                ok: false,
+                detail: "dispatch target_ref missing",
+            };
+        }
+        const assignmentKey = runtimeAssignmentKey(event.source_message_id, event.idempotency_key);
+        const existingAssignment = runtimeAssignments.get(assignmentKey);
+        const taken = undispatchedInbox.take(event.undispatched_id) ??
+            (!existingAssignment?.accepted &&
+                existingAssignment?.context.undispatchedId === event.undispatched_id &&
+                existingAssignment.context.targetRef.projection_endpoint_id ===
+                    targetEndpointId
+                ? existingAssignment.message
+                : undefined);
         if (!taken) {
             return {
                 ok: false,
                 detail: `unknown undispatched_id: ${event.undispatched_id}`,
             };
         }
-        const target = registry.get(event.target_endpoint_id);
+        if (taken.source_message_id === undefined) {
+            undispatchedInbox.tryAdd(taken);
+            return {
+                ok: false,
+                detail: "dispatch source_message_id missing from inbox row",
+            };
+        }
+        if (event.source_message_id !== taken.source_message_id) {
+            undispatchedInbox.tryAdd(taken);
+            return {
+                ok: false,
+                detail: `source_message_id mismatch: ${event.source_message_id}`,
+            };
+        }
+        const target = registry.get(targetEndpointId);
         if (!target) {
             const readded = undispatchedInbox.tryAdd(taken);
             if (!readded) {
                 logger.error("dispatch_reinsert_failed_at_capacity", {
                     undispatched_id: event.undispatched_id,
-                    target_endpoint_id: event.target_endpoint_id,
+                    target_endpoint_id: targetEndpointId,
                     inbox_size: undispatchedInbox.size(),
                     inbox_capacity: undispatchedInbox.capacity(),
                 });
@@ -326,14 +657,82 @@ export async function startBrokerDaemon(opts) {
                     version: 1,
                     action: "remove",
                     undispatched_id: event.undispatched_id,
+                    ...(taken.source_message_id !== undefined && {
+                        source_message_id: taken.source_message_id,
+                    }),
                     remove_reason: "lost_at_capacity_during_redispatch_bounce",
                 });
             }
             return {
                 ok: false,
-                detail: `target endpoint not found: ${event.target_endpoint_id}`,
+                detail: `target endpoint not found: ${targetEndpointId}`,
+            };
+        }
+        const targetRef = event.target_ref ?? {
+            kind: "projection_endpoint",
+            projection_endpoint_id: targetEndpointId,
+            background_exchange_mode: normalizeBackgroundExchangeMode(target.background_exchange_mode),
+        };
+        if (!target.runtime_capabilities?.includes("background_exchange_v1") ||
+            target.background_exchange_mode !== targetRef.background_exchange_mode) {
+            undispatchedInbox.tryAdd(taken);
+            return {
+                ok: false,
+                detail: `target endpoint is not background-exchange capable: ${targetEndpointId}`,
             };
         }
+        const assignedAt = event.emitted_at;
+        const processingEvent = {
+            type: "runtime_processing_state",
+            version: 1,
+            source_message_id: event.source_message_id,
+            assignment_id: event.idempotency_key,
+            state_event_id: `${event.idempotency_key}:assigned`,
+            state_sequence: 1,
+            target_ref: targetRef,
+            state: "assigned_to_runtime_target",
+            occurred_at: assignedAt,
+            undispatched_id: event.undispatched_id,
+        };
+        const processingResult = await emitRuntimeProcessingState(processingEvent);
+        if (!processingResult.ok) {
+            undispatchedInbox.tryAdd(taken);
+            logger.warn("runtime_processing_state_emit_failed", {
+                undispatched_id: event.undispatched_id,
+                source_message_id: event.source_message_id,
+                target_endpoint_id: targetEndpointId,
+                status: processingResult.status,
+                terminal: processingResult.terminal,
+                detail: processingResult.detail,
+            });
+            return {
+                ok: false,
+                detail: processingResult.detail ??
+                    "runtime_processing_state_assignment_failed",
+            };
+        }
+        const runtimeAssignment = {
+            undispatchedId: event.undispatched_id,
+            sourceMessageId: event.source_message_id,
+            assignmentId: event.idempotency_key,
+            targetRef,
+            deliveryIntent: "external_handoff",
+            ...(typeof taken.metadata.correlation_id === "string" && {
+                replyCorrelationId: taken.metadata.correlation_id,
+            }),
+        };
+        const storedAssignment = existingAssignment ?? {
+            context: runtimeAssignment,
+            message: taken,
+            accepted: false,
+            assignedAt,
+        };
+        storedAssignment.context = runtimeAssignment;
+        storedAssignment.message = taken;
+        storedAssignment.assignedAt = assignedAt;
+        runtimeAssignments.set(assignmentKey, storedAssignment);
+        scheduleAcceptanceTimeout(assignmentKey);
+        rememberRuntimeAssignmentCorrelation(target.endpoint_id, taken.metadata.correlation_id, assignmentKey);
         const stamped = {
             event: "message_received",
             from: taken.sender_address,
@@ -342,38 +741,50 @@ export async function startBrokerDaemon(opts) {
             metadata: {
                 ...taken.metadata,
                 dispatched_from: event.undispatched_id,
-                dispatched_by: "passport",
+                delivery_intent: "external_handoff",
             },
+            sourceMessageId: taken.source_message_id,
+            runtimeAssignment,
         };
         recordInboundCorrelation(target.endpoint_id, stamped.metadata);
         if (target.state === "active") {
-            pushToPlugin(target.ipc_ws, stamped);
+            if (!pushToPlugin(target.ipc_ws, stamped)) {
+                await restorePendingAssignmentsForEndpoint(target.endpoint_id, "ipc_push_failed");
+                return {
+                    ok: false,
+                    detail: `target endpoint not writable: ${targetEndpointId}`,
+                };
+            }
         }
         else {
-            const buf = reconnectingBuffers.forEndpoint(target.endpoint_id);
-            buf.append({
-                id: randomUUID(),
-                arrived_at: new Date().toISOString(),
-                envelope: {
-                    from: taken.sender_address,
-                    content: taken.content,
-                    contentType: taken.content_type,
-                    metadata: stamped.metadata,
-                },
-            });
+            try {
+                const buf = reconnectingBuffers.forEndpoint(target.endpoint_id);
+                buf.append({
+                    id: randomUUID(),
+                    arrived_at: new Date().toISOString(),
+                    envelope: {
+                        from: taken.sender_address,
+                        content: taken.content,
+                        contentType: taken.content_type,
+                        metadata: stamped.metadata,
+                        sourceMessageId: stamped.sourceMessageId,
+                        runtimeAssignment,
+                    },
+                });
+            }
+            catch (err) {
+                await restorePendingAssignmentsForEndpoint(target.endpoint_id, "reconnecting_buffer_failed");
+                return {
+                    ok: false,
+                    detail: err instanceof Error ? err.message : String(err),
+                };
+            }
         }
         logger.info("undispatched_dispatched", {
             undispatched_id: event.undispatched_id,
-            target_endpoint_id: event.target_endpoint_id,
+            target_endpoint_id: targetEndpointId,
             target_state: target.state,
         });
-        emitUndispatched({
-            type: "undispatched_changed",
-            version: 1,
-            action: "dispatch",
-            undispatched_id: event.undispatched_id,
-            dispatched_to_endpoint_id: event.target_endpoint_id,
-        });
         return { ok: true };
     });
     controlEventDispatcher.on("spawn_request", async (event) => {
@@ -461,6 +872,17 @@ export async function startBrokerDaemon(opts) {
                 throw new BrokerHttpError(400, "ipc_channel_missing", `no open IPC channel for plugin_pid=${body.plugin_pid}; ` +
                     "open WS /v1/stream with x-plugin-pid header first");
             }
+            const spawnDriver = spawnDriverRegistry?.lookup(body.kind);
+            const spawnAvailability = spawnDriver
+                ? await spawnDriver.isAvailable().catch(() => ({ available: false }))
+                : null;
+            const runtimeCapabilitiesSet = new Set(body.runtime_capabilities?.filter((capability) => capability !== REMOTE_SPAWN_CAPABILITY) ?? []);
+            if (spawnAvailability?.available) {
+                runtimeCapabilitiesSet.add(REMOTE_SPAWN_CAPABILITY);
+            }
+            const runtimeCapabilities = runtimeCapabilitiesSet.size > 0
+                ? Array.from(runtimeCapabilitiesSet)
+                : undefined;
             const apiResp = await apiPort.register({
                 runtime_kind: body.kind,
                 endpoint_nonce: randomUUID(),
@@ -471,12 +893,18 @@ export async function startBrokerDaemon(opts) {
                 tracking_ref: body.tracking_ref,
                 session_name: body.session_name,
                 task_hint: body.task_hint,
+                runtime_capabilities: runtimeCapabilities,
+                execution_surface: body.execution_surface,
+                background_exchange_mode: body.background_exchange_mode,
             });
             registry.register({
                 endpoint_id: apiResp.endpoint_id,
                 agent_id: body.agent_id,
                 plugin_pid: body.plugin_pid,
                 ipc_ws: boundWs,
+                runtime_capabilities: runtimeCapabilities,
+                execution_surface: body.execution_surface,
+                background_exchange_mode: body.background_exchange_mode,
                 display_metadata: {
                     session_name: body.session_name,
                     kind: body.kind,
@@ -593,8 +1021,86 @@ export async function startBrokerDaemon(opts) {
                 }
                 throw err;
             }
+            if (inReplyTo) {
+                const assignmentKey = runtimeAssignmentByEndpointCorrelation.get(endpointCorrelationKey(body.endpoint_id, inReplyTo));
+                const assignment = assignmentKey !== undefined
+                    ? runtimeAssignments.get(assignmentKey)
+                    : undefined;
+                if (assignmentKey !== undefined && assignment?.accepted) {
+                    emitReplyEmittedWithRetry(assignmentKey, {
+                        type: "runtime_processing_state",
+                        version: 1,
+                        source_message_id: assignment.context.sourceMessageId,
+                        assignment_id: assignment.context.assignmentId,
+                        state_event_id: `${assignment.context.assignmentId}:reply:${ack.messageId}`,
+                        state_sequence: Number.MAX_SAFE_INTEGER,
+                        target_ref: assignment.context.targetRef,
+                        state: "reply_emitted",
+                        occurred_at: Date.now(),
+                        undispatched_id: assignment.context.undispatchedId,
+                        outbound_message_id: ack.messageId,
+                    });
+                }
+            }
             return { messageId: ack.messageId, status: ack.status };
         },
+        async reportProcessingState(body) {
+            const entry = registry.get(body.endpoint_id);
+            if (!entry) {
+                throw new BrokerHttpError(404, "unknown_endpoint", `unknown endpoint_id: ${body.endpoint_id}`);
+            }
+            if (entry.plugin_pid !== body.plugin_pid) {
+                throw new BrokerHttpError(403, "endpoint_pid_mismatch", "plugin_pid does not own endpoint_id");
+            }
+            const key = runtimeAssignmentKey(body.source_message_id, body.assignment_id);
+            const assignment = runtimeAssignments.get(key);
+            if (!assignment) {
+                throw new BrokerHttpError(409, "runtime_assignment_unknown", "source assignment is not known to this broker");
+            }
+            if (assignment.context.targetRef.projection_endpoint_id !== body.endpoint_id) {
+                throw new BrokerHttpError(403, "runtime_assignment_target_mismatch", "endpoint_id does not match the assigned target");
+            }
+            if (body.state === "reply_emitted") {
+                throw new BrokerHttpError(403, "reply_emitted_broker_owned", "reply_emitted is emitted by the broker after outbound wire send");
+            }
+            const event = {
+                type: "runtime_processing_state",
+                version: 1,
+                source_message_id: body.source_message_id,
+                assignment_id: body.assignment_id,
+                state_event_id: body.state_event_id,
+                state_sequence: body.state_sequence,
+                target_ref: assignment.context.targetRef,
+                state: body.state,
+                occurred_at: body.occurred_at ?? Date.now(),
+                undispatched_id: body.undispatched_id ?? assignment.context.undispatchedId,
+                ...(body.retryable !== undefined && { retryable: body.retryable }),
+                ...(body.reason_code !== undefined && {
+                    reason_code: body.reason_code,
+                }),
+                ...(body.detail !== undefined && { detail: body.detail }),
+                ...(body.reply_request_id !== undefined && {
+                    reply_request_id: body.reply_request_id,
+                }),
+                ...(body.outbound_message_id !== undefined && {
+                    outbound_message_id: body.outbound_message_id,
+                }),
+            };
+            const outcome = await emitRuntimeProcessingState(event);
+            if (!outcome.ok) {
+                throw new BrokerHttpError(outcome.terminal ? 409 : 503, "runtime_processing_state_rejected", outcome.detail ?? "runtime processing state was not accepted");
+            }
+            if (body.state === "adapter_accepted") {
+                assignment.accepted = true;
+                if (assignment.acceptanceTimer) {
+                    clearTimeout(assignment.acceptanceTimer);
+                    assignment.acceptanceTimer = undefined;
+                }
+            }
+            if (body.state === "processing_failed" || body.state === "expired") {
+                forgetRuntimeAssignment(key);
+            }
+        },
         async reattachEndpoint(endpoint_id, plugin_pid, _ipcWsHint) {
             const ipcWs = channels.get(plugin_pid);
             if (!ipcWs) {
@@ -617,18 +1123,37 @@ export async function startBrokerDaemon(opts) {
             }
             bound.add(endpoint_id);
             const buf = reconnectingBuffers.forEndpoint(endpoint_id);
-            const drained = buf.drain();
+            const drained = buf.read();
             graceTimers.cancel(endpoint_id);
             registry.markActive(endpoint_id, ipcWs);
+            let allPushed = true;
             for (const msg of drained) {
-                pushToPlugin(ipcWs, {
+                const pushed = pushToPlugin(ipcWs, {
                     event: "message_received",
                     from: msg.envelope.from,
                     content: msg.envelope.content,
                     contentType: msg.envelope.contentType,
                     metadata: msg.envelope.metadata,
+                    ...(msg.envelope.sourceMessageId !== undefined && {
+                        sourceMessageId: msg.envelope.sourceMessageId,
+                    }),
+                    ...(msg.envelope.runtimeAssignment !== undefined && {
+                        runtimeAssignment: msg.envelope.runtimeAssignment,
+                    }),
                 });
+                if (!pushed) {
+                    allPushed = false;
+                    break;
+                }
             }
+            if (!allPushed) {
+                handleTransition(endpoint_id, {
+                    type: "ipc_close",
+                    plugin_alive: livenessProbe(plugin_pid),
+                });
+                throw new BrokerHttpError(503, "reconnecting_buffer_drain_failed", "failed to push buffered messages to reattached endpoint", { retryable: true });
+            }
+            buf.delete();
             emitTransition(endpoint_id, "active", "reattach");
             logger.info("endpoint_reattached", {
                 endpoint_id,
@@ -641,13 +1166,21 @@ export async function startBrokerDaemon(opts) {
             return undispatchedInbox.list();
         },
         async dispatch(undispatched_id, target_endpoint_id) {
+            const pending = undispatchedInbox.get(undispatched_id);
+            const target = registry.get(target_endpoint_id);
             const result = await controlEventDispatcher.dispatch({
                 type: "dispatch_undispatched",
                 version: 1,
                 idempotency_key: `local-${randomUUID()}`,
                 emitted_at: Date.now(),
                 undispatched_id,
-                target_endpoint_id,
+                source_message_id: pending?.source_message_id ?? undispatched_id,
+                target_ref: {
+                    kind: "projection_endpoint",
+                    projection_endpoint_id: target_endpoint_id,
+                    background_exchange_mode: normalizeBackgroundExchangeMode(target?.background_exchange_mode),
+                },
+                delivery_intent: "external_handoff",
             });
             if (!result.ok) {
                 throw new BrokerHttpError(400, "dispatch_failed", result.detail ?? "unknown");
@@ -724,42 +1257,140 @@ export async function startBrokerDaemon(opts) {
             }
         },
     });
-    connector.on("message_received", async ({ payload }) => {
+    const inboundProcessing = new Set();
+    const blockedInboundSeqs = new Set();
+    let unsequencedInboundBlocked = false;
+    const handleInboundMessage = async (payload) => {
         const meta = payload.metadata ?? {};
         const target_endpoint_id = typeof meta.target_endpoint_id === "string"
             ? meta.target_endpoint_id
             : undefined;
         const correlation_id = typeof meta.correlation_id === "string" ? meta.correlation_id : undefined;
+        const addUndispatchedFromPayload = async (opts) => {
+            const undispatched = {
+                id: payload.messageId ?? randomUUID(),
+                ...(payload.messageId !== undefined && {
+                    source_message_id: payload.messageId,
+                }),
+                arrived_at: new Date().toISOString(),
+                sender_address: payload.from,
+                content: payload.content,
+                content_type: payload.contentType,
+                metadata: meta,
+                reason: opts.reason,
+                original_target_endpoint_id: opts.originalTargetEndpointId,
+                original_correlation_id: opts.originalCorrelationId,
+            };
+            const added = undispatchedInbox.tryAdd(undispatched);
+            if (!added) {
+                logger.warn("undispatched_inbox_at_capacity", {
+                    from: payload.from,
+                    size: undispatchedInbox.size(),
+                    capacity: undispatchedInbox.capacity(),
+                });
+                return false;
+            }
+            logger.info("undispatched_added", {
+                id: undispatched.id,
+                reason: opts.reason,
+            });
+            const wireReason = undispatched.reason === "unaddressed"
+                ? "no_target_endpoint"
+                : undispatched.reason;
+            const posted = await directPostUndispatched({
+                type: "undispatched_changed",
+                version: 1,
+                action: "add",
+                undispatched_id: undispatched.id,
+                ...(undispatched.source_message_id !== undefined && {
+                    source_message_id: undispatched.source_message_id,
+                }),
+                sender_address: undispatched.sender_address,
+                content_preview: truncateContentPreview(undispatched.content, CONTENT_PREVIEW_MAX_CODEPOINTS),
+                reason: wireReason,
+                ...(opts.originalTargetEndpointId !== undefined && {
+                    target_endpoint_id_hint: opts.originalTargetEndpointId,
+                }),
+                ...(typeof correlation_id === "string" && {
+                    in_reply_to: correlation_id,
+                }),
+                arrived_at: Date.parse(undispatched.arrived_at),
+            }, { queueOnTransientFailure: typeof payload.seq !== "number" });
+            if (!posted)
+                return false;
+            return markDeliverySeqAccepted(payload.seq);
+        };
         const decision = router.route({ target_endpoint_id, correlation_id });
         if (decision.kind === "endpoint") {
+            const isCorrelationContinuation = typeof correlation_id === "string" &&
+                decision.entry.correlation_ring.has(correlation_id);
+            if (isBackgroundExchangeTarget(decision.entry) &&
+                !isCorrelationContinuation) {
+                return addUndispatchedFromPayload({
+                    reason: "unaddressed",
+                    originalTargetEndpointId: decision.entry.endpoint_id,
+                    originalCorrelationId: correlation_id,
+                });
+            }
             recordInboundCorrelation(decision.entry.endpoint_id, meta);
             if (decision.entry.state === "active") {
-                pushToPlugin(decision.entry.ipc_ws, {
+                const pushed = pushToPlugin(decision.entry.ipc_ws, {
                     event: "message_received",
                     from: payload.from,
                     content: payload.content,
                     contentType: payload.contentType,
                     metadata: meta,
+                    ...(payload.messageId !== undefined && {
+                        sourceMessageId: payload.messageId,
+                    }),
                 });
+                if (!pushed)
+                    return false;
             }
             else {
                 const buf = reconnectingBuffers.forEndpoint(decision.entry.endpoint_id);
-                buf.append({
-                    id: randomUUID(),
-                    arrived_at: new Date().toISOString(),
-                    envelope: {
-                        from: payload.from,
-                        content: payload.content,
-                        contentType: payload.contentType,
-                        metadata: meta,
-                    },
-                });
+                try {
+                    buf.append({
+                        id: payload.messageId ?? randomUUID(),
+                        arrived_at: new Date().toISOString(),
+                        envelope: {
+                            from: payload.from,
+                            content: payload.content,
+                            contentType: payload.contentType,
+                            metadata: meta,
+                            ...(payload.messageId !== undefined && {
+                                sourceMessageId: payload.messageId,
+                            }),
+                        },
+                    });
+                }
+                catch (err) {
+                    logger.warn("buffer_for_reconnecting_endpoint_failed", {
+                        endpoint_id: decision.entry.endpoint_id,
+                        err: err instanceof Error ? err.message : String(err),
+                    });
+                    return false;
+                }
                 logger.info("buffered_for_reconnecting_endpoint", {
                     endpoint_id: decision.entry.endpoint_id,
                     buffer_size: buf.size(),
                 });
             }
-            return;
+            if (payload.messageId !== undefined) {
+                const posted = await directPostRuntimeInboundRouted({
+                    type: "runtime_inbound_routed",
+                    version: 1,
+                    source_message_id: payload.messageId,
+                    routed_to_endpoint_id: decision.entry.endpoint_id,
+                    route_kind: decision.entry.state === "active"
+                        ? "active_endpoint"
+                        : "reconnecting_endpoint",
+                    routed_at: Date.now(),
+                }, { queueOnTransientFailure: typeof payload.seq !== "number" });
+                if (!posted && typeof payload.seq === "number")
+                    return false;
+            }
+            return markDeliverySeqAccepted(payload.seq);
         }
         let reason = "unaddressed";
         let originalTarget;
@@ -778,49 +1409,59 @@ export async function startBrokerDaemon(opts) {
                 });
             }
         }
-        const undispatched = {
-            id: randomUUID(),
-            arrived_at: new Date().toISOString(),
-            sender_address: payload.from,
-            content: payload.content,
-            content_type: payload.contentType,
-            metadata: meta,
+        return addUndispatchedFromPayload({
             reason,
-            original_target_endpoint_id: originalTarget,
-            original_correlation_id: correlation_id,
-        };
-        const added = undispatchedInbox.tryAdd(undispatched);
-        if (!added) {
-            logger.warn("undispatched_inbox_at_capacity", {
+            originalTargetEndpointId: originalTarget,
+            originalCorrelationId: correlation_id,
+        });
+    };
+    connector.on("message_received", ({ payload }) => {
+        let task;
+        task = handleInboundMessage(payload)
+            .then((accepted) => {
+            if (typeof payload.seq === "number") {
+                if (accepted)
+                    blockedInboundSeqs.delete(payload.seq);
+                else
+                    blockedInboundSeqs.add(payload.seq);
+                return;
+            }
+            if (accepted)
+                unsequencedInboundBlocked = false;
+            else
+                unsequencedInboundBlocked = true;
+        })
+            .catch((err) => {
+            if (typeof payload.seq === "number") {
+                blockedInboundSeqs.add(payload.seq);
+            }
+            else {
+                unsequencedInboundBlocked = true;
+            }
+            logger.error("inbound_message_handling_failed", {
+                err: err instanceof Error ? err.message : String(err),
+                messageId: payload.messageId,
                 from: payload.from,
-                size: undispatchedInbox.size(),
-                capacity: undispatchedInbox.capacity(),
             });
+        })
+            .finally(() => {
+            inboundProcessing.delete(task);
+        });
+        inboundProcessing.add(task);
+    });
+    connector.on("delivery_pending", async ({ payload }) => {
+        if (inboundProcessing.size > 0) {
+            await Promise.allSettled([...inboundProcessing]);
         }
-        else {
-            logger.info("undispatched_added", { id: undispatched.id, reason });
-            const wireReason = undispatched.reason === "unaddressed"
-                ? "no_target_endpoint"
-                : undispatched.reason;
-            emitUndispatched({
-                type: "undispatched_changed",
-                version: 1,
-                action: "add",
-                undispatched_id: undispatched.id,
-                sender_address: undispatched.sender_address,
-                content_preview: truncateContentPreview(undispatched.content, CONTENT_PREVIEW_MAX_CODEPOINTS),
-                reason: wireReason,
-                ...(undispatched.original_target_endpoint_id !== undefined && {
-                    target_endpoint_id_hint: undispatched.original_target_endpoint_id,
-                }),
-                ...(typeof correlation_id === "string" && {
-                    in_reply_to: correlation_id,
-                }),
-                arrived_at: Date.parse(undispatched.arrived_at),
+        if (unsequencedInboundBlocked || blockedInboundSeqs.size > 0) {
+            logger.warn("delivery_ack_deferred_until_replay", {
+                upTo: payload.upTo,
+                count: payload.count,
+                blockedSeqs: [...blockedInboundSeqs].sort((a, b) => a - b),
+                unsequencedBlocked: unsequencedInboundBlocked,
             });
+            return;
         }
-    });
-    connector.on("delivery_pending", ({ payload }) => {
         connector.ackDelivery(payload.upTo);
     });
     writeDiscoveryFile(paths.discoveryFile, {
@@ -887,6 +1528,14 @@ export async function startBrokerDaemon(opts) {
             });
         }
         await networkPresenceEmitter.shutdown();
+        if (runtimeInboundRoutedEmitter.size() > 0) {
+            logger.warn("runtime_inbound_routed_queue_dropped_on_shutdown", {
+                size: runtimeInboundRoutedEmitter.size(),
+                reason,
+            });
+        }
+        await runtimeInboundRoutedEmitter.shutdown();
+        await flushRuntimeAssignmentsOnShutdown();
         const endpoints = registry.list();
         await Promise.all(endpoints.map(async (entry) => {
             try {
@@ -923,6 +1572,7 @@ export async function startBrokerDaemon(opts) {
         retryQueueSize: () => retryQueue.size(),
         undispatchedChangedQueueSize: () => undispatchedEmitter.size(),
         networkPresenceChangedQueueSize: () => networkPresenceEmitter.size(),
+        runtimeInboundRoutedQueueSize: () => runtimeInboundRoutedEmitter.size(),
         replyCorrelationCacheSize: (endpoint_id) => replyCorrelationCache.sizeForEndpoint(endpoint_id),
     };
 }