@masons/runtime-broker 0.2.2 → 0.2.6

package/dist/broker/broker-daemon.js

@@ -1,6 +1,7 @@
 import { spawn as spawnChild } from "node:child_process";
 import { randomUUID } from "node:crypto";
 import { basename } from "node:path";
+import { ConnectorClientUnavailableError, ConnectorSendAckTimeoutError, } from "../connector-client.js";
 import { createControlEventDispatcher, } from "./control-event-dispatcher.js";
 import { readDeliveryCursorFile, writeDeliveryCursorFile, } from "./delivery-cursor-file.js";
 import { deleteDiscoveryFile, mintBearerToken, writeDiscoveryFile, } from "./discovery-file.js";
@@ -23,13 +24,27 @@ import { createUndispatchedChangedEmitter, postUndispatchedChangedViaPort, } fro
 import { createUndispatchedInbox, } from "./undispatched-inbox.js";
 const REMOTE_SPAWN_CAPABILITY = "remote_spawn_v1";
 const DEFAULT_RUNTIME_ASSIGNMENT_ACCEPTANCE_TIMEOUT_MS = 30_000;
+const RUNTIME_ASSIGNMENT_REPLY_RECORD_TTL_MS = 24 * 60 * 60 * 1000;
 const RUNTIME_PROCESSING_RETRY_INITIAL_MS = 1_000;
 const RUNTIME_PROCESSING_RETRY_MAX_MS = 30_000;
 function isBackgroundExchangeMode(value) {
     return value === "resident_endpoint" || value === "adapter_managed_turn";
 }
-function normalizeBackgroundExchangeMode(value) {
-    return isBackgroundExchangeMode(value) ? value : "resident_endpoint";
+function buildLocalDispatchTargetRef(targetEndpointId, target) {
+    if (target?.background_exchange_mode === "adapter_managed_turn" &&
+        target.runtime_session_id) {
+        return {
+            kind: "projection_endpoint",
+            projection_endpoint_id: targetEndpointId,
+            runtime_session_id: target.runtime_session_id,
+            background_exchange_mode: "adapter_managed_turn",
+        };
+    }
+    return {
+        kind: "projection_endpoint",
+        projection_endpoint_id: targetEndpointId,
+        background_exchange_mode: "resident_endpoint",
+    };
 }
 function derivePublicDisplayLabel(body) {
     const parts = [
@@ -76,6 +91,8 @@ export async function startBrokerDaemon(opts) {
     const controlEventDispatcher = createControlEventDispatcher();
     const runtimeAssignments = new Map();
     const runtimeAssignmentByEndpointCorrelation = new Map();
+    const runtimeAssignmentReplyRecords = new Map();
+    const runtimeAssignmentReplyInFlight = new Map();
     const spawnCorrelation = createSpawnCorrelationManager({
         timeoutMs: opts.spawnCorrelationTimeoutMs,
     });
@@ -135,6 +152,21 @@ export async function startBrokerDaemon(opts) {
         maxRetries: opts.runtimeInboundRoutedMaxRetries,
     });
     const runtimeAssignmentKey = (sourceMessageId, assignmentId) => `${sourceMessageId}\u0000${assignmentId}`;
+    const runtimeAssignmentReplyKey = (sourceMessageId, assignmentId, replyRequestId) => `${sourceMessageId}\u0000${assignmentId}\u0000${replyRequestId}`;
+    const runtimeAssignmentReplyFingerprint = (content, contentType) => JSON.stringify([content, contentType ?? "text"]);
+    const pruneRuntimeAssignmentReplyRecords = (now = Date.now()) => {
+        for (const [key, record] of runtimeAssignmentReplyRecords.entries()) {
+            const [sourceMessageId, assignmentId] = key.split("\u0000");
+            if (sourceMessageId !== undefined &&
+                assignmentId !== undefined &&
+                runtimeAssignments.has(runtimeAssignmentKey(sourceMessageId, assignmentId))) {
+                continue;
+            }
+            if (record.expiresAt <= now) {
+                runtimeAssignmentReplyRecords.delete(key);
+            }
+        }
+    };
     const endpointCorrelationKey = (endpointId, correlationId) => `${endpointId}\u0000${correlationId}`;
     const isBackgroundExchangeTarget = (entry) => Boolean(entry.runtime_capabilities?.includes("background_exchange_v1") &&
         isBackgroundExchangeMode(entry.background_exchange_mode));
@@ -158,6 +190,11 @@ export async function startBrokerDaemon(opts) {
                 runtimeAssignmentByEndpointCorrelation.delete(correlationKey);
             }
         }
+        for (const replyKey of Array.from(runtimeAssignmentReplyInFlight.keys())) {
+            if (replyKey.startsWith(`${key}\u0000`)) {
+                runtimeAssignmentReplyInFlight.delete(replyKey);
+            }
+        }
     };
     const scheduleAcceptanceTimeout = (key) => {
         if (runtimeAssignmentAcceptanceTimeoutMs <= 0)
@@ -247,10 +284,45 @@ export async function startBrokerDaemon(opts) {
             detail: outcome.detail,
         });
         if (!outcome.terminal && opts.queueOnTransientFailure !== false) {
-            runtimeInboundRoutedEmitter.enqueue(event);
+            const queued = runtimeInboundRoutedEmitter.enqueue(event, {
+                onSuccess: opts.onQueuedSuccess,
+            });
+            return opts.allowQueuedAsSuccess === true ? queued : false;
         }
         return false;
     };
+    const directClaimRuntimeInboundRoute = async (event) => {
+        const claim = opts.runtimeInboundRouteClaimPost ?? apiPort.claimRuntimeInboundRoute;
+        if (!claim) {
+            logger.warn("runtime_inbound_route_claim_unavailable", {
+                source_message_id: event.source_message_id,
+                routed_to_endpoint_id: event.routed_to_endpoint_id,
+            });
+            return "retry";
+        }
+        let outcome;
+        try {
+            outcome = await claim(event);
+        }
+        catch (err) {
+            outcome = {
+                ok: false,
+                terminal: false,
+                detail: err instanceof Error ? err.message : String(err),
+            };
+        }
+        if (outcome.ok) {
+            return outcome.action;
+        }
+        logger.warn("runtime_inbound_route_claim_failed", {
+            source_message_id: event.source_message_id,
+            routed_to_endpoint_id: event.routed_to_endpoint_id,
+            terminal: outcome.terminal,
+            status: outcome.status,
+            detail: outcome.detail,
+        });
+        return "retry";
+    };
     const markDeliverySeqAccepted = (seq) => {
         if (typeof seq !== "number")
             return true;
@@ -327,6 +399,27 @@ export async function startBrokerDaemon(opts) {
         }
         return Promise.allSettled(tasks).then(() => undefined);
     };
+    const restorePendingAssignmentBeforeAcceptance = (key, assignment, reason) => {
+        if (assignment.acceptanceTimer) {
+            clearTimeout(assignment.acceptanceTimer);
+            assignment.acceptanceTimer = undefined;
+        }
+        const readded = undispatchedInbox.tryAdd(assignment.message);
+        if (!readded) {
+            logger.error("runtime_assignment_reinsert_failed_at_capacity", {
+                undispatched_id: assignment.context.undispatchedId,
+                source_message_id: assignment.context.sourceMessageId,
+                target_endpoint_id: assignment.context.targetRef.projection_endpoint_id,
+                inbox_size: undispatchedInbox.size(),
+                inbox_capacity: undispatchedInbox.capacity(),
+                reason,
+            });
+            scheduleAcceptanceTimeout(key);
+            return false;
+        }
+        forgetRuntimeAssignment(key);
+        return true;
+    };
     const emitReplyEmittedWithRetry = (assignmentKey, event, attempt = 0) => {
         const assignment = runtimeAssignments.get(assignmentKey);
         if (!assignment)
@@ -351,8 +444,10 @@ export async function startBrokerDaemon(opts) {
                 terminal: outcome.terminal,
                 detail: outcome.detail,
             });
-            if (outcome.terminal)
+            if (outcome.terminal) {
+                forgetRuntimeAssignment(assignmentKey);
                 return;
+            }
             const delay = Math.min(RUNTIME_PROCESSING_RETRY_INITIAL_MS * 2 ** attempt, RUNTIME_PROCESSING_RETRY_MAX_MS);
             const timer = setTimeout(() => {
                 emitReplyEmittedWithRetry(assignmentKey, event, attempt + 1);
@@ -380,6 +475,9 @@ export async function startBrokerDaemon(opts) {
                     terminal: outcome.terminal,
                     detail: outcome.detail,
                 });
+                if (outcome.terminal) {
+                    forgetRuntimeAssignment(key);
+                }
             }
         }
         for (const endpointId of endpointIds) {
@@ -606,6 +704,12 @@ export async function startBrokerDaemon(opts) {
         }
     };
     controlEventDispatcher.on("dispatch_undispatched", async (event) => {
+        if (event.broker_projection_token === undefined) {
+            return {
+                ok: false,
+                detail: "dispatch broker_projection_token missing",
+            };
+        }
         const targetEndpointId = event.target_ref?.projection_endpoint_id ?? event.target_endpoint_id;
         if (!targetEndpointId) {
             return {
@@ -668,13 +772,31 @@ export async function startBrokerDaemon(opts) {
                 detail: `target endpoint not found: ${targetEndpointId}`,
             };
         }
-        const targetRef = event.target_ref ?? {
-            kind: "projection_endpoint",
-            projection_endpoint_id: targetEndpointId,
-            background_exchange_mode: normalizeBackgroundExchangeMode(target.background_exchange_mode),
-        };
-        if (!target.runtime_capabilities?.includes("background_exchange_v1") ||
-            target.background_exchange_mode !== targetRef.background_exchange_mode) {
+        let targetRef = event.target_ref;
+        if (!targetRef) {
+            if (target.background_exchange_mode === "resident_endpoint") {
+                targetRef = {
+                    kind: "projection_endpoint",
+                    projection_endpoint_id: targetEndpointId,
+                    background_exchange_mode: "resident_endpoint",
+                };
+            }
+            else if (target.background_exchange_mode === "adapter_managed_turn" &&
+                target.runtime_session_id) {
+                targetRef = {
+                    kind: "projection_endpoint",
+                    projection_endpoint_id: targetEndpointId,
+                    runtime_session_id: target.runtime_session_id,
+                    background_exchange_mode: "adapter_managed_turn",
+                };
+            }
+        }
+        if (!targetRef ||
+            !target.runtime_capabilities?.includes("background_exchange_v1") ||
+            target.background_exchange_mode !==
+                targetRef.background_exchange_mode ||
+            (targetRef.background_exchange_mode === "adapter_managed_turn" &&
+                target.runtime_session_id !== targetRef.runtime_session_id)) {
             undispatchedInbox.tryAdd(taken);
             return {
                 ok: false,
@@ -693,6 +815,9 @@ export async function startBrokerDaemon(opts) {
             state: "assigned_to_runtime_target",
             occurred_at: assignedAt,
             undispatched_id: event.undispatched_id,
+            ...(event.broker_projection_token !== undefined && {
+                broker_projection_token: event.broker_projection_token,
+            }),
         };
         const processingResult = await emitRuntimeProcessingState(processingEvent);
         if (!processingResult.ok) {
@@ -724,12 +849,20 @@ export async function startBrokerDaemon(opts) {
         const storedAssignment = existingAssignment ?? {
             context: runtimeAssignment,
             message: taken,
+            ...(event.broker_projection_token !== undefined && {
+                brokerProjectionToken: event.broker_projection_token,
+            }),
             accepted: false,
             assignedAt,
+            lastStateSequence: processingEvent.state_sequence,
         };
         storedAssignment.context = runtimeAssignment;
         storedAssignment.message = taken;
+        if (event.broker_projection_token !== undefined) {
+            storedAssignment.brokerProjectionToken = event.broker_projection_token;
+        }
         storedAssignment.assignedAt = assignedAt;
+        storedAssignment.lastStateSequence = Math.max(storedAssignment.lastStateSequence, processingEvent.state_sequence);
         runtimeAssignments.set(assignmentKey, storedAssignment);
         scheduleAcceptanceTimeout(assignmentKey);
         rememberRuntimeAssignmentCorrelation(target.endpoint_id, taken.metadata.correlation_id, assignmentKey);
@@ -883,6 +1016,7 @@ export async function startBrokerDaemon(opts) {
             const runtimeCapabilities = runtimeCapabilitiesSet.size > 0
                 ? Array.from(runtimeCapabilitiesSet)
                 : undefined;
+            const runtimeSessionId = body.runtime_session_id?.trim() || undefined;
             const apiResp = await apiPort.register({
                 runtime_kind: body.kind,
                 endpoint_nonce: randomUUID(),
@@ -895,6 +1029,7 @@ export async function startBrokerDaemon(opts) {
                 task_hint: body.task_hint,
                 runtime_capabilities: runtimeCapabilities,
                 execution_surface: body.execution_surface,
+                runtime_session_id: runtimeSessionId,
                 background_exchange_mode: body.background_exchange_mode,
             });
             registry.register({
@@ -904,6 +1039,7 @@ export async function startBrokerDaemon(opts) {
                 ipc_ws: boundWs,
                 runtime_capabilities: runtimeCapabilities,
                 execution_surface: body.execution_surface,
+                runtime_session_id: runtimeSessionId,
                 background_exchange_mode: body.background_exchange_mode,
                 display_metadata: {
                     session_name: body.session_name,
@@ -1014,35 +1150,186 @@ export async function startBrokerDaemon(opts) {
             }
             catch (err) {
                 const msg = err instanceof Error ? err.message : String(err);
-                if (msg === "WebSocket not connected" || msg === "SEND_ACK timeout") {
+                if (err instanceof ConnectorClientUnavailableError) {
                     throw new BrokerHttpError(503, "connector_unavailable", msg, {
                         retryable: true,
                     });
                 }
+                if (err instanceof ConnectorSendAckTimeoutError) {
+                    throw new BrokerHttpError(503, "connector_send_outcome_unknown", msg, { retryable: false });
+                }
                 throw err;
             }
-            if (inReplyTo) {
-                const assignmentKey = runtimeAssignmentByEndpointCorrelation.get(endpointCorrelationKey(body.endpoint_id, inReplyTo));
-                const assignment = assignmentKey !== undefined
-                    ? runtimeAssignments.get(assignmentKey)
-                    : undefined;
-                if (assignmentKey !== undefined && assignment?.accepted) {
-                    emitReplyEmittedWithRetry(assignmentKey, {
+            return { messageId: ack.messageId, status: ack.status };
+        },
+        async sendRuntimeAssignmentReply(body) {
+            const entry = registry.get(body.endpoint_id);
+            if (!entry) {
+                throw new BrokerHttpError(404, "unknown_endpoint", `unknown endpoint_id: ${body.endpoint_id}`);
+            }
+            if (entry.plugin_pid !== body.plugin_pid) {
+                throw new BrokerHttpError(403, "endpoint_pid_mismatch", "plugin_pid does not own endpoint_id");
+            }
+            if (entry.state !== "active") {
+                throw new BrokerHttpError(409, "endpoint_not_active", `endpoint ${body.endpoint_id} is ${entry.state}; assignment reply is only allowed from Active`);
+            }
+            if (!body.reply_request_id) {
+                throw new BrokerHttpError(400, "reply_request_id_required", "reply_request_id is required for assignment replies");
+            }
+            const assignmentKey = runtimeAssignmentKey(body.source_message_id, body.assignment_id);
+            const replyKey = runtimeAssignmentReplyKey(body.source_message_id, body.assignment_id, body.reply_request_id);
+            pruneRuntimeAssignmentReplyRecords();
+            const requestFingerprint = runtimeAssignmentReplyFingerprint(body.content, body.contentType);
+            const assignment = runtimeAssignments.get(assignmentKey);
+            const existingRecord = runtimeAssignmentReplyRecords.get(replyKey);
+            const inFlight = runtimeAssignmentReplyInFlight.get(replyKey);
+            if (assignment) {
+                if (assignment.context.targetRef.projection_endpoint_id !==
+                    body.endpoint_id) {
+                    throw new BrokerHttpError(403, "runtime_assignment_target_mismatch", "endpoint_id does not match the assigned target");
+                }
+                if (!assignment.accepted) {
+                    throw new BrokerHttpError(409, "runtime_assignment_not_accepted", "assignment replies require adapter_accepted first");
+                }
+            }
+            else if (existingRecord || inFlight) {
+                const owner = existingRecord ?? inFlight;
+                if (owner?.endpointId !== body.endpoint_id ||
+                    owner.pluginPid !== body.plugin_pid) {
+                    throw new BrokerHttpError(403, "runtime_assignment_target_mismatch", "endpoint_id does not match the assigned target");
+                }
+            }
+            else {
+                throw new BrokerHttpError(409, "runtime_assignment_unknown", "source assignment is not known to this broker");
+            }
+            if (existingRecord &&
+                existingRecord.requestFingerprint !== requestFingerprint) {
+                throw new BrokerHttpError(409, "runtime_assignment_reply_idempotency_conflict", "reply_request_id was already used with different reply content");
+            }
+            if (existingRecord?.outcomeUnknown) {
+                throw new BrokerHttpError(503, "connector_send_outcome_unknown", existingRecord.outcomeUnknown.message, { retryable: false });
+            }
+            if (existingRecord?.result)
+                return existingRecord.result;
+            if (inFlight) {
+                if (inFlight.requestFingerprint !== requestFingerprint) {
+                    throw new BrokerHttpError(409, "runtime_assignment_reply_idempotency_conflict", "reply_request_id was already used with different reply content");
+                }
+                return inFlight.task;
+            }
+            if (!assignment) {
+                throw new BrokerHttpError(409, "runtime_assignment_unknown", "source assignment is not known to this broker");
+            }
+            if (assignment.brokerProjectionToken === undefined) {
+                throw new BrokerHttpError(409, "runtime_assignment_projection_authority_missing", "runtime assignment reply requires broker projection authority");
+            }
+            const task = (async () => {
+                let record = runtimeAssignmentReplyRecords.get(replyKey);
+                if (!record) {
+                    const replyRequestedSequence = assignment.lastStateSequence + 1;
+                    const replyRequestedEvent = {
                         type: "runtime_processing_state",
                         version: 1,
                         source_message_id: assignment.context.sourceMessageId,
                         assignment_id: assignment.context.assignmentId,
-                        state_event_id: `${assignment.context.assignmentId}:reply:${ack.messageId}`,
-                        state_sequence: Number.MAX_SAFE_INTEGER,
+                        state_event_id: `${assignment.context.assignmentId}:reply_requested:${body.reply_request_id}`,
+                        state_sequence: replyRequestedSequence,
                         target_ref: assignment.context.targetRef,
-                        state: "reply_emitted",
+                        state: "reply_requested",
                         occurred_at: Date.now(),
                         undispatched_id: assignment.context.undispatchedId,
-                        outbound_message_id: ack.messageId,
-                    });
+                        reply_request_id: body.reply_request_id,
+                        ...(assignment.brokerProjectionToken !== undefined && {
+                            broker_projection_token: assignment.brokerProjectionToken,
+                        }),
+                    };
+                    const replyRequested = await emitRuntimeProcessingState(replyRequestedEvent);
+                    if (!replyRequested.ok) {
+                        throw new BrokerHttpError(replyRequested.terminal ? 409 : 503, "runtime_reply_request_rejected", replyRequested.detail ?? "runtime reply request was not accepted");
+                    }
+                    record = {
+                        replyRequestedEvent,
+                        endpointId: body.endpoint_id,
+                        pluginPid: body.plugin_pid,
+                        requestFingerprint,
+                        wireMessageId: randomUUID(),
+                        expiresAt: Date.now() + RUNTIME_ASSIGNMENT_REPLY_RECORD_TTL_MS,
+                    };
+                    runtimeAssignmentReplyRecords.set(replyKey, record);
+                    assignment.lastStateSequence = replyRequestedSequence;
                 }
+                const correlationId = typeof assignment.context.replyCorrelationId === "string" &&
+                    assignment.context.replyCorrelationId.length > 0
+                    ? assignment.context.replyCorrelationId
+                    : randomUUID();
+                entry.correlation_ring.add(correlationId);
+                const remoteMessageId = assignment.message.metadata.message_id;
+                const metadata = {
+                    message_id: record.wireMessageId,
+                    ...(typeof remoteMessageId === "string" &&
+                        remoteMessageId.length > 0 && { in_reply_to: remoteMessageId }),
+                    correlation_id: correlationId,
+                    source_endpoint_id: body.endpoint_id,
+                    require_live: false,
+                };
+                let ack;
+                try {
+                    ack = await connector.send(assignment.message.sender_address, body.content, body.contentType ?? "text", metadata);
+                }
+                catch (err) {
+                    const msg = err instanceof Error ? err.message : String(err);
+                    if (err instanceof ConnectorClientUnavailableError) {
+                        throw new BrokerHttpError(503, "connector_unavailable", msg, {
+                            retryable: true,
+                        });
+                    }
+                    if (err instanceof ConnectorSendAckTimeoutError) {
+                        record.outcomeUnknown = { message: msg };
+                        record.expiresAt =
+                            Date.now() + RUNTIME_ASSIGNMENT_REPLY_RECORD_TTL_MS;
+                        throw new BrokerHttpError(503, "connector_send_outcome_unknown", msg, { retryable: false });
+                    }
+                    throw err;
+                }
+                const result = {
+                    messageId: record.wireMessageId,
+                    status: ack.status,
+                };
+                record.result = result;
+                record.expiresAt = Date.now() + RUNTIME_ASSIGNMENT_REPLY_RECORD_TTL_MS;
+                const replyEmittedSequence = assignment.lastStateSequence + 1;
+                emitReplyEmittedWithRetry(assignmentKey, {
+                    type: "runtime_processing_state",
+                    version: 1,
+                    source_message_id: assignment.context.sourceMessageId,
+                    assignment_id: assignment.context.assignmentId,
+                    state_event_id: `${assignment.context.assignmentId}:reply_emitted:${body.reply_request_id}`,
+                    state_sequence: replyEmittedSequence,
+                    target_ref: assignment.context.targetRef,
+                    state: "reply_emitted",
+                    occurred_at: Date.now(),
+                    undispatched_id: assignment.context.undispatchedId,
+                    reply_request_id: body.reply_request_id,
+                    outbound_message_id: record.wireMessageId,
+                    ...(assignment.brokerProjectionToken !== undefined && {
+                        broker_projection_token: assignment.brokerProjectionToken,
+                    }),
+                });
+                assignment.lastStateSequence = replyEmittedSequence;
+                return result;
+            })();
+            runtimeAssignmentReplyInFlight.set(replyKey, {
+                endpointId: body.endpoint_id,
+                pluginPid: body.plugin_pid,
+                requestFingerprint,
+                task,
+            });
+            try {
+                return await task;
+            }
+            finally {
+                runtimeAssignmentReplyInFlight.delete(replyKey);
             }
-            return { messageId: ack.messageId, status: ack.status };
         },
         async reportProcessingState(body) {
             const entry = registry.get(body.endpoint_id);
@@ -1060,8 +1347,10 @@ export async function startBrokerDaemon(opts) {
             if (assignment.context.targetRef.projection_endpoint_id !== body.endpoint_id) {
                 throw new BrokerHttpError(403, "runtime_assignment_target_mismatch", "endpoint_id does not match the assigned target");
             }
-            if (body.state === "reply_emitted") {
-                throw new BrokerHttpError(403, "reply_emitted_broker_owned", "reply_emitted is emitted by the broker after outbound wire send");
+            if (body.state === "assigned_to_runtime_target" ||
+                body.state === "reply_requested" ||
+                body.state === "reply_emitted") {
+                throw new BrokerHttpError(403, `${body.state}_broker_owned`, `${body.state} is emitted by the broker assignment lifecycle path`);
             }
             const event = {
                 type: "runtime_processing_state",
@@ -1090,6 +1379,7 @@ export async function startBrokerDaemon(opts) {
             if (!outcome.ok) {
                 throw new BrokerHttpError(outcome.terminal ? 409 : 503, "runtime_processing_state_rejected", outcome.detail ?? "runtime processing state was not accepted");
             }
+            assignment.lastStateSequence = Math.max(assignment.lastStateSequence, body.state_sequence);
             if (body.state === "adapter_accepted") {
                 assignment.accepted = true;
                 if (assignment.acceptanceTimer) {
@@ -1097,6 +1387,21 @@ export async function startBrokerDaemon(opts) {
                     assignment.acceptanceTimer = undefined;
                 }
             }
+            if (!assignment.accepted &&
+                (body.state === "processing_failed" ||
+                    body.state === "waiting_for_runtime_capability" ||
+                    body.state === "target_lost_before_acceptance")) {
+                if (body.state === "processing_failed" && body.retryable !== true) {
+                    if (assignment.acceptanceTimer) {
+                        clearTimeout(assignment.acceptanceTimer);
+                        assignment.acceptanceTimer = undefined;
+                    }
+                    forgetRuntimeAssignment(key);
+                    return;
+                }
+                restorePendingAssignmentBeforeAcceptance(key, assignment, body.state);
+                return;
+            }
             if (body.state === "processing_failed" || body.state === "expired") {
                 forgetRuntimeAssignment(key);
             }
@@ -1175,12 +1480,9 @@ export async function startBrokerDaemon(opts) {
                 emitted_at: Date.now(),
                 undispatched_id,
                 source_message_id: pending?.source_message_id ?? undispatched_id,
-                target_ref: {
-                    kind: "projection_endpoint",
-                    projection_endpoint_id: target_endpoint_id,
-                    background_exchange_mode: normalizeBackgroundExchangeMode(target?.background_exchange_mode),
-                },
+                target_ref: buildLocalDispatchTargetRef(target_endpoint_id, target),
                 delivery_intent: "external_handoff",
+                broker_projection_token: randomUUID(),
             });
             if (!result.ok) {
                 throw new BrokerHttpError(400, "dispatch_failed", result.detail ?? "unknown");
@@ -1259,7 +1561,36 @@ export async function startBrokerDaemon(opts) {
     });
     const inboundProcessing = new Set();
     const blockedInboundSeqs = new Set();
+    const asyncAcceptedInboundSeqs = new Set();
     let unsequencedInboundBlocked = false;
+    let deferredDeliveryPendingUpTo;
+    const deliveredInboundSourceRoutes = new Map();
+    const deliveredInboundSourceRouteOrder = [];
+    const DELIVERED_INBOUND_SOURCE_ROUTE_CAP = 4096;
+    const maybeAckDeferredDeliveryPending = () => {
+        if (deferredDeliveryPendingUpTo === undefined ||
+            unsequencedInboundBlocked ||
+            blockedInboundSeqs.size > 0) {
+            return;
+        }
+        const upTo = deferredDeliveryPendingUpTo;
+        deferredDeliveryPendingUpTo = undefined;
+        connector.ackDelivery(upTo);
+    };
+    const rememberDeliveredInboundSourceRoute = (sourceMessageId, event) => {
+        if (deliveredInboundSourceRoutes.has(sourceMessageId)) {
+            deliveredInboundSourceRoutes.set(sourceMessageId, event);
+            return;
+        }
+        deliveredInboundSourceRoutes.set(sourceMessageId, event);
+        deliveredInboundSourceRouteOrder.push(sourceMessageId);
+        while (deliveredInboundSourceRouteOrder.length >
+            DELIVERED_INBOUND_SOURCE_ROUTE_CAP) {
+            const evicted = deliveredInboundSourceRouteOrder.shift();
+            if (evicted)
+                deliveredInboundSourceRoutes.delete(evicted);
+        }
+    };
     const handleInboundMessage = async (payload) => {
         const meta = payload.metadata ?? {};
         const target_endpoint_id = typeof meta.target_endpoint_id === "string"
@@ -1332,6 +1663,58 @@ export async function startBrokerDaemon(opts) {
                     originalCorrelationId: correlation_id,
                 });
             }
+            const routeKind = decision.entry.state === "active"
+                ? "active_endpoint"
+                : "reconnecting_endpoint";
+            const deliverySeq = payload.seq;
+            const onQueuedFinalizationSuccess = typeof deliverySeq === "number"
+                ? () => {
+                    const wasAlreadyBlocked = blockedInboundSeqs.delete(deliverySeq);
+                    if (!wasAlreadyBlocked) {
+                        asyncAcceptedInboundSeqs.add(deliverySeq);
+                    }
+                    markDeliverySeqAccepted(deliverySeq);
+                    maybeAckDeferredDeliveryPending();
+                }
+                : undefined;
+            if (payload.messageId !== undefined) {
+                const delivered = deliveredInboundSourceRoutes.get(payload.messageId);
+                if (delivered) {
+                    const posted = await directPostRuntimeInboundRouted(delivered, {
+                        allowQueuedAsSuccess: payload.seq === undefined,
+                        onQueuedSuccess: onQueuedFinalizationSuccess,
+                    });
+                    if (!posted)
+                        return false;
+                    const accepted = markDeliverySeqAccepted(payload.seq);
+                    if (accepted)
+                        maybeAckDeferredDeliveryPending();
+                    return accepted;
+                }
+                const claim = await directClaimRuntimeInboundRoute({
+                    type: "runtime_inbound_route_claim",
+                    version: 1,
+                    source_message_id: payload.messageId,
+                    routed_to_endpoint_id: decision.entry.endpoint_id,
+                    route_kind: routeKind,
+                    claimed_at: Date.now(),
+                });
+                if (claim === "retry")
+                    return false;
+                if (claim === "skip")
+                    return markDeliverySeqAccepted(payload.seq);
+            }
+            let routedEvent;
+            if (payload.messageId !== undefined) {
+                routedEvent = {
+                    type: "runtime_inbound_routed",
+                    version: 1,
+                    source_message_id: payload.messageId,
+                    routed_to_endpoint_id: decision.entry.endpoint_id,
+                    route_kind: routeKind,
+                    routed_at: Date.now(),
+                };
+            }
             recordInboundCorrelation(decision.entry.endpoint_id, meta);
             if (decision.entry.state === "active") {
                 const pushed = pushToPlugin(decision.entry.ipc_ws, {
@@ -1376,18 +1759,13 @@ export async function startBrokerDaemon(opts) {
                     buffer_size: buf.size(),
                 });
             }
-            if (payload.messageId !== undefined) {
-                const posted = await directPostRuntimeInboundRouted({
-                    type: "runtime_inbound_routed",
-                    version: 1,
-                    source_message_id: payload.messageId,
-                    routed_to_endpoint_id: decision.entry.endpoint_id,
-                    route_kind: decision.entry.state === "active"
-                        ? "active_endpoint"
-                        : "reconnecting_endpoint",
-                    routed_at: Date.now(),
-                }, { queueOnTransientFailure: typeof payload.seq !== "number" });
-                if (!posted && typeof payload.seq === "number")
+            if (routedEvent !== undefined && payload.messageId !== undefined) {
+                rememberDeliveredInboundSourceRoute(payload.messageId, routedEvent);
+                const posted = await directPostRuntimeInboundRouted(routedEvent, {
+                    allowQueuedAsSuccess: payload.seq === undefined,
+                    onQueuedSuccess: onQueuedFinalizationSuccess,
+                });
+                if (!posted)
                     return false;
             }
             return markDeliverySeqAccepted(payload.seq);
@@ -1420,10 +1798,13 @@ export async function startBrokerDaemon(opts) {
         task = handleInboundMessage(payload)
             .then((accepted) => {
             if (typeof payload.seq === "number") {
-                if (accepted)
+                if (accepted || asyncAcceptedInboundSeqs.has(payload.seq)) {
                     blockedInboundSeqs.delete(payload.seq);
-                else
+                    asyncAcceptedInboundSeqs.delete(payload.seq);
+                }
+                else {
                     blockedInboundSeqs.add(payload.seq);
+                }
                 return;
             }
             if (accepted)
@@ -1454,6 +1835,7 @@ export async function startBrokerDaemon(opts) {
             await Promise.allSettled([...inboundProcessing]);
         }
         if (unsequencedInboundBlocked || blockedInboundSeqs.size > 0) {
+            deferredDeliveryPendingUpTo = payload.upTo;
             logger.warn("delivery_ack_deferred_until_replay", {
                 upTo: payload.upTo,
                 count: payload.count,
@@ -1462,6 +1844,7 @@ export async function startBrokerDaemon(opts) {
             });
             return;
         }
+        deferredDeliveryPendingUpTo = undefined;
         connector.ackDelivery(payload.upTo);
     });
     writeDiscoveryFile(paths.discoveryFile, {