@zlink-systems/framework 0.30.3 → 0.32.0

package/dist/runtime/host/service-relocation-host-runtime.js

@@ -5,6 +5,7 @@ exports.decodeQueuedHandoffPacket = decodeQueuedHandoffPacket;
 exports.createServiceRelocationId = createServiceRelocationId;
 exports.relocationFailedFailureCode = relocationFailedFailureCode;
 exports.encodeHandoffQueuedMessages = encodeHandoffQueuedMessages;
+const internal_observation_1 = require("../internal-observation");
 const actor_reply_1 = require("../actors/actor-reply");
 const protocol_1 = require("../streams/protocol");
 const service_wire_m6a_codec_1 = require("../foundation/service-wire-m6a-codec");
@@ -24,6 +25,7 @@ const runtime_values_1 = require("../backend/runtime-values");
 const diagnostics_1 = require("../diagnostics");
 const runtime_metrics_1 = require("../diagnostics/runtime-metrics");
 const message_flow_1 = require("../diagnostics/message-flow");
+const flow_context_1 = require("../diagnostics/flow-context");
 const dispatch_error_details_1 = require("../diagnostics/dispatch-error-details");
 const state_lane_1 = require("../execution/state-lane");
 const actor_join_recovery_codec_1 = require("../foundation/actor-join-recovery-codec");
@@ -71,6 +73,23 @@ const INITIAL_MEMBERSHIP_EPOCH = 1n;
 /** Production host bridge from Retire inventory to remote RouteMesh owners. */
 class ZLinkHostServiceRelocationRuntime {
     options;
+    [internal_observation_1.inspectRuntime]() {
+        const sourceWindow = this.sourceCutoverWindows.values().next().value;
+        return Object.freeze({
+            targetStages: this.targetStages.size,
+            targetAssemblies: this.targetAssemblies.size,
+            pendingReplyRelays: this.pendingReplyRelays.size,
+            sourceActorJoinProfiles: this.sourceActorJoinProfiles.size,
+            sourceCutoverWindow: sourceWindow === undefined
+                ? undefined
+                : Object.freeze({
+                    get frameCount() {
+                        return sourceWindow.frames?.length;
+                    }
+                }),
+            terminalTargets: [...this.terminalTargets].length
+        });
+    }
     targetStages = new Map();
     terminalTargets = new bounded_replay_map_1.BoundedReplayMap(RELOCATION_TARGET_TOMBSTONE_LIMIT);
     targetPrepareOperations = new Map();
@@ -1132,8 +1151,9 @@ class ZLinkHostServiceRelocationRuntime {
                         ? failures[0]
                         : new AggregateError(failures, 'Spot source retirement failed.');
             },
-            abortSeal: async () => {
+            abortSeal: async (onCleanupError) => {
                 const failures = [];
+                const cleanupFailures = [];
                 if (spotCapture !== undefined) {
                     // Spot model §5.1: Continued completes while the source seal still holds jobs.
                     if (readinessBoundaryConsumed) {
@@ -1141,11 +1161,13 @@ class ZLinkHostServiceRelocationRuntime {
                             await activation.completeConsumedRelocationBoundary(contracts_1.ZLinkSpotRelocationReadyOutcome.Continued);
                         }
                         catch (error) {
-                            failures.push(error);
+                            cleanupFailures.push(error);
                         }
                     }
                     try {
-                        activation.abortRelocation(spotCapture);
+                        if (!activation.abortRelocation(spotCapture)) {
+                            throw new Error(`Spot '${String(activation.spotId)}' relocation seal became stale.`);
+                        }
                     }
                     catch (error) {
                         failures.push(error);
@@ -1160,7 +1182,7 @@ class ZLinkHostServiceRelocationRuntime {
                         }
                     }
                     catch (error) {
-                        failures.push(error);
+                        cleanupFailures.push(error);
                     }
                 }
                 for (const session of [...sessions].reverse()) {
@@ -1172,7 +1194,13 @@ class ZLinkHostServiceRelocationRuntime {
                     }
                 }
                 if (failures.length !== 0) {
-                    throw new AggregateError(failures, 'Spot relocation source rollback was incomplete.');
+                    throw new AggregateError([...failures, ...cleanupFailures], 'Spot relocation source rollback was incomplete.');
+                }
+                if (cleanupFailures.length !== 0) {
+                    const cleanup = new AggregateError(cleanupFailures, 'Spot relocation source rollback was incomplete.');
+                    if (onCleanupError === undefined)
+                        throw cleanup;
+                    onCleanupError(cleanup);
                 }
             }
         };
@@ -1185,7 +1213,7 @@ class ZLinkHostServiceRelocationRuntime {
         }));
         const relocationMetric = spotRegistration.relocation.kind === 'disabled'
             ? undefined
-            : this.options.metrics?.startRelocation(meshName, kind, spotRegistration.relocation.kind);
+            : this.startSourceRelocationMetric(meshName, kind, spotRegistration.relocation.kind);
         try {
             const waitForRelocationBoundary = activation.waitForRelocationBoundary;
             readinessBoundaryConsumed =
@@ -1261,7 +1289,7 @@ class ZLinkHostServiceRelocationRuntime {
         };
         const relocationMetric = actorPolicy === 'disabled'
             ? undefined
-            : this.options.metrics?.startRelocation(meshName, 'actor', actorPolicy);
+            : this.startSourceRelocationMetric(meshName, 'actor', actorPolicy, state.actorId);
         try {
             if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
                 (0, diagnostics_1.relocationDebug)('standalone_actor.wait_budget', {
@@ -1340,14 +1368,22 @@ class ZLinkHostServiceRelocationRuntime {
                 }
                 await this.requireSpotManager().completeRelocationSource(activation);
             },
-            abortSeal: async () => {
-                if (spotCapture !== undefined)
-                    activation.abortRelocation(spotCapture);
+            abortSeal: async (onCleanupError) => {
+                if (spotCapture !== undefined && !activation.abortRelocation(spotCapture)) {
+                    throw new Error(`PerActor Spot '${String(activation.spotId)}' relocation seal became stale.`);
+                }
                 if (spotMessageFollowSeal !== undefined) {
-                    const node = this.requireMeshNode(meshName);
-                    const abort = node.abortSpotMessageFollowIngress;
-                    if (abort === undefined || !abort.call(node, spotMessageFollowSeal)) {
-                        throw new Error('Spot Message Follow abort fence became stale.');
+                    try {
+                        const node = this.requireMeshNode(meshName);
+                        const abort = node.abortSpotMessageFollowIngress;
+                        if (abort === undefined || !abort.call(node, spotMessageFollowSeal)) {
+                            throw new Error('Spot Message Follow abort fence became stale.');
+                        }
+                    }
+                    catch (error) {
+                        if (onCleanupError === undefined)
+                            throw error;
+                        onCleanupError(error);
                     }
                 }
             }
@@ -1355,7 +1391,7 @@ class ZLinkHostServiceRelocationRuntime {
         const aggregateId = this.reserveRelocationId();
         const relocationMetric = spotRegistration.relocation.kind === 'disabled'
             ? undefined
-            : this.options.metrics?.startRelocation(meshName, 'user_spot', spotRegistration.relocation.kind);
+            : this.startSourceRelocationMetric(meshName, 'user_spot', spotRegistration.relocation.kind);
         try {
             await this.waitForRelocationBudgetHeadroom(String(target.rid), signal);
             const captured = await new service_relocation_object_owner_1.ServiceRelocationObjectCaptureOwner().captureUserSpotAggregate(aggregateId, 1n, spotUnit, [], [], signal);
@@ -1530,112 +1566,151 @@ class ZLinkHostServiceRelocationRuntime {
         // configured limit unchanged (spec 15 §4.2).
         const chunkLimitBytes = (0, relocation_direct_transfer_1.effectiveActorJoinChunkLimitBytes)(limits.relocationPayloadChunkLimitBytes, advertisedReceiveChunkLimitBytes);
         const plan = (0, relocation_direct_transfer_1.planRelocationChunks)(encoded, chunkLimitBytes);
-        // The Restore absolute deadline is this unit's relocation deadline: the
-        // caller's signal, or the control deadline without one. From it the
-        // source settles the authority with its Preserve fence (Location runtime
-        // §10). The target keeps no deadline of its own.
-        const restoreDeadlineReached = () => signal?.aborted === true || performance.now() >= controlDeadlineAtMs;
-        let readyReceived = false;
-        let settlement;
+        // The unit's original Restore absolute deadline authorizes the source
+        // Preserve fence (Location runtime §10). Caller cancellation cannot
+        // advance that deadline. The target keeps no deadline of its own.
+        const restoreDeadlineReached = () => performance.now() >= controlDeadlineAtMs;
+        let settlement = 'source';
+        let failure;
+        let window;
         try {
-            const prepare = {
-                kind: 'prepare',
-                relocation,
-                targetAttemptGeneration,
-                coordinator,
-                target: targetFence,
-                initiatorRole: 'source',
-                object,
-                sourceNodeRid: coordinator.nodeRid,
-                sourceNodeGeneration: coordinator.nodeGeneration,
-                payloadTotalLength: BigInt(plan.totalLength),
-                payloadChunkCount: plan.chunkCount,
-                payloadChecksumCrc32c: plan.checksumCrc32c,
-                applicationVersion: target.applicationVersion,
-                deadlineUnixMs: BigInt(Date.now() + Math.max(0, Math.ceil(controlDeadlineAtMs - performance.now())))
-            };
-            // Start the Restore request first (it also registers the temporary
-            // queue and the assembly on the target), then stream the payload chunks
-            // on the same ordered connection. The Ready reply arrives only after
-            // the target assembled, verified, and restored the payload.
-            const ready = await this.sendControl(meshName, target.rid, prepare, signal, controlDeadlineAtMs, async (attemptSignal) => {
-                if ((0, diagnostics_1.isRelocationDebugEnabled)())
-                    (0, diagnostics_1.relocationDebug)('coordinator.state_transfer_begin', {
-                        aggregateId: captured.envelope.aggregateId
-                    });
-                await this.sendRelocationStateChunks(meshName, target, prepare, encoded, plan, attemptSignal);
-                if ((0, diagnostics_1.isRelocationDebugEnabled)())
-                    (0, diagnostics_1.relocationDebug)('coordinator.await_ready', {
+            try {
+                const prepare = {
+                    kind: 'prepare',
+                    relocation,
+                    targetAttemptGeneration,
+                    coordinator,
+                    target: targetFence,
+                    initiatorRole: 'source',
+                    object,
+                    sourceNodeRid: coordinator.nodeRid,
+                    sourceNodeGeneration: coordinator.nodeGeneration,
+                    payloadTotalLength: BigInt(plan.totalLength),
+                    payloadChunkCount: plan.chunkCount,
+                    payloadChecksumCrc32c: plan.checksumCrc32c,
+                    applicationVersion: target.applicationVersion,
+                    deadlineUnixMs: BigInt(Date.now() + Math.max(0, Math.ceil(controlDeadlineAtMs - performance.now())))
+                };
+                // Start the Restore request first (it also registers the temporary
+                // queue and the assembly on the target), then stream the payload chunks
+                // on the same ordered connection. The Ready reply arrives only after
+                // the target assembled, verified, and restored the payload.
+                const ready = await this.sendControl(meshName, target.rid, prepare, signal, controlDeadlineAtMs, async (attemptSignal) => {
+                    settlement = undefined;
+                    if ((0, diagnostics_1.isRelocationDebugEnabled)())
+                        (0, diagnostics_1.relocationDebug)('coordinator.state_transfer_begin', {
+                            aggregateId: captured.envelope.aggregateId
+                        });
+                    await this.sendRelocationStateChunks(meshName, target, prepare, encoded, plan, attemptSignal);
+                    if ((0, diagnostics_1.isRelocationDebugEnabled)())
+                        (0, diagnostics_1.relocationDebug)('coordinator.await_ready', {
+                            aggregateId: captured.envelope.aggregateId
+                        });
+                });
+                if (ready.kind === 'failed') {
+                    settlement = 'source';
+                    throw relocationFailureException(ready);
+                }
+                validateControlResponse(prepare, ready);
+                if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
+                    (0, diagnostics_1.relocationDebug)('coordinator.ready_received', {
                         aggregateId: captured.envelope.aggregateId
                     });
-            });
-            if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
-                (0, diagnostics_1.relocationDebug)('coordinator.ready_received', {
-                    aggregateId: captured.envelope.aggregateId
+                }
+                actorJoinProfile?.relayReady();
+                const boundaryBatch = [];
+                for (const session of sessions) {
+                    const participant = captured.envelope.participants.find((value) => value.objectKind === 'actor' &&
+                        (0, authority_key_codec_1.decodeAuthorityKey)({ value: value.key }).globalId ===
+                            session.state.actorId);
+                    if (participant === undefined) {
+                        throw new Error(`Relocation Actor '${session.state.actorId}' is missing from its envelope.`);
+                    }
+                    for (const packet of session.prepared.takeRelocationRelay()) {
+                        const frozenRecord = canonicalRelayFrozenRecord(participant, packet, coordinator, targetFence);
+                        const data = {
+                            kind: 'data',
+                            relocation,
+                            targetAttemptGeneration,
+                            coordinator,
+                            senderRole: 'source',
+                            object: relocationObjectForParticipant(participant),
+                            frozenRecord
+                        };
+                        boundaryBatch.push({
+                            frame: (0, service_relocation_control_1.encodeServiceRelocationControlRequest)(data),
+                            canonicalBytes: frozenRecord.canonicalBytes
+                        });
+                    }
+                }
+                const cutoverFrame = (0, service_relocation_control_1.encodeServiceRelocationControlRequest)({
+                    kind: 'cutover',
+                    relocation,
+                    targetAttemptGeneration,
+                    coordinator,
+                    senderRole: 'source',
+                    object,
+                    boundaryRecordCount: BigInt(boundaryBatch.length),
+                    boundaryChecksumCrc32c: (0, service_relocation_runtime_1.crc32c)(Buffer.concat(boundaryBatch.map((value) => value.canonicalBytes)))
                 });
-            }
-            validateControlResponse(prepare, ready);
-            // Ready arms the target's cutover-wait fallback. From this point
-            // forward the source can no longer reopen dispatch without risking two
-            // owners.
-            readyReceived = true;
-            actorJoinProfile?.relayReady();
-            const boundaryBatch = [];
-            for (const session of sessions) {
-                const participant = captured.envelope.participants.find((value) => value.objectKind === 'actor' &&
-                    (0, authority_key_codec_1.decodeAuthorityKey)({ value: value.key }).globalId ===
-                        session.state.actorId);
-                if (participant === undefined) {
-                    throw new Error(`Relocation Actor '${session.state.actorId}' is missing from its envelope.`);
+                // The cutover submit result is not the authority result. The memory
+                // copy of the boundary batch and cutover stays until the authority
+                // settlement below (spec 28 §4.4).
+                const submittedRouteGeneration = this.requireMeshNode(meshName).observePeerRoute?.(target.rid).generation;
+                try {
+                    for (const record of boundaryBatch) {
+                        await this.submitControlFrame(meshName, target.rid, record.frame, 'relay');
+                    }
+                    await this.submitControlFrame(meshName, target.rid, cutoverFrame, 'cutover');
                 }
-                for (const packet of session.prepared.takeRelocationRelay()) {
-                    const frozenRecord = canonicalRelayFrozenRecord(participant, packet, coordinator, targetFence);
-                    const data = {
-                        kind: 'data',
-                        relocation,
-                        targetAttemptGeneration,
-                        coordinator,
-                        senderRole: 'source',
-                        object: relocationObjectForParticipant(participant),
-                        frozenRecord
-                    };
-                    boundaryBatch.push({
-                        frame: (0, service_relocation_control_1.encodeServiceRelocationControlRequest)(data),
-                        canonicalBytes: frozenRecord.canonicalBytes
-                    });
+                catch (error) {
+                    if (!(0, durable_service_request_1.durableRequestCanReplay)(error))
+                        throw error;
+                    console.warn('[zlink.runtime.relocation.cutover_submit_failed]', relayAuthorityId, error);
                 }
+                window = this.beginSourceCutoverWindow(meshName, target.rid, relayAuthorityId, [...boundaryBatch.map((value) => value.frame), cutoverFrame], submittedRouteGeneration, limits);
+            }
+            catch (error) {
+                failure = { error };
             }
-            const cutoverFrame = (0, service_relocation_control_1.encodeServiceRelocationControlRequest)({
-                kind: 'cutover',
-                relocation,
-                targetAttemptGeneration,
-                coordinator,
-                senderRole: 'source',
-                object,
-                boundaryRecordCount: BigInt(boundaryBatch.length),
-                boundaryChecksumCrc32c: (0, service_relocation_runtime_1.crc32c)(Buffer.concat(boundaryBatch.map((value) => value.canonicalBytes)))
-            });
-            // The cutover submit result is not the authority result. The memory
-            // copy of the boundary batch and cutover stays until the authority
-            // settlement below (spec 28 §4.4).
-            const submittedRouteGeneration = this.requireMeshNode(meshName).observePeerRoute?.(target.rid).generation;
             try {
-                for (const record of boundaryBatch) {
-                    await this.submitControlFrame(meshName, target.rid, record.frame, 'relay');
+                try {
+                    if (settlement === undefined) {
+                        settlement = await this.settleSourceAuthority(captured.envelope, primary, targetFence, restoreDeadlineReached, window, relocationMetric);
+                    }
+                    else if (failure !== undefined) {
+                        // Restoration without authority settlement did not confirm Preserve.
+                        throw failure.error;
+                    }
+                }
+                finally {
+                    if (settlement === 'source') {
+                        // Source restoration is authorized: captured queue, timers and held records
+                        // return in their accepted order, and source dispatch reopens.
+                        try {
+                            await captured.abortSource((error) => {
+                                failure ??= { error };
+                            });
+                        }
+                        catch (error) {
+                            throw failure === undefined ? error : failure.error;
+                        }
+                    }
                 }
-                await this.submitControlFrame(meshName, target.rid, cutoverFrame, 'cutover');
+                this.completeSourceRelocationMetric(relocationMetric, settlement);
             }
             catch (error) {
-                if (!(0, durable_service_request_1.durableRequestCanReplay)(error))
-                    throw error;
-                console.warn('[zlink.runtime.relocation.cutover_submit_failed]', relayAuthorityId, error);
-            }
-            const window = this.beginSourceCutoverWindow(meshName, target.rid, relayAuthorityId, [...boundaryBatch.map((value) => value.frame), cutoverFrame], submittedRouteGeneration, limits);
-            try {
-                settlement = await this.settleSourceAuthority(captured.envelope, primary, targetFence, restoreDeadlineReached, window, relocationMetric);
+                failure = { error };
+                throw error;
             }
             finally {
-                this.closeSourceCutoverWindow(relayAuthorityId, window);
+                try {
+                    if (window !== undefined)
+                        this.closeSourceCutoverWindow(relayAuthorityId, window);
+                }
+                catch (error) {
+                    throw failure === undefined ? error : failure.error;
+                }
             }
             if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
                 (0, diagnostics_1.relocationDebug)('coordinator.authority_settled', {
@@ -1644,25 +1719,23 @@ class ZLinkHostServiceRelocationRuntime {
                 });
             }
             if (settlement === 'source') {
-                // Preserve won: the captured queue, timers and held records return to
-                // the source queue in their accepted order, and dispatch reopens.
-                await captured.abortSource();
-                this.completeSourceRelocationMetric(relocationMetric, settlement);
+                if (failure !== undefined)
+                    throw failure.error;
                 throw new Error(`Relocation '${relayAuthorityId}' ended by the source Preserve fence; the source kept the authority.`);
             }
             if (settlement === 'lost') {
-                this.completeSourceRelocationMetric(relocationMetric, settlement);
                 // Expired owner: dispatch and Store changes stop for good and every
                 // pending request without a terminal gets one Unavailable.
                 const unavailable = (0, framework_errors_internal_1.createInternalFrameworkException)(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.RouteNotConnected, `Relocation '${relayAuthorityId}' source authority ended (source owner lease expired, authority Missing, or owner changed).`, true);
                 await captured.discardSource(unavailable);
                 throw new service_relocation_coordinator_1.ServiceRelocationAuthorityError(unavailable.message, { cause: unavailable });
             }
-            this.completeSourceRelocationMetric(relocationMetric, settlement);
             for (const session of sessions)
                 session.prepared.setReplayResults([]);
             await captured.commitSource(settlement.followAuthorities);
             await this.options.reconcileStatefulAuthorityRoutes?.(signal);
+            if (failure !== undefined)
+                throw failure.error;
         }
         catch (error) {
             if (this.disposed &&
@@ -1680,11 +1753,6 @@ class ZLinkHostServiceRelocationRuntime {
         }
         finally {
             this.completeSourceRelocationMetric(relocationMetric);
-            if (!readyReceived) {
-                // Only an explicit failure before the Ready reply becomes accepted
-                // restores the source from the memory payload (spec 28 §9).
-                await captured.abortSource().catch(() => undefined);
-            }
         }
     }
     completeSourceRelocationMetric(metric, settlement, sourceStopped = false) {
@@ -1698,6 +1766,41 @@ class ZLinkHostServiceRelocationRuntime {
                         ? 'shutdown'
                         : 'failed');
     }
+    startSourceRelocationMetric(meshName, objectKind, policy, actorId) {
+        const flow = objectKind === 'actor' ? this.options.messageFlow?.() : undefined;
+        const flowContext = flow?.flowCreationEnabled() === true ? (0, flow_context_1.currentFlowContext)() : undefined;
+        const traceTerminal = (outcome) => {
+            if (flow === undefined || flowContext === undefined)
+                return;
+            const result = outcome === 'completed' ? 'succeeded' : outcome === 'shutdown' ? 'shutdown' : 'failed';
+            (0, message_flow_1.flowIfEnabled)(flow, ZLinkDispatchOptions_1.ZLinkRuntimeMessageFlowOutcome.Completed, result, flowContext.flowId)?.trace({
+                outcome: ZLinkDispatchOptions_1.ZLinkRuntimeMessageFlowOutcome.Completed,
+                result,
+                surface: ZLinkDispatchOptions_1.ZLinkDispatchErrorSurface.ActorRelocation,
+                messageKind: ZLinkDispatchOptions_1.ZLinkDispatchMessageKind.Control,
+                meshName,
+                actorId,
+                flowId: flowContext.flowId,
+                flowOrigin: flowContext.flowOrigin,
+                errorMessage: result === 'failed' ? `source_relocation_${outcome}` : undefined
+            });
+        };
+        const metric = this.options.metrics?.startRelocation(meshName, objectKind, policy, traceTerminal);
+        if (metric !== undefined)
+            return metric;
+        if (flow === undefined || flowContext === undefined)
+            return undefined;
+        let completed = false;
+        return {
+            recordBytes: () => { },
+            complete: (outcome) => {
+                if (completed)
+                    return;
+                completed = true;
+                traceTerminal(outcome);
+            }
+        };
+    }
     /** Streams the payload chunks of one stage under the effective in-flight budgets. */
     async sendRelocationStateChunks(meshName, target, identity, encoded, plan, signal) {
         for (let ordinal = 0; ordinal < plan.chunkCount; ordinal += 1) {
@@ -1825,7 +1928,7 @@ class ZLinkHostServiceRelocationRuntime {
         }
     }
     /**
-     * The one source authority settlement after Ready (Location runtime
+     * The one source authority settlement after request creation (Location runtime
      * §6.1·§10, spec 28 §4.4). It reads the Store until the target commit or
      * the Restore deadline. From the Restore deadline it runs the source
      * Preserve fence: a CAS on the StoreVersion expected by target NewOwner
@@ -1844,7 +1947,8 @@ class ZLinkHostServiceRelocationRuntime {
                 this.completeSourceRelocationMetric(relocationMetric, undefined, true);
                 throw new ServiceRelocationRuntimeStoppedError();
             }
-            await this.resendSourceCutover(window);
+            if (window !== undefined)
+                await this.resendSourceCutover(window);
             const observed = await this.readSourceSettlement(envelope, primary, target).catch((error) => {
                 if (!(0, location_store_failure_1.isIndeterminateLocationStoreFailure)(error))
                     throw error;
@@ -1941,6 +2045,9 @@ class ZLinkHostServiceRelocationRuntime {
             if (result.kind !== 'stored') {
                 throw new Error(`Participant '${participant.key}' recovery pointer release failed.`);
             }
+            if (participant.objectKind === 'actor') {
+                this.options.observeActorAuthorityVersion?.((0, authority_key_codec_1.decodeAuthorityKey)(key).globalId, result.storeVersion.value);
+            }
         }
     }
     /** Restores the hidden object and its saved-work prefix before Ready. */
@@ -2236,15 +2343,6 @@ class ZLinkHostServiceRelocationRuntime {
                 hidden.onReplayResult = (packet, result) => this.relayTerminalReply(meshName, stage, coordinator, BigInt(index + 1), packet, result, signal);
             }
             await stage.owner.normalize(stage.staging, authority, signal);
-            // The target may have visited this Actor before and still retain that
-            // tenure in its private Actor-client resolver. The target-only CAS is
-            // now authoritative, so expire the old projection before OnJoinedActor
-            // can issue an Actor request and deadlock behind source leave.
-            for (const hidden of stage.staging.hidden.values()) {
-                if (hidden.actor !== undefined) {
-                    this.options.invalidateActorRoute?.(hidden.actor.context.actorId);
-                }
-            }
             await this.finalizeActorJoinProfiles(meshName, stage, signal);
             await this.publishSessionRoutes(stage.staging);
             stage.phase = 'open';
@@ -2381,6 +2479,9 @@ class ZLinkHostServiceRelocationRuntime {
         if (result.kind !== 'stored') {
             throw new Error('Relocation target authority normalization CAS failed.');
         }
+        if (primary.allocation.objectKind === 'actor') {
+            this.options.observeActorAuthorityVersion?.((0, authority_key_codec_1.decodeAuthorityKey)(stage.staging.primaryAuthorityKey).globalId, result.storeVersion.value);
+        }
         return { ...result, kind: 'snapshot' };
     }
     async handleReplyRelay(meshName, relay, sourceNodeRid, signal) {
@@ -2583,17 +2684,23 @@ class ZLinkHostServiceRelocationRuntime {
         if (remainingMs <= 0) {
             throw (0, framework_errors_internal_1.createInternalFrameworkException)(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.DeadlineExceeded, `Relocation control ACK '${key}' timed out.`, true);
         }
-        const replyFrames = await this.requestInfrastructureControl(meshName, targetNodeRid, frames, remainingMs, signal, onSubmitted);
-        if (replyFrames.length !== 1) {
-            throw new service_wire_m6a_codec_2.ServiceWireProtocolError(`Relocation control '${key}' reply must contain exactly one frame.`);
-        }
-        const response = (0, service_relocation_control_1.decodeServiceRelocationControlResponse)(Buffer.from(replyFrames[0]));
-        if (response.kind === 'failed') {
-            validateControlFailureResponse(request, response);
-            throw relocationFailureException(response);
-        }
-        validateControlResponse(request, response);
-        return response;
+        const pending = this.options.requests.begin(remainingMs, signal, undefined, deadlineAtMs);
+        return this.options.requests.run(pending, (receiver) => {
+            void this.requestInfrastructureControl(meshName, targetNodeRid, frames, remainingMs, signal, onSubmitted, pending).then((reply) => receiver.operationCompleted(pending.id, reply, undefined), (error) => receiver.operationFailed(pending.id, error, undefined));
+        }, (raw) => {
+            const replyFrames = raw;
+            if (replyFrames.length !== 1) {
+                throw new service_wire_m6a_codec_2.ServiceWireProtocolError(`Relocation control '${key}' reply must contain exactly one frame.`);
+            }
+            const response = (0, service_relocation_control_1.decodeServiceRelocationControlResponse)(Buffer.from(replyFrames[0]));
+            if (response.kind === 'failed') {
+                validateControlFailureResponse(request, response);
+            }
+            else {
+                validateControlResponse(request, response);
+            }
+            return response;
+        }, (error) => error);
     }
     async selectTarget(meshName, localRid, targetApplicationVersion, requirements = [], signal) {
         const descriptors = await this.options.liveDescriptors(meshName, signal);
@@ -3163,13 +3270,18 @@ class ZLinkHostServiceRelocationRuntime {
         }
         return await send.call(node, targetNodeRid, record);
     }
-    async requestInfrastructureControl(meshName, targetNodeRid, frames, timeoutMs, signal, onSubmitted) {
+    async requestInfrastructureControl(meshName, targetNodeRid, frames, timeoutMs, signal, onSubmitted, requestOperation) {
         const node = this.requireMeshNode(meshName);
         const request = node.requestInfrastructureControlFrames;
         if (request === undefined) {
             throw new Error('RouteMesh backend cannot submit bare infrastructure requests.');
         }
-        return await request.call(node, targetNodeRid, frames, { timeoutMs, signal, onSubmitted });
+        return await request.call(node, targetNodeRid, frames, {
+            timeoutMs,
+            signal,
+            onSubmitted,
+            requestOperation
+        });
     }
 }
 exports.ZLinkHostServiceRelocationRuntime = ZLinkHostServiceRelocationRuntime;
@@ -3327,6 +3439,7 @@ class LocalTargetPort {
                 value: hidden.authorityKey
             });
             const actorId = hidden.actor.context.actorId;
+            this.options.observeActorAuthorityVersion?.(actorId, current.storeVersion.value);
             const node = this.options.meshNode(this.meshName);
             const restore = node?.restoreActorAuthority;
             if (node === undefined || restore === undefined) {
@@ -3818,7 +3931,7 @@ function maintenanceReplyRelay(stage, coordinator, participantId, completion, pa
     const value = completion.result.ok ? completion.result.response : undefined;
     const direct = packet.messageFollowOrigin?.requestSource !== undefined;
     const application = direct && completion.result.ok
-        ? (0, service_wire_m6a_codec_1.decodeApplicationPayloadView)((0, service_wire_m6a_codec_1.encodeMultipartApplicationPayload)((0, actor_reply_1.encodeActorReply)(Buffer.from(packet.header, 'base64'), protocol_1.ZLinkStreamMessageKind.Response, value, serializers, flowEnabled), service_wire_constants_generated_1.SERVICE_FRAMEWORK_MULTIPART_PACKET_NAME, service_wire_constants_generated_1.SERVICE_FRAMEWORK_MULTIPART_CONTENT_TYPE))
+        ? (0, service_wire_m6a_codec_1.decodeApplicationPayload)((0, service_wire_m6a_codec_1.encodeMultipartApplicationPayload)((0, actor_reply_1.encodeActorReply)(Buffer.from(packet.header, 'base64'), protocol_1.ZLinkStreamMessageKind.Response, value, serializers, flowEnabled), service_wire_constants_generated_1.SERVICE_FRAMEWORK_MULTIPART_PACKET_NAME, service_wire_constants_generated_1.SERVICE_FRAMEWORK_MULTIPART_CONTENT_TYPE))
         : undefined;
     return {
         relocation: relocationWireId(stage.staging.envelope.aggregateId),

package/dist/runtime/host/spot-address-transport.d.ts

@@ -1,3 +1,4 @@
+import { RequestRuntime } from '../foundation/request-runtime';
 import { type RoutingId } from '../../contracts';
 import type { ZLinkBackendMeshNode } from '../backend/contracts';
 import { type ZLinkMeshCompletionTable } from '../backend/mesh-completion-table';
@@ -7,6 +8,7 @@ import { type ZLinkSubmitResult } from '../messaging/submission-result';
 import type { ZLinkSpotAddressCallOptions, ZLinkSpotAddressTransport, ZLinkSpotRoutedTransport } from '../spots/spot-outbound';
 import type { ZLinkSpotRouteResolver } from '../spots/spot-routing-internal';
 export interface ZLinkHostSpotAddressTransportOptions {
+    readonly requests?: RequestRuntime;
     readonly resolver: () => ZLinkSpotRouteResolver | undefined;
     readonly routed: ZLinkSpotRoutedTransport;
     readonly meshNames: () => readonly string[];
@@ -21,6 +23,7 @@ export declare function hasObjectClientCapability(role: 'none' | 'client' | 'ser
 /** Owns global Spot authority lookup and Missing Instance placement. */
 export declare class ZLinkHostSpotAddressTransport implements ZLinkSpotAddressTransport {
     private readonly options;
+    private readonly requests;
     constructor(options: ZLinkHostSpotAddressTransportOptions);
     sendToSpotAddress(spotId: RoutingId, message: unknown, call: Omit<ZLinkSpotAddressCallOptions, 'timeoutMs' | 'signal'>): Promise<ZLinkSubmitResult>;
     private sendToSpotAddressScoped;