@zlink-systems/framework 0.30.3 → 0.32.0

package/src/runtime/host/service-relocation-host-runtime.ts

@@ -1,7 +1,10 @@
+import type { RequestRuntime } from '../foundation/request-runtime';
+import type { PendingOperation } from '../foundation/operation-registry';
+import { inspectRuntime } from '../internal-observation';
 import { encodeActorReply } from '../actors/actor-reply';
 import { ZLinkStreamMessageKind } from '../streams/protocol';
 import {
-  decodeApplicationPayloadView,
+  decodeApplicationPayload,
   encodeMultipartApplicationPayload
 } from '../foundation/service-wire-m6a-codec';
 import {
@@ -77,6 +80,7 @@ import {
   flowIfEnabled,
   type ZLinkMessageFlowTracer
 } from '../diagnostics/message-flow';
+import { currentFlowContext } from '../diagnostics/flow-context';
 import { dispatchErrorDetails } from '../diagnostics/dispatch-error-details';
 import { isStructuralGuardEnabled, trackDiagnosticCompletion } from '../execution/state-lane';
 import {
@@ -243,6 +247,7 @@ const SERVICE_CONTROL_TERMINAL_CAPACITY = 4096;
 const INITIAL_MEMBERSHIP_EPOCH = 1n;
 
 interface ZLinkHostRelocationOptions {
+  readonly requests: RequestRuntime;
   readonly flowCreationEnabled?: () => boolean;
   readonly messageFlow?: () => ZLinkMessageFlowTracer | undefined;
   readonly registration: ZLinkFrameworkRegistration;
@@ -272,7 +277,7 @@ interface ZLinkHostRelocationOptions {
     clear?(): void | Promise<void>;
   };
   readonly trackInstanceSpot?: (input: ZLinkTrackedInstanceAuthority) => void;
-  readonly invalidateActorRoute?: (actorId: string) => void;
+  readonly observeActorAuthorityVersion?: (actorId: string, storeVersion: string) => void;
   readonly reconcileStatefulAuthorityRoutes?: (signal?: AbortSignal) => Promise<void>;
   readonly runtimeEventPublisher?: ZLinkRuntimeEventPublisher;
   readonly metrics?: ZLinkRuntimeMetrics;
@@ -414,13 +419,32 @@ interface TerminalSessionRelocationControl {
 
 /** Production host bridge from Retire inventory to remote RouteMesh owners. */
 export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocation {
-  private readonly targetStages = new Map<string, LocalStage>();
+  [inspectRuntime]() {
+    const sourceWindow = this.sourceCutoverWindows.values().next().value;
+    return Object.freeze({
+      targetStages: this.targetStages.size,
+      targetAssemblies: this.targetAssemblies.size,
+      pendingReplyRelays: this.pendingReplyRelays.size,
+      sourceActorJoinProfiles: this.sourceActorJoinProfiles.size,
+      sourceCutoverWindow:
+        sourceWindow === undefined
+          ? undefined
+          : Object.freeze({
+              get frameCount() {
+                return sourceWindow.frames?.length;
+              }
+            }),
+      terminalTargets: [...this.terminalTargets].length
+    });
+  }
+
+  readonly targetStages = new Map<string, LocalStage>();
   private readonly terminalTargets = new BoundedReplayMap<string, string>(
     RELOCATION_TARGET_TOMBSTONE_LIMIT
   );
-  private readonly targetPrepareOperations = new Map<string, TargetPrepareOperation>();
+  readonly targetPrepareOperations = new Map<string, TargetPrepareOperation>();
   private readonly pendingReplyRelays = new Map<string, PendingRelocationReplyRelay>();
-  private readonly pendingSessionRelocations = new Map<string, PendingSessionRelocation>();
+  readonly pendingSessionRelocations = new Map<string, PendingSessionRelocation>();
   private readonly terminalSessionRelocations = new BoundedReplayMap<
     string,
     TerminalSessionRelocationControl
@@ -434,7 +458,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     SERVICE_CONTROL_TERMINAL_CAPACITY
   );
   private readonly sourceRelocationIds = new Set<string>();
-  private readonly targetAssemblies = new Map<string, TargetPayloadAssembly>();
+  readonly targetAssemblies = new Map<string, TargetPayloadAssembly>();
   /**
    * Terminal restore failures (assembly/checksum/factory/restore error): the
    * target already sent relocationFailed (command 53) once. An exact-identity
@@ -461,7 +485,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
   private readonly terminalActorJoinSourceLeaves = new BoundedReplayMap<string, string>(
     SERVICE_CONTROL_TERMINAL_CAPACITY
   );
-  private readonly codec = new ServiceRelocationAuthorityPayloadCodec();
+  readonly codec = new ServiceRelocationAuthorityPayloadCodec();
   /**
    * Set once dispose() starts tearing assemblies down. A payload() rejection
    * observed after this point is the shutdown itself, not a chunk-assembly or
@@ -470,7 +494,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
   private disposed = false;
   private readonly targetPreparationStop = new AbortController();
 
-  constructor(private readonly options: ZLinkHostRelocationOptions) {}
+  constructor(readonly options: ZLinkHostRelocationOptions) {}
 
   async relocateActorJoin(
     input: Parameters<ZLinkActorJoinRelocation['relocateActorJoin']>[0]
@@ -1106,7 +1130,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
   }
 
   /** Sends the Ready reply once the async restore finished; failures replay on resend. */
-  private async deliverTargetReady(
+  async deliverTargetReady(
     meshName: string,
     operationKey: string,
     request: ServiceMaintenanceRelocationPrepare,
@@ -1658,7 +1682,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     }
   }
 
-  private async relocateSpotAggregate(
+  async relocateSpotAggregate(
     meshName: string,
     activation: ZLinkSpotActivation,
     kind: 'user_spot' | 'instance_spot',
@@ -1861,8 +1885,9 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
             ? failures[0]
             : new AggregateError(failures, 'Spot source retirement failed.');
       },
-      abortSeal: async () => {
+      abortSeal: async (onCleanupError) => {
         const failures: unknown[] = [];
+        const cleanupFailures: unknown[] = [];
         if (spotCapture !== undefined) {
           // Spot model §5.1: Continued completes while the source seal still holds jobs.
           if (readinessBoundaryConsumed) {
@@ -1871,11 +1896,13 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
                 ZLinkSpotRelocationReadyOutcome.Continued
               );
             } catch (error) {
-              failures.push(error);
+              cleanupFailures.push(error);
             }
           }
           try {
-            activation.abortRelocation(spotCapture);
+            if (!activation.abortRelocation(spotCapture)) {
+              throw new Error(`Spot '${String(activation.spotId)}' relocation seal became stale.`);
+            }
           } catch (error) {
             failures.push(error);
           }
@@ -1888,7 +1915,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
               throw new Error('Spot Message Follow abort fence became stale.');
             }
           } catch (error) {
-            failures.push(error);
+            cleanupFailures.push(error);
           }
         }
         for (const session of [...sessions].reverse()) {
@@ -1899,7 +1926,18 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
           }
         }
         if (failures.length !== 0) {
-          throw new AggregateError(failures, 'Spot relocation source rollback was incomplete.');
+          throw new AggregateError(
+            [...failures, ...cleanupFailures],
+            'Spot relocation source rollback was incomplete.'
+          );
+        }
+        if (cleanupFailures.length !== 0) {
+          const cleanup = new AggregateError(
+            cleanupFailures,
+            'Spot relocation source rollback was incomplete.'
+          );
+          if (onCleanupError === undefined) throw cleanup;
+          onCleanupError(cleanup);
         }
       }
     };
@@ -1925,7 +1963,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     const relocationMetric =
       spotRegistration.relocation.kind === 'disabled'
         ? undefined
-        : this.options.metrics?.startRelocation(meshName, kind, spotRegistration.relocation.kind);
+        : this.startSourceRelocationMetric(meshName, kind, spotRegistration.relocation.kind);
     try {
       const waitForRelocationBoundary = (
         activation as unknown as {
@@ -2004,7 +2042,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     }
   }
 
-  private async relocateStandaloneActor(
+  async relocateStandaloneActor(
     meshName: string,
     state: ZLinkActorRuntimeState,
     target: ZLinkMeshNodeDescriptor | undefined,
@@ -2078,7 +2116,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     const relocationMetric =
       actorPolicy === 'disabled'
         ? undefined
-        : this.options.metrics?.startRelocation(meshName, 'actor', actorPolicy);
+        : this.startSourceRelocationMetric(meshName, 'actor', actorPolicy, state.actorId);
     try {
       if (isRelocationDebugEnabled()) {
         relocationDebug('standalone_actor.wait_budget', {
@@ -2121,7 +2159,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     }
   }
 
-  private async relocatePerActorSpotShell(
+  async relocatePerActorSpotShell(
     meshName: string,
     activation: ZLinkSpotActivation,
     target: ZLinkMeshNodeDescriptor | undefined,
@@ -2216,13 +2254,22 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
         }
         await this.requireSpotManager().completeRelocationSource(activation);
       },
-      abortSeal: async () => {
-        if (spotCapture !== undefined) activation.abortRelocation(spotCapture);
+      abortSeal: async (onCleanupError) => {
+        if (spotCapture !== undefined && !activation.abortRelocation(spotCapture)) {
+          throw new Error(
+            `PerActor Spot '${String(activation.spotId)}' relocation seal became stale.`
+          );
+        }
         if (spotMessageFollowSeal !== undefined) {
-          const node = this.requireMeshNode(meshName);
-          const abort = node.abortSpotMessageFollowIngress;
-          if (abort === undefined || !abort.call(node, spotMessageFollowSeal)) {
-            throw new Error('Spot Message Follow abort fence became stale.');
+          try {
+            const node = this.requireMeshNode(meshName);
+            const abort = node.abortSpotMessageFollowIngress;
+            if (abort === undefined || !abort.call(node, spotMessageFollowSeal)) {
+              throw new Error('Spot Message Follow abort fence became stale.');
+            }
+          } catch (error) {
+            if (onCleanupError === undefined) throw error;
+            onCleanupError(error);
           }
         }
       }
@@ -2231,11 +2278,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     const relocationMetric =
       spotRegistration.relocation.kind === 'disabled'
         ? undefined
-        : this.options.metrics?.startRelocation(
-            meshName,
-            'user_spot',
-            spotRegistration.relocation.kind
-          );
+        : this.startSourceRelocationMetric(meshName, 'user_spot', spotRegistration.relocation.kind);
     try {
       await this.waitForRelocationBudgetHeadroom(String(target.rid), signal);
       const captured = await new ServiceRelocationObjectCaptureOwner().captureUserSpotAggregate(
@@ -2427,7 +2470,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     this.sourceRelocationIds.add(id);
   }
 
-  private async runCoordinator(
+  async runCoordinator(
     meshName: string,
     target: ZLinkMeshNodeDescriptor,
     primary: ZLinkAuthoritySnapshot,
@@ -2497,159 +2540,192 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
       advertisedReceiveChunkLimitBytes
     );
     const plan = planRelocationChunks(encoded, chunkLimitBytes);
-    // The Restore absolute deadline is this unit's relocation deadline: the
-    // caller's signal, or the control deadline without one. From it the
-    // source settles the authority with its Preserve fence (Location runtime
-    // §10). The target keeps no deadline of its own.
-    const restoreDeadlineReached = () =>
-      signal?.aborted === true || performance.now() >= controlDeadlineAtMs;
-    let readyReceived = false;
+    // The unit's original Restore absolute deadline authorizes the source
+    // Preserve fence (Location runtime §10). Caller cancellation cannot
+    // advance that deadline. The target keeps no deadline of its own.
+    const restoreDeadlineReached = () => performance.now() >= controlDeadlineAtMs;
     let settlement:
-      Awaited<ReturnType<ZLinkHostServiceRelocationRuntime['settleSourceAuthority']>> | undefined;
+      Awaited<ReturnType<ZLinkHostServiceRelocationRuntime['settleSourceAuthority']>> | undefined =
+      'source' as
+        Awaited<ReturnType<ZLinkHostServiceRelocationRuntime['settleSourceAuthority']>> | undefined;
+    let failure: { readonly error: unknown } | undefined;
+    let window: SourceCutoverWindow | undefined;
     try {
-      const prepare = {
-        kind: 'prepare',
-        relocation,
-        targetAttemptGeneration,
-        coordinator,
-        target: targetFence,
-        initiatorRole: 'source',
-        object,
-        sourceNodeRid: coordinator.nodeRid,
-        sourceNodeGeneration: coordinator.nodeGeneration,
-        payloadTotalLength: BigInt(plan.totalLength),
-        payloadChunkCount: plan.chunkCount,
-        payloadChecksumCrc32c: plan.checksumCrc32c,
-        applicationVersion: target.applicationVersion,
-        deadlineUnixMs: BigInt(
-          Date.now() + Math.max(0, Math.ceil(controlDeadlineAtMs - performance.now()))
-        )
-      } satisfies ServiceMaintenanceRelocationPrepare;
-      // Start the Restore request first (it also registers the temporary
-      // queue and the assembly on the target), then stream the payload chunks
-      // on the same ordered connection. The Ready reply arrives only after
-      // the target assembled, verified, and restored the payload.
-      const ready = await this.sendControl(
-        meshName,
-        target.rid,
-        prepare,
-        signal,
-        controlDeadlineAtMs,
-        async (attemptSignal) => {
-          if (isRelocationDebugEnabled())
-            relocationDebug('coordinator.state_transfer_begin', {
-              aggregateId: captured.envelope.aggregateId
-            });
-          await this.sendRelocationStateChunks(
-            meshName,
-            target,
-            prepare,
-            encoded,
-            plan,
-            attemptSignal
+      try {
+        const prepare = {
+          kind: 'prepare',
+          relocation,
+          targetAttemptGeneration,
+          coordinator,
+          target: targetFence,
+          initiatorRole: 'source',
+          object,
+          sourceNodeRid: coordinator.nodeRid,
+          sourceNodeGeneration: coordinator.nodeGeneration,
+          payloadTotalLength: BigInt(plan.totalLength),
+          payloadChunkCount: plan.chunkCount,
+          payloadChecksumCrc32c: plan.checksumCrc32c,
+          applicationVersion: target.applicationVersion,
+          deadlineUnixMs: BigInt(
+            Date.now() + Math.max(0, Math.ceil(controlDeadlineAtMs - performance.now()))
+          )
+        } satisfies ServiceMaintenanceRelocationPrepare;
+        // Start the Restore request first (it also registers the temporary
+        // queue and the assembly on the target), then stream the payload chunks
+        // on the same ordered connection. The Ready reply arrives only after
+        // the target assembled, verified, and restored the payload.
+        const ready = await this.sendControl(
+          meshName,
+          target.rid,
+          prepare,
+          signal,
+          controlDeadlineAtMs,
+          async (attemptSignal) => {
+            settlement = undefined;
+            if (isRelocationDebugEnabled())
+              relocationDebug('coordinator.state_transfer_begin', {
+                aggregateId: captured.envelope.aggregateId
+              });
+            await this.sendRelocationStateChunks(
+              meshName,
+              target,
+              prepare,
+              encoded,
+              plan,
+              attemptSignal
+            );
+            if (isRelocationDebugEnabled())
+              relocationDebug('coordinator.await_ready', {
+                aggregateId: captured.envelope.aggregateId
+              });
+          }
+        );
+        if (ready.kind === 'failed') {
+          settlement = 'source';
+          throw relocationFailureException(ready);
+        }
+        validateControlResponse(prepare, ready);
+        if (isRelocationDebugEnabled()) {
+          relocationDebug('coordinator.ready_received', {
+            aggregateId: captured.envelope.aggregateId
+          });
+        }
+        actorJoinProfile?.relayReady();
+
+        const boundaryBatch: Array<{
+          readonly frame: Buffer;
+          readonly canonicalBytes: Buffer;
+        }> = [];
+        for (const session of sessions) {
+          const participant = captured.envelope.participants.find(
+            (value) =>
+              value.objectKind === 'actor' &&
+              decodeAuthorityKey({ value: value.key } as ZLinkAuthorityKey).globalId ===
+                session.state.actorId
           );
-          if (isRelocationDebugEnabled())
-            relocationDebug('coordinator.await_ready', {
-              aggregateId: captured.envelope.aggregateId
+          if (participant === undefined) {
+            throw new Error(
+              `Relocation Actor '${session.state.actorId}' is missing from its envelope.`
+            );
+          }
+          for (const packet of session.prepared.takeRelocationRelay()) {
+            const frozenRecord = canonicalRelayFrozenRecord(
+              participant,
+              packet,
+              coordinator,
+              targetFence
+            );
+            const data = {
+              kind: 'data',
+              relocation,
+              targetAttemptGeneration,
+              coordinator,
+              senderRole: 'source',
+              object: relocationObjectForParticipant(participant),
+              frozenRecord
+            } satisfies Extract<ServiceMaintenanceRelocationControl, { kind: 'data' }>;
+            boundaryBatch.push({
+              frame: encodeServiceRelocationControlRequest(data),
+              canonicalBytes: frozenRecord.canonicalBytes
             });
+          }
         }
-      );
-      if (isRelocationDebugEnabled()) {
-        relocationDebug('coordinator.ready_received', {
-          aggregateId: captured.envelope.aggregateId
+        const cutoverFrame = encodeServiceRelocationControlRequest({
+          kind: 'cutover',
+          relocation,
+          targetAttemptGeneration,
+          coordinator,
+          senderRole: 'source',
+          object,
+          boundaryRecordCount: BigInt(boundaryBatch.length),
+          boundaryChecksumCrc32c: crc32c(
+            Buffer.concat(boundaryBatch.map((value) => value.canonicalBytes))
+          )
         });
-      }
-      validateControlResponse(prepare, ready);
-      // Ready arms the target's cutover-wait fallback. From this point
-      // forward the source can no longer reopen dispatch without risking two
-      // owners.
-      readyReceived = true;
-      actorJoinProfile?.relayReady();
-
-      const boundaryBatch: Array<{
-        readonly frame: Buffer;
-        readonly canonicalBytes: Buffer;
-      }> = [];
-      for (const session of sessions) {
-        const participant = captured.envelope.participants.find(
-          (value) =>
-            value.objectKind === 'actor' &&
-            decodeAuthorityKey({ value: value.key } as ZLinkAuthorityKey).globalId ===
-              session.state.actorId
-        );
-        if (participant === undefined) {
-          throw new Error(
-            `Relocation Actor '${session.state.actorId}' is missing from its envelope.`
-          );
-        }
-        for (const packet of session.prepared.takeRelocationRelay()) {
-          const frozenRecord = canonicalRelayFrozenRecord(
-            participant,
-            packet,
-            coordinator,
-            targetFence
-          );
-          const data = {
-            kind: 'data',
-            relocation,
-            targetAttemptGeneration,
-            coordinator,
-            senderRole: 'source',
-            object: relocationObjectForParticipant(participant),
-            frozenRecord
-          } satisfies Extract<ServiceMaintenanceRelocationControl, { kind: 'data' }>;
-          boundaryBatch.push({
-            frame: encodeServiceRelocationControlRequest(data),
-            canonicalBytes: frozenRecord.canonicalBytes
-          });
+        // The cutover submit result is not the authority result. The memory
+        // copy of the boundary batch and cutover stays until the authority
+        // settlement below (spec 28 §4.4).
+        const submittedRouteGeneration = this.requireMeshNode(meshName).observePeerRoute?.(
+          target.rid
+        ).generation;
+        try {
+          for (const record of boundaryBatch) {
+            await this.submitControlFrame(meshName, target.rid, record.frame, 'relay');
+          }
+          await this.submitControlFrame(meshName, target.rid, cutoverFrame, 'cutover');
+        } catch (error) {
+          if (!durableRequestCanReplay(error)) throw error;
+          console.warn('[zlink.runtime.relocation.cutover_submit_failed]', relayAuthorityId, error);
         }
+        window = this.beginSourceCutoverWindow(
+          meshName,
+          target.rid,
+          relayAuthorityId,
+          [...boundaryBatch.map((value) => value.frame), cutoverFrame],
+          submittedRouteGeneration,
+          limits
+        );
+      } catch (error) {
+        failure = { error };
       }
-      const cutoverFrame = encodeServiceRelocationControlRequest({
-        kind: 'cutover',
-        relocation,
-        targetAttemptGeneration,
-        coordinator,
-        senderRole: 'source',
-        object,
-        boundaryRecordCount: BigInt(boundaryBatch.length),
-        boundaryChecksumCrc32c: crc32c(
-          Buffer.concat(boundaryBatch.map((value) => value.canonicalBytes))
-        )
-      });
-      // The cutover submit result is not the authority result. The memory
-      // copy of the boundary batch and cutover stays until the authority
-      // settlement below (spec 28 §4.4).
-      const submittedRouteGeneration = this.requireMeshNode(meshName).observePeerRoute?.(
-        target.rid
-      ).generation;
       try {
-        for (const record of boundaryBatch) {
-          await this.submitControlFrame(meshName, target.rid, record.frame, 'relay');
+        try {
+          if (settlement === undefined) {
+            settlement = await this.settleSourceAuthority(
+              captured.envelope,
+              primary,
+              targetFence,
+              restoreDeadlineReached,
+              window,
+              relocationMetric
+            );
+          } else if (failure !== undefined) {
+            // Restoration without authority settlement did not confirm Preserve.
+            throw failure.error;
+          }
+        } finally {
+          if (settlement === 'source') {
+            // Source restoration is authorized: captured queue, timers and held records
+            // return in their accepted order, and source dispatch reopens.
+            try {
+              await captured.abortSource((error) => {
+                failure ??= { error };
+              });
+            } catch (error) {
+              throw failure === undefined ? error : failure.error;
+            }
+          }
         }
-        await this.submitControlFrame(meshName, target.rid, cutoverFrame, 'cutover');
+        this.completeSourceRelocationMetric(relocationMetric, settlement);
       } catch (error) {
-        if (!durableRequestCanReplay(error)) throw error;
-        console.warn('[zlink.runtime.relocation.cutover_submit_failed]', relayAuthorityId, error);
-      }
-      const window = this.beginSourceCutoverWindow(
-        meshName,
-        target.rid,
-        relayAuthorityId,
-        [...boundaryBatch.map((value) => value.frame), cutoverFrame],
-        submittedRouteGeneration,
-        limits
-      );
-      try {
-        settlement = await this.settleSourceAuthority(
-          captured.envelope,
-          primary,
-          targetFence,
-          restoreDeadlineReached,
-          window,
-          relocationMetric
-        );
+        failure = { error };
+        throw error;
       } finally {
-        this.closeSourceCutoverWindow(relayAuthorityId, window);
+        try {
+          if (window !== undefined) this.closeSourceCutoverWindow(relayAuthorityId, window);
+        } catch (error) {
+          throw failure === undefined ? error : failure.error;
+        }
       }
       if (isRelocationDebugEnabled()) {
         relocationDebug('coordinator.authority_settled', {
@@ -2658,16 +2734,12 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
         });
       }
       if (settlement === 'source') {
-        // Preserve won: the captured queue, timers and held records return to
-        // the source queue in their accepted order, and dispatch reopens.
-        await captured.abortSource();
-        this.completeSourceRelocationMetric(relocationMetric, settlement);
+        if (failure !== undefined) throw failure.error;
         throw new Error(
           `Relocation '${relayAuthorityId}' ended by the source Preserve fence; the source kept the authority.`
         );
       }
       if (settlement === 'lost') {
-        this.completeSourceRelocationMetric(relocationMetric, settlement);
         // Expired owner: dispatch and Store changes stop for good and every
         // pending request without a terminal gets one Unavailable.
         const unavailable = createInternalFrameworkException(
@@ -2678,10 +2750,10 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
         await captured.discardSource(unavailable);
         throw new ServiceRelocationAuthorityError(unavailable.message, { cause: unavailable });
       }
-      this.completeSourceRelocationMetric(relocationMetric, settlement);
       for (const session of sessions) session.prepared.setReplayResults([]);
       await captured.commitSource(settlement.followAuthorities);
       await this.options.reconcileStatefulAuthorityRoutes?.(signal);
+      if (failure !== undefined) throw failure.error;
     } catch (error) {
       if (
         this.disposed &&
@@ -2706,11 +2778,6 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
       );
     } finally {
       this.completeSourceRelocationMetric(relocationMetric);
-      if (!readyReceived) {
-        // Only an explicit failure before the Ready reply becomes accepted
-        // restores the source from the memory payload (spec 28 §9).
-        await captured.abortSource().catch(() => undefined);
-      }
     }
   }
 
@@ -2732,6 +2799,50 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     );
   }
 
+  startSourceRelocationMetric(
+    meshName: string,
+    objectKind: 'actor' | 'user_spot' | 'instance_spot',
+    policy: 'recreate' | 'snapshot',
+    actorId?: string
+  ): ZLinkRelocationMetricOperation | undefined {
+    const flow = objectKind === 'actor' ? this.options.messageFlow?.() : undefined;
+    const flowContext = flow?.flowCreationEnabled() === true ? currentFlowContext() : undefined;
+    const traceTerminal = (outcome: string): void => {
+      if (flow === undefined || flowContext === undefined) return;
+      const result =
+        outcome === 'completed' ? 'succeeded' : outcome === 'shutdown' ? 'shutdown' : 'failed';
+      flowIfEnabled(flow, ZLinkMessageFlowOutcome.Completed, result, flowContext.flowId)?.trace({
+        outcome: ZLinkMessageFlowOutcome.Completed,
+        result,
+        surface: ZLinkDispatchErrorSurface.ActorRelocation,
+        messageKind: ZLinkDispatchMessageKind.Control,
+        meshName,
+        actorId,
+        flowId: flowContext.flowId,
+        flowOrigin: flowContext.flowOrigin,
+        errorMessage: result === 'failed' ? `source_relocation_${outcome}` : undefined
+      });
+    };
+    const metric = this.options.metrics?.startRelocation(
+      meshName,
+      objectKind,
+      policy,
+      traceTerminal
+    );
+    if (metric !== undefined) return metric;
+    if (flow === undefined || flowContext === undefined) return undefined;
+
+    let completed = false;
+    return {
+      recordBytes: (): void => {},
+      complete: (outcome: string): void => {
+        if (completed) return;
+        completed = true;
+        traceTerminal(outcome);
+      }
+    };
+  }
+
   /** Streams the payload chunks of one stage under the effective in-flight budgets. */
   private async sendRelocationStateChunks(
     meshName: string,
@@ -2794,10 +2905,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
   }
 
   /** Pre-seal admission wait — a full budget delays the next unit before seal. */
-  private async waitForRelocationBudgetHeadroom(
-    targetRid: string,
-    signal?: AbortSignal
-  ): Promise<void> {
+  async waitForRelocationBudgetHeadroom(targetRid: string, signal?: AbortSignal): Promise<void> {
     await this.requireNodePayloadBudget().waitForHeadroom(signal);
     await this.requirePeerPayloadBudget(targetRid).waitForHeadroom(signal);
   }
@@ -2884,7 +2992,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
   }
 
   /** Relocation flow §4.4: replay the whole batch on a different ready route. */
-  private async resendSourceCutover(window: SourceCutoverWindow): Promise<void> {
+  async resendSourceCutover(window: SourceCutoverWindow): Promise<void> {
     const frames = window.frames;
     if (frames === undefined) return;
     const route = this.requireMeshNode(window.meshName).observePeerRoute?.(window.targetNodeRid);
@@ -2905,7 +3013,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
   }
 
   /**
-   * The one source authority settlement after Ready (Location runtime
+   * The one source authority settlement after request creation (Location runtime
    * §6.1·§10, spec 28 §4.4). It reads the Store until the target commit or
    * the Restore deadline. From the Restore deadline it runs the source
    * Preserve fence: a CAS on the StoreVersion expected by target NewOwner
@@ -2916,12 +3024,12 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
    * A committed later owner has no Follow snapshot (Location runtime §10). An uncertain
    * Store or lease result keeps the work retained and repeats the same decision.
    */
-  private async settleSourceAuthority(
+  async settleSourceAuthority(
     envelope: ServiceRelocationEnvelope,
     primary: ZLinkAuthoritySnapshot,
     target: ServiceWireRelocationTarget,
     restoreDeadlineReached: () => boolean,
-    window: SourceCutoverWindow,
+    window: SourceCutoverWindow | undefined,
     relocationMetric?: ZLinkRelocationMetricOperation
   ): Promise<SourceAuthorityCommit | 'source' | 'lost'> {
     const owner = { ownerId: primary.ownerId, leaseGeneration: primary.ownerLeaseGeneration };
@@ -2931,7 +3039,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
         this.completeSourceRelocationMetric(relocationMetric, undefined, true);
         throw new ServiceRelocationRuntimeStoppedError();
       }
-      await this.resendSourceCutover(window);
+      if (window !== undefined) await this.resendSourceCutover(window);
       const observed = await this.readSourceSettlement(envelope, primary, target).catch(
         (error: unknown) => {
           if (!isIndeterminateLocationStoreFailure(error)) throw error;
@@ -3028,7 +3136,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     return this.sourceCutoverWindows.size === 0;
   }
 
-  private async releaseParticipantAuthorities(
+  async releaseParticipantAuthorities(
     envelope: ServiceRelocationEnvelope,
     primaryKey: string,
     signal?: AbortSignal
@@ -3052,11 +3160,17 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
       if (result.kind !== 'stored') {
         throw new Error(`Participant '${participant.key}' recovery pointer release failed.`);
       }
+      if (participant.objectKind === 'actor') {
+        this.options.observeActorAuthorityVersion?.(
+          decodeAuthorityKey(key).globalId,
+          result.storeVersion.value
+        );
+      }
     }
   }
 
   /** Restores the hidden object and its saved-work prefix before Ready. */
-  private async handlePrepareControl(
+  async handlePrepareControl(
     meshName: string,
     stagingId: string,
     request: ServiceMaintenanceRelocationPrepare,
@@ -3314,7 +3428,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
    * whole (the retransmission always resends the entire batch, so the exact
    * span is the staged suffix of `boundaryRecordCount` records).
    */
-  private reconcileBoundaryRelay(
+  reconcileBoundaryRelay(
     stage: LocalStage,
     request: ServiceMaintenanceRelocationCutover,
     stagingId: string
@@ -3345,7 +3459,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     );
   }
 
-  private beginTargetFinalize(
+  beginTargetFinalize(
     meshName: string,
     stagingId: string,
     stage: LocalStage,
@@ -3362,7 +3476,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     return finalize;
   }
 
-  private async finalizeTargetStage(
+  async finalizeTargetStage(
     meshName: string,
     stagingId: string,
     stage: LocalStage,
@@ -3414,15 +3528,6 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
           );
       }
       await stage.owner.normalize(stage.staging, authority, signal);
-      // The target may have visited this Actor before and still retain that
-      // tenure in its private Actor-client resolver. The target-only CAS is
-      // now authoritative, so expire the old projection before OnJoinedActor
-      // can issue an Actor request and deadlock behind source leave.
-      for (const hidden of stage.staging.hidden.values()) {
-        if (hidden.actor !== undefined) {
-          this.options.invalidateActorRoute?.(hidden.actor.context.actorId);
-        }
-      }
       await this.finalizeActorJoinProfiles(meshName, stage, signal);
       await this.publishSessionRoutes(stage.staging);
       stage.phase = 'open';
@@ -3502,7 +3607,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     );
   }
 
-  private async finalizeActorJoinProfiles(
+  async finalizeActorJoinProfiles(
     meshName: string,
     stage: LocalStage,
     signal?: AbortSignal
@@ -3535,7 +3640,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     }
   }
 
-  private async submitActorJoinSourceLeave(
+  async submitActorJoinSourceLeave(
     meshName: string,
     sourceNodeRid: RoutingId,
     relocationId: string,
@@ -3561,7 +3666,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     }
   }
 
-  private acceptActorJoinSourceLeave(input: {
+  acceptActorJoinSourceLeave(input: {
     readonly transferId: string;
     readonly actorId: string;
     readonly succeeded: boolean;
@@ -3601,7 +3706,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     );
   }
 
-  private async submitControlFrame(
+  async submitControlFrame(
     meshName: string,
     targetNodeRid: RoutingId,
     frame: Buffer,
@@ -3615,7 +3720,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     }
   }
 
-  private async clearTargetRelocationPublication(
+  async clearTargetRelocationPublication(
     stage: LocalStage,
     primary: ZLinkAuthoritySnapshot,
     signal?: AbortSignal
@@ -3644,6 +3749,12 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     if (result.kind !== 'stored') {
       throw new Error('Relocation target authority normalization CAS failed.');
     }
+    if (primary.allocation.objectKind === 'actor') {
+      this.options.observeActorAuthorityVersion?.(
+        decodeAuthorityKey(stage.staging.primaryAuthorityKey).globalId,
+        result.storeVersion.value
+      );
+    }
     return { ...result, kind: 'snapshot' };
   }
 
@@ -3723,7 +3834,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     }
   }
 
-  private async sendReplyRelay(
+  async sendReplyRelay(
     meshName: string,
     ackTargetNodeRid: RoutingId,
     request: ServiceMaintenanceReplyRelay,
@@ -3841,7 +3952,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     }
   }
 
-  private targetReplyRelayCoordinator(
+  targetReplyRelayCoordinator(
     meshName: string,
     stage: LocalStage,
     authority: ZLinkAuthoritySnapshot
@@ -3867,7 +3978,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     };
   }
 
-  private async relayTerminalReply(
+  async relayTerminalReply(
     meshName: string,
     stage: LocalStage,
     coordinator: ServiceWireRelocationCoordinatorFence,
@@ -3917,9 +4028,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     );
   }
 
-  private async publishSessionRoutes(
-    staging: ServiceObjectRelocationStaging<LocalHidden>
-  ): Promise<void> {
+  async publishSessionRoutes(staging: ServiceObjectRelocationStaging<LocalHidden>): Promise<void> {
     for (const [key, hidden] of staging.hidden) {
       if (hidden.actor === undefined) continue;
       // The Session seal travels with the relocation envelope, not the Actor.
@@ -3931,7 +4040,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     }
   }
 
-  private async sendControl(
+  async sendControl(
     meshName: string,
     targetNodeRid: RoutingId,
     request: ZLinkServiceRelocationControlRequest,
@@ -3950,26 +4059,42 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
         true
       );
     }
-    const replyFrames = await this.requestInfrastructureControl(
-      meshName,
-      targetNodeRid,
-      frames,
-      remainingMs,
-      signal,
-      onSubmitted
+    const pending = this.options.requests.begin<
+      ReturnType<typeof decodeServiceRelocationControlResponse>
+    >(remainingMs, signal, undefined, deadlineAtMs);
+    return this.options.requests.run(
+      pending,
+      (receiver) => {
+        void this.requestInfrastructureControl(
+          meshName,
+          targetNodeRid,
+          frames,
+          remainingMs,
+          signal,
+          onSubmitted,
+          pending
+        ).then(
+          (reply) => receiver.operationCompleted(pending.id, reply, undefined),
+          (error) => receiver.operationFailed(pending.id, error, undefined)
+        );
+      },
+      (raw) => {
+        const replyFrames = raw as readonly Uint8Array[];
+        if (replyFrames.length !== 1) {
+          throw new ServiceWireProtocolError(
+            `Relocation control '${key}' reply must contain exactly one frame.`
+          );
+        }
+        const response = decodeServiceRelocationControlResponse(Buffer.from(replyFrames[0]!));
+        if (response.kind === 'failed') {
+          validateControlFailureResponse(request, response);
+        } else {
+          validateControlResponse(request, response);
+        }
+        return response;
+      },
+      (error) => error
     );
-    if (replyFrames.length !== 1) {
-      throw new ServiceWireProtocolError(
-        `Relocation control '${key}' reply must contain exactly one frame.`
-      );
-    }
-    const response = decodeServiceRelocationControlResponse(Buffer.from(replyFrames[0]!));
-    if (response.kind === 'failed') {
-      validateControlFailureResponse(request, response);
-      throw relocationFailureException(response);
-    }
-    validateControlResponse(request, response);
-    return response;
   }
 
   private async selectTarget(
@@ -4046,7 +4171,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     return authorities;
   }
 
-  private async reserveTargetForPrepare(
+  async reserveTargetForPrepare(
     meshName: string,
     prepare: ServiceMaintenanceRelocationPrepare,
     envelope: ServiceRelocationEnvelope,
@@ -4120,7 +4245,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     };
   }
 
-  private authorityPayloadForPublication(
+  authorityPayloadForPublication(
     payload: Uint8Array,
     publication: ServiceRelocationPublication,
     target?: {
@@ -4251,7 +4376,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     }
   }
 
-  private async commitTargetReservation(
+  async commitTargetReservation(
     stage: LocalStage,
     reservation: TargetRelocationReservation,
     signal?: AbortSignal
@@ -4326,7 +4451,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     }
   }
 
-  private async readAggregateForCommitRetry(
+  async readAggregateForCommitRetry(
     prepared: ServicePreparedRelocationAggregate,
     signal?: AbortSignal
   ): Promise<
@@ -4374,7 +4499,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     return source === prepared.plan.participants.length ? { kind: 'source' } : { kind: 'stale' };
   }
 
-  private async materializeTargetEnvelope(
+  async materializeTargetEnvelope(
     meshName: string,
     envelope: ServiceRelocationEnvelope,
     signal?: AbortSignal
@@ -4392,7 +4517,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
   }
 
   /** Maps schema ordinals to the authoritative Location Store inventory. */
-  private async resolveTargetParticipantInventory(
+  async resolveTargetParticipantInventory(
     meshName: string,
     request: ServiceMaintenanceRelocationPrepare,
     envelope: ServiceRelocationEnvelope,
@@ -4549,7 +4674,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     return value;
   }
 
-  private async captureApplication<T>(
+  async captureApplication<T>(
     policy: { readonly kind: 'disabled' | 'recreate' | 'snapshot'; readonly adapterType?: Type },
     value: T,
     signal?: AbortSignal
@@ -4690,11 +4815,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     return value;
   }
 
-  private afterSpotCloseDecision<T>(
-    meshName: string,
-    spotId: RoutingId,
-    seal: () => T
-  ): T | Promise<T> {
+  afterSpotCloseDecision<T>(meshName: string, spotId: RoutingId, seal: () => T): T | Promise<T> {
     const manager = this.requireSpotManager();
     const sealAfterDecision = (): T => {
       if (manager.isSpotClosing(meshName, spotId)) {
@@ -4778,14 +4899,20 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
     frames: readonly Uint8Array[],
     timeoutMs: number,
     signal?: AbortSignal,
-    onSubmitted?: (signal: AbortSignal) => Promise<void>
+    onSubmitted?: (signal: AbortSignal) => Promise<void>,
+    requestOperation?: PendingOperation<unknown>
   ): Promise<readonly Uint8Array[]> {
     const node = this.requireMeshNode(meshName);
     const request = node.requestInfrastructureControlFrames;
     if (request === undefined) {
       throw new Error('RouteMesh backend cannot submit bare infrastructure requests.');
     }
-    return await request.call(node, targetNodeRid, frames, { timeoutMs, signal, onSubmitted });
+    return await request.call(node, targetNodeRid, frames, {
+      timeoutMs,
+      signal,
+      onSubmitted,
+      requestOperation
+    });
   }
 }
 
@@ -5037,6 +5164,7 @@ class LocalTargetPort implements ServiceRelocationTargetObjectPort<LocalHidden>
         value: hidden.authorityKey
       } as ZLinkAuthorityKey);
       const actorId = hidden.actor.context.actorId;
+      this.options.observeActorAuthorityVersion?.(actorId, current.storeVersion.value);
       const node = this.options.meshNode(this.meshName);
       const restore = node?.restoreActorAuthority;
       if (node === undefined || restore === undefined) {
@@ -5744,7 +5872,7 @@ function maintenanceReplyRelay(
   const direct = packet.messageFollowOrigin?.requestSource !== undefined;
   const application =
     direct && completion.result.ok
-      ? decodeApplicationPayloadView(
+      ? decodeApplicationPayload(
           encodeMultipartApplicationPayload(
             encodeActorReply(
               Buffer.from(packet.header, 'base64'),