@zlink-systems/framework 0.30.3 → 0.32.0

package/src/runtime/host/index.ts

@@ -1,3 +1,6 @@
+import { ZLinkFrameworkException, ZLinkFrameworkErrorKind } from '../../contracts';
+import { RequestRuntime } from '../foundation/request-runtime';
+import { inspectRuntime } from '../internal-observation';
 import { ZLinkProviderRelocationRepository } from '../locations/relocation-blob';
 import type { ApplicationJobOrigin } from '../application-jobs/contracts';
 import { currentApplicationJobOrigin } from '../application-jobs/application-job-queue-scope';
@@ -9,6 +12,7 @@ import { isDeepStrictEqual } from 'node:util';
 import {
   type ActorRef,
   type RoutingId,
+  type ZLinkSessionBindingIdentity,
   type ZLinkClientServerRuntime,
   type ZLinkFanoutRuntime,
   type ZLinkFrameworkLifecycleOptions,
@@ -125,6 +129,11 @@ import {
 } from '../diagnostics';
 import { METRIC_NAMES, MILLISECONDS_PER_SECOND } from '../diagnostics/runtime-metrics';
 import { RuntimeEventQueue } from '../diagnostics/runtime-observation-queue';
+import {
+  openTelemetryLease,
+  renewTelemetryLease,
+  retireTelemetryLease
+} from '../diagnostics/message-flow-delivery';
 import {
   ZLinkClientServerRuntimeProjection,
   ZLinkFanoutRuntimeProjection
@@ -207,7 +216,7 @@ import {
   nodeEffectiveProcessorCount,
   resolveApplicationJobQueueConfiguration
 } from './application-job-queue';
-import { ZLinkBoundSessionRelay } from './bound-session-relay';
+import { ZLinkBoundSessionRelay, type ZLinkBoundSessionRelayOptions } from './bound-session-relay';
 import { ZLinkChannelRuntimeOptionsFactory } from './channel-runtime-options-factory';
 import { ZLinkEntryActorRuntimeService } from './entry-actor-runtime';
 import { HostCapacityStatusProjection } from './host-capacity-status';
@@ -255,10 +264,28 @@ export interface ZLinkFrameworkRuntimeHostOptions {
 export class ZLinkFrameworkRuntimeHost
   implements ZLinkFrameworkRuntimeLifecycle, ZLinkFrameworkRuntime, ZLinkMessageFlowControl
 {
+  [inspectRuntime]() {
+    return Object.freeze({
+      ownerLeaseUsable: this.locationOwner.currentRuntime?.ownerLeaseUsable,
+      ownerId: this.locationOwner.currentRuntime?.ownerId,
+      hasOwnerToken: this.locationOwner.currentRuntime?.currentOwnerToken !== undefined,
+      ownerAdmissionOpen: this.admission.ownerAdmissionOpen(),
+      channelRuntimeStarted: this.channelRuntime !== undefined,
+      fanoutStoreIsLocationStore:
+        this.locationOwner.currentStores?.fanoutStore ===
+        this.locationOwner.currentStores?.locationStore,
+      fanoutStoreMethods: Object.freeze({
+        update: typeof this.locationOwner.currentStores?.fanoutStore?.updateFanoutPublisher,
+        remove: typeof this.locationOwner.currentStores?.fanoutStore?.removeFanoutPublisher,
+        list: typeof this.locationOwner.currentStores?.fanoutStore?.listFanoutPublishers
+      })
+    });
+  }
+
   private readonly backendAdapterFactory: ZLinkBackendAdapterFactory;
   private readonly lifecycleSink?: string[];
-  private executionState?: ZLinkFrameworkExecutionState;
-  private runtimeState = ZLinkFrameworkRuntimeState.Preparing;
+  executionState?: ZLinkFrameworkExecutionState;
+  runtimeState = ZLinkFrameworkRuntimeState.Preparing;
   private runtimeStatus?: ZLinkFrameworkRuntimeStatus;
   private readonly runtimeObservationSource = Symbol('zlink.framework-runtime');
   private runtimeDeadline?: Date;
@@ -273,21 +300,21 @@ export class ZLinkFrameworkRuntimeHost
   private shutdownOperationStartedAt?: number;
   private relocationTargetApplicationVersion?: bigint;
   private readonly runtimeObservers = new Set<RuntimeEventQueue<ZLinkFrameworkRuntimeStatus>>();
-  private channelRuntime?: ZLinkChannelRuntimeManager;
-  private spotNodeRuntime?: ZLinkSpotNodeRuntimeManager;
+  channelRuntime?: ZLinkChannelRuntimeManager;
+  spotNodeRuntime?: ZLinkSpotNodeRuntimeManager;
   private streamRuntime?: ZLinkStreamRuntimeManager;
   private statefulAuthorityRoutes?: ZLinkStatefulAuthorityRouteRuntime;
-  private readonly locationOwner: ZLinkLocationRuntimeOwner;
+  readonly locationOwner: ZLinkLocationRuntimeOwner;
   private readonly meshRouters: MeshRouterResolver;
-  private readonly boundSessionRelay: ZLinkBoundSessionRelay;
+  readonly boundSessionRelay: ZLinkBoundSessionRelay;
   private readonly actorHandoff: ZLinkActorHandoffCoordinator;
   private readonly actorTransferRegistry: ZLinkActorTransferRegistry;
   private readonly actorTransferRuntime: ZLinkActorTransferRuntime;
   private readonly actorTransferAuthorityRuntime: ZLinkActorTransferAuthorityRuntime;
-  private readonly serviceRelocation: ZLinkHostServiceRelocationRuntime;
-  private readonly entryActorRuntime: ZLinkEntryActorRuntimeService;
+  readonly serviceRelocation: ZLinkHostServiceRelocationRuntime;
+  readonly entryActorRuntime: ZLinkEntryActorRuntimeService;
   private actorManager?: DefaultZLinkActorManager;
-  private actorPlacement?: ZLinkActorPlacementCoordinator;
+  actorPlacement?: ZLinkActorPlacementCoordinator;
   private spotManager?: DefaultZLinkSpotManager;
   private userSpotCoordinator?: ZLinkUserSpotCreationCoordinator;
   private ownerLeaseRecoveryRuntime?: ZLinkLocationRuntime;
@@ -298,11 +325,11 @@ export class ZLinkFrameworkRuntimeHost
   private readonly destroyedActorRefs = new Map<string, ActorRef>();
   private readonly runtimeEventPublisher: ZLinkRuntimeEventPublisher;
   private readonly metrics: ZLinkRuntimeMetrics;
-  private readonly applicationJobQueue: ApplicationJobQueue;
+  readonly applicationJobQueue: ApplicationJobQueue;
   private readonly capacityStatus: HostCapacityStatusProjection;
   private readonly metricRegistrations: import('../diagnostics').ZLinkRuntimeMetricRegistration[] =
     [];
-  private readonly admission = new ZLinkRuntimeAdmissionGate(() => this.ownerAdmissionOpen());
+  readonly admission: ZLinkRuntimeAdmissionGate;
   private readonly activationAdmission = new ZLinkActivationAdmission(
     (meshName) =>
       this.options.registration.spotNodes.get(meshName)?.activationConcurrencyLimit ??
@@ -315,11 +342,12 @@ export class ZLinkFrameworkRuntimeHost
   private readonly messageFlowModeCell: ZLinkMessageFlowModeCell = {
     mode: 'errors'
   };
-  private readonly dispatchErrorReporters = new WeakMap<
+  private telemetryLease = openTelemetryLease();
+  private dispatchErrorReporters = new WeakMap<
     ZLinkDispatchErrorSink,
     ZLinkDispatchErrorReporter
   >();
-  private readonly runtimeOrPreStartErrorSink: ZLinkDispatchErrorSink = {
+  readonly runtimeOrPreStartErrorSink: ZLinkDispatchErrorSink = {
     reportRuntimeTaskException: (taskName: string, error: unknown) =>
       (this.errorSink ?? this.preStartErrorSink).reportRuntimeTaskException(taskName, error)
   };
@@ -330,6 +358,7 @@ export class ZLinkFrameworkRuntimeHost
   readonly routeMeshRuntimeOptions = new DefaultZLinkRouteMeshRuntimeOptions(
     () => this.spotNodeRuntime
   );
+  private readonly requests = new RequestRuntime();
   readonly routeTransport: ZLinkRuntimeRouteTransport;
   readonly spotAddressTransport: ZLinkHostSpotAddressTransport;
   readonly spotPublisherTransport = new ZLinkRuntimeSpotPublisherTransport(
@@ -342,11 +371,27 @@ export class ZLinkFrameworkRuntimeHost
   readonly clientServerRuntime: ZLinkClientServerRuntime;
   readonly fanoutRuntime: ZLinkFanoutRuntime;
   private readonly relocationStore?: import('../../contracts/Locations/RelocationStore').ZLinkRelocationStore;
-  private readonly routeMeshCoordinator: ZLinkRouteMeshRuntimeCoordinator;
+  readonly routeMeshCoordinator: ZLinkRouteMeshRuntimeCoordinator;
   constructor(
     readonly options: ZLinkFrameworkRuntimeHostOptions,
     internalOptions?: unknown
   ) {
+    const factories = internalOptions as
+      | {
+          readonly spotNodeRuntime?: ZLinkSpotNodeRuntimeManager;
+          readonly createBoundSessionRelay?: (
+            options: ZLinkBoundSessionRelayOptions
+          ) => ZLinkBoundSessionRelay;
+          readonly createRouteMeshCoordinator?: (
+            options: ConstructorParameters<typeof ZLinkRouteMeshRuntimeCoordinator>[0]
+          ) => ZLinkRouteMeshRuntimeCoordinator;
+          readonly createAdmissionGate?: (ownerOpen: () => boolean) => ZLinkRuntimeAdmissionGate;
+        }
+      | undefined;
+    this.spotNodeRuntime = factories?.spotNodeRuntime;
+    this.admission = (
+      factories?.createAdmissionGate ?? ((ownerOpen) => new ZLinkRuntimeAdmissionGate(ownerOpen))
+    )(() => this.ownerAdmissionOpen());
     this.backendAdapterFactory = resolveBackendAdapterFactory(internalOptions);
     this.lifecycleSink = options.lifecycleSink;
     this.runtimeEventPublisher =
@@ -387,7 +432,18 @@ export class ZLinkFrameworkRuntimeHost
     this.meshRouters = new MeshRouterResolver(options.registration);
     this.spotRouterChannelIdForMesh = this.meshRouters.spotRouterChannelIdByMesh();
     this.routeTransport = new ZLinkRuntimeRouteTransport(
-      () => this.channelRuntime,
+      () => {
+        if (
+          this.channelRuntime === undefined &&
+          this.runtimeState === ZLinkFrameworkRuntimeState.Stopped
+        ) {
+          throw createInternalFrameworkException(
+            ZLinkFrameworkInternalErrorKind.RuntimeShutdown,
+            'Route channel runtime is stopped.'
+          );
+        }
+        return this.channelRuntime;
+      },
       (routerChannelId) => this.meshRouters.canUseRouterChannel(routerChannelId),
       () => this.spotNodeRuntime,
       { serializers: options.registration.messageSerializers },
@@ -396,9 +452,11 @@ export class ZLinkFrameworkRuntimeHost
       (meshName, sourceNodeRid, parts) => this.submitLocalMeshRoute(meshName, sourceNodeRid, parts),
       this.metrics,
       () => this.flowCreationEnabled(),
-      () => this.createDispatchErrorReporter(this.runtimeOrPreStartErrorSink).flow
+      () => this.createDispatchErrorReporter(this.runtimeOrPreStartErrorSink).flow,
+      this.requests
     );
     this.spotAddressTransport = new ZLinkHostSpotAddressTransport({
+      requests: this.requests,
       resolver: () => this.createLocationSpotRouteResolver(),
       routed: this.routeTransport,
       meshNames: () =>
@@ -462,10 +520,16 @@ export class ZLinkFrameworkRuntimeHost
       },
       relay: (actor, header, payload) =>
         this.boundSessionRelay.actorPackets.relayActorPacket(actor, header, payload),
-      notifyDisconnected: (actor, signal, binding) =>
-        this.boundSessionRelay.actorPackets.notifyBoundActorDisconnected(actor, signal, binding)
+      notifyDisconnected: (actor, signal, binding, authorityFence) =>
+        this.boundSessionRelay.actorPackets.notifyBoundActorDisconnected(
+          actor,
+          signal,
+          binding,
+          authorityFence
+        )
     });
     this.actorHandoff = new ZLinkActorHandoffCoordinator({
+      requests: this.requests,
       routedTransport: this.routeTransport,
       messageFollowDurationMs: options.registration.locations.options.messageFollowDurationMs,
       requestTimeoutMs: options.registration.requestTimeoutMs,
@@ -657,7 +721,10 @@ export class ZLinkFrameworkRuntimeHost
       options.registration.messageSerializers
     );
     this.boundSessionFactory = new DefaultZLinkBoundSessionFactory(this.streamBindingRuntime);
-    this.boundSessionRelay = new ZLinkBoundSessionRelay({
+    const createBoundSessionRelay =
+      factories?.createBoundSessionRelay ??
+      ((relayOptions: ZLinkBoundSessionRelayOptions) => new ZLinkBoundSessionRelay(relayOptions));
+    this.boundSessionRelay = createBoundSessionRelay({
       requestTimeoutMs: options.registration.requestTimeoutMs,
       sessionRelocationSealTimeoutMs:
         options.registration.locations.options.sessionRelocationSealTimeoutMs ??
@@ -785,6 +852,7 @@ export class ZLinkFrameworkRuntimeHost
       shutdownSignal: () => this.executionState?.abortController.signal
     });
     this.serviceRelocation = new ZLinkHostServiceRelocationRuntime({
+      requests: this.requests,
       flowCreationEnabled: () => this.flowCreationEnabled(),
       messageFlow: () => this.createDispatchErrorReporter(this.runtimeOrPreStartErrorSink).flow,
       registration: options.registration,
@@ -816,8 +884,8 @@ export class ZLinkFrameworkRuntimeHost
         clear: () => this.boundSessionRelay.boundSessions.clearServiceWireSessionRelocations()
       },
       trackInstanceSpot: (input) => this.locationOwner.currentLifecycle?.trackInstanceSpot(input),
-      invalidateActorRoute: (actorId) =>
-        this.actorClientLocationResolver?.invalidateActorRoute(actorId),
+      observeActorAuthorityVersion: (actorId, storeVersion) =>
+        this.actorClientLocationResolver?.observeActorAuthorityVersion(actorId, storeVersion),
       reconcileStatefulAuthorityRoutes: (signal) =>
         this.statefulAuthorityRoutes?.reconcile(signal) ?? Promise.resolve(),
       runtimeEventPublisher: this.runtimeEventPublisher,
@@ -832,7 +900,10 @@ export class ZLinkFrameworkRuntimeHost
         ...options.registration.locations.options
       }.ownerLeaseTtlMs
     });
-    this.routeMeshCoordinator = new ZLinkRouteMeshRuntimeCoordinator({
+    this.routeMeshCoordinator = (
+      factories?.createRouteMeshCoordinator ??
+      ((meshOptions) => new ZLinkRouteMeshRuntimeCoordinator(meshOptions))
+    )({
       meshNames: [...options.registration.spotNodes.keys()],
       meshOptions: options.registration.spotNodes,
       meshNode: (meshName) => this.spotNodeRuntime?.meshNode(meshName),
@@ -1426,6 +1497,13 @@ export class ZLinkFrameworkRuntimeHost
       return;
     }
 
+    const previousLease = this.telemetryLease;
+    this.telemetryLease = renewTelemetryLease(previousLease);
+    if (this.telemetryLease !== previousLease) {
+      this.dispatchErrorReporters = new WeakMap();
+      this.cachedDiagnosticsContext = undefined;
+    }
+
     this.setRuntimeState(ZLinkFrameworkRuntimeState.Preparing);
     this.lifecycleSink?.push('framework:start');
     const listenerRecords = new ZLinkListenerRecords();
@@ -1451,11 +1529,12 @@ export class ZLinkFrameworkRuntimeHost
         channelAdapter,
         context,
         this.options.providerResolver,
-        { ...this.createChannelRuntimeOptions(), listenerRecords }
+        { ...this.createChannelRuntimeOptions(), listenerRecords, requests: this.requests }
       );
       this.channelRuntime = channelRuntime;
       channelRuntime.prepareMeshDispatch(this.executionState.taskRunner);
       spotNodeRuntime = new ZLinkSpotNodeRuntimeManager({
+        requests: this.requests,
         ...this.createSpotNodeRuntimeOptions(context, dispatchErrors),
         listenerRecords
       });
@@ -1553,12 +1632,16 @@ export class ZLinkFrameworkRuntimeHost
   }
 
   async stop(): Promise<void> {
+    retireTelemetryLease(this.telemetryLease);
+    this.requests.close(
+      new ZLinkFrameworkException(ZLinkFrameworkErrorKind.ShuttingDown, 'Host stopped.')
+    );
     await this.runWithShutdownDeadline(DEFAULT_HOST_CONTROL_TIMEOUT_MS, async () =>
       this.runLifecycle(() => this.stopCore())
     );
   }
 
-  private runtimeMetricMeshSnapshots(): readonly ZLinkRuntimeMetricMeshSnapshot[] {
+  runtimeMetricMeshSnapshots(): readonly ZLinkRuntimeMetricMeshSnapshot[] {
     const snapshots: ZLinkRuntimeMetricMeshSnapshot[] = [];
     for (const [meshName, options] of this.options.registration.spotNodes) {
       let status;
@@ -1649,7 +1732,7 @@ export class ZLinkFrameworkRuntimeHost
     }
   }
 
-  private installOwnerLeaseRecoveryPublication(
+  installOwnerLeaseRecoveryPublication(
     runtime: ZLinkLocationRuntime,
     spotNodeRuntime: ZLinkSpotNodeRuntimeManager,
     channelRuntime: ZLinkChannelRuntimeManager
@@ -1676,7 +1759,7 @@ export class ZLinkFrameworkRuntimeHost
     runtime.addOwnerLeaseRenewedHandler(handler);
   }
 
-  private async resumeOwnerLeaseOwnedWork(
+  async resumeOwnerLeaseOwnedWork(
     runtime: ZLinkLocationRuntime,
     spotNodeRuntime: ZLinkSpotNodeRuntimeManager,
     channelRuntime: ZLinkChannelRuntimeManager
@@ -1700,7 +1783,7 @@ export class ZLinkFrameworkRuntimeHost
 
   // The one transition from Preparing to Serving: the host publishes its
   // Serving descriptors first, then takes new application work.
-  private async enterServing(
+  async enterServing(
     spotNodeRuntime: ZLinkSpotNodeRuntimeManager,
     signal: AbortSignal | undefined
   ): Promise<void> {
@@ -1736,7 +1819,7 @@ export class ZLinkFrameworkRuntimeHost
     }
   }
 
-  private async startStatefulAuthority(
+  async startStatefulAuthority(
     locationStore: import('../locations/domain-store-contract').ZLinkDomainLocationStore,
     spotNodeRuntime: ZLinkSpotNodeRuntimeManager,
     signal: AbortSignal | undefined
@@ -1842,7 +1925,7 @@ export class ZLinkFrameworkRuntimeHost
     }
   }
 
-  private removeOwnerLeaseRecoveryPublication(): void {
+  removeOwnerLeaseRecoveryPublication(): void {
     if (
       this.ownerLeaseRecoveryRuntime !== undefined &&
       this.ownerLeaseRecoveryHandler !== undefined
@@ -1936,7 +2019,7 @@ export class ZLinkFrameworkRuntimeHost
     );
   }
 
-  private async preflightAutomaticPeerReadiness(
+  async preflightAutomaticPeerReadiness(
     deadlineAt: number,
     targetApplicationVersion: bigint
   ): Promise<ZLinkFrameworkRelocationReason | undefined> {
@@ -2017,7 +2100,7 @@ export class ZLinkFrameworkRuntimeHost
     return true;
   }
 
-  private async publishHostRelocated(): Promise<void> {
+  async publishHostRelocated(): Promise<void> {
     await Promise.all(
       [...this.options.registration.spotNodes.keys()].map((meshName) =>
         this.spotNodeRuntime?.publishMeshNodeState(
@@ -2340,11 +2423,45 @@ export class ZLinkFrameworkRuntimeHost
         };
       }
     });
+    const selectUserSpotMesh = (requestedMeshName: string | undefined): string => {
+      const clientMeshes = [...this.options.registration.spotNodes]
+        .filter(([, node]) => hasObjectClientCapability(node.objectRole))
+        .map(([meshName]) => meshName);
+      const meshes = requestedMeshName === undefined ? clientMeshes : [requestedMeshName];
+      if (meshes.length === 0) {
+        throw createInternalFrameworkException(
+          ZLinkFrameworkInternalErrorKind.ObjectClientNotConfigured,
+          'No object-client RouteMesh is configured.'
+        );
+      }
+      if (requestedMeshName === undefined && meshes.length > 1) {
+        throw createInternalFrameworkException(
+          ZLinkFrameworkInternalErrorKind.MeshSelectionRequired,
+          'Multiple object-client RouteMeshes are configured; call inMesh(...).'
+        );
+      }
+      const meshName = meshes[0]!;
+      if (!this.options.registration.spotNodes.has(meshName)) {
+        throw createInternalFrameworkException(
+          ZLinkFrameworkInternalErrorKind.MeshNotFound,
+          `RouteMesh '${meshName}' is not configured.`
+        );
+      }
+      if (!clientMeshes.includes(meshName)) {
+        throw createInternalFrameworkException(
+          ZLinkFrameworkInternalErrorKind.ObjectClientNotConfigured,
+          `RouteMesh '${meshName}' has no object-client role.`
+        );
+      }
+      return meshName;
+    };
     const coordinator = (this.userSpotCoordinator = new ZLinkUserSpotCreationCoordinator({
       store: locationStore,
-      relocationPolicy: (meshName, stableType) =>
-        this.options.registration.spotNodes.get(meshName)?.spotFactoryRegistrations?.[stableType]
-          ?.relocation.kind,
+      leaseTracker: this.locationOwner.currentLeaseTracker!,
+      relocationPolicy: (meshName, stableType) => {
+        const node = this.options.registration.spotNodes.get(selectUserSpotMesh(meshName));
+        return node?.spotFactoryRegistrations?.[stableType]?.relocation.kind;
+      },
       publishReadyRoute: (meshName, route) => {
         this.cachedLocationSpotRouteResolver?.invalidate?.(route.spot.spotId);
         this.spotNodeRuntime?.meshNode(meshName)?.rememberSpotRoute?.(route);
@@ -2377,35 +2494,7 @@ export class ZLinkFrameworkRuntimeHost
         }
       },
       target: async (request, signal, excludedNodeRids) => {
-        const clientMeshes = [...this.options.registration.spotNodes]
-          .filter(([, node]) => hasObjectClientCapability(node.objectRole))
-          .map(([meshName]) => meshName);
-        const meshes = request.meshName === undefined ? clientMeshes : [request.meshName];
-        if (meshes.length === 0) {
-          throw createInternalFrameworkException(
-            ZLinkFrameworkInternalErrorKind.ObjectClientNotConfigured,
-            'No object-client RouteMesh is configured.'
-          );
-        }
-        if (request.meshName === undefined && meshes.length > 1) {
-          throw createInternalFrameworkException(
-            ZLinkFrameworkInternalErrorKind.MeshSelectionRequired,
-            'Multiple object-client RouteMeshes are configured; call inMesh(...).'
-          );
-        }
-        const meshName = meshes[0]!;
-        if (!this.options.registration.spotNodes.has(meshName)) {
-          throw createInternalFrameworkException(
-            ZLinkFrameworkInternalErrorKind.MeshNotFound,
-            `RouteMesh '${meshName}' is not configured.`
-          );
-        }
-        if (!clientMeshes.includes(meshName)) {
-          throw createInternalFrameworkException(
-            ZLinkFrameworkInternalErrorKind.ObjectClientNotConfigured,
-            `RouteMesh '${meshName}' has no object-client role.`
-          );
-        }
+        const meshName = selectUserSpotMesh(request.meshName);
         const meshNode = this.spotNodeRuntime?.meshNode(meshName);
         const descriptors = await listAllMeshNodeDescriptors(locationStore, meshName, signal);
         const selected = await selectReadyLocationPlacementDescriptor(
@@ -2672,7 +2761,8 @@ export class ZLinkFrameworkRuntimeHost
                       spot.meshName,
                       spot.spotId,
                       beginAuthority,
-                      closeSignal
+                      closeSignal,
+                      new Date(Number(record.deadlineUnixMs))
                     ),
                   signal,
                   (spot) => local.isSpotClosing(spot.meshName, spot.spotId)
@@ -2716,6 +2806,7 @@ export class ZLinkFrameworkRuntimeHost
 
   private actorRuntimeOptionsFactory(): ZLinkActorRuntimeOptionsFactory {
     return new ZLinkActorRuntimeOptionsFactory({
+      requests: this.requests,
       terminateActorActivation: (actor, terminal) =>
         this.spotNodeRuntime?.terminateActorActivation(actor, terminal) ?? terminal(),
       registration: this.options.registration,
@@ -2763,7 +2854,7 @@ export class ZLinkFrameworkRuntimeHost
             `Actor '${actorRef.actorId}' authority requires its RouteMesh Entry Spot.`
           );
         }
-        await publishInitialActorAuthority(
+        const authority = await publishInitialActorAuthority(
           store,
           {
             actorType,
@@ -2777,7 +2868,10 @@ export class ZLinkFrameworkRuntimeHost
           },
           signal
         );
-        this.actorClientLocationResolver?.invalidateActorRoute(actorRef.actorId);
+        this.actorClientLocationResolver?.observeActorAuthorityVersion(
+          actorRef.actorId,
+          authority.storeVersion.value
+        );
       },
       reportPostCommitError: (error) =>
         this.runtimeOrPreStartErrorSink.reportRuntimeTaskException(
@@ -2816,6 +2910,7 @@ export class ZLinkFrameworkRuntimeHost
     return new ZLinkChannelRuntimeOptionsFactory({
       monitoringAdapter: this.backendAdapterFactory.createMonitoringAdapter(),
       messageFlowModeCell: this.messageFlowModeCell,
+      telemetryLease: this.telemetryLease,
       boundSessionRelay: this.boundSessionRelay,
       applicationJobQueue: this.applicationJobQueue,
       spotManager: () => this.spotManager,
@@ -2824,7 +2919,7 @@ export class ZLinkFrameworkRuntimeHost
     }).create();
   }
 
-  private createSpotNodeRuntimeOptions(
+  createSpotNodeRuntimeOptions(
     context: ZLinkBackendContext,
     dispatchErrors: ZLinkDispatchErrorReporter
   ) {
@@ -2854,6 +2949,7 @@ export class ZLinkFrameworkRuntimeHost
     }).create();
     return {
       ...options,
+      admission: this.admission,
       errorSink: this.runtimeOrPreStartErrorSink,
       peerAdmissionSealed: (meshName: string) => !this.admission.accepts(meshName),
       messageFollowReceiver: (record: ServiceMessageFollowRecord) => {
@@ -2946,7 +3042,7 @@ export class ZLinkFrameworkRuntimeHost
     return;
   }
 
-  private dispatchMeshRecord(
+  dispatchMeshRecord(
     applicationJobOrigin: () => ApplicationJobOrigin,
     meshName: string,
     owner: ReadyRecord,
@@ -2983,28 +3079,54 @@ export class ZLinkFrameworkRuntimeHost
           meshName,
           record,
           signal,
-          (dispatch, onRejected) =>
+          (dispatch, onRejected, receiver, context) =>
             this.admitMeshDispatch(
               meshName,
               record,
               'RouteMesh channel dispatch',
               dispatch,
-              onRejected
+              onRejected,
+              receiver,
+              context
             )
         );
       }
       case ReceiveKind.NodeSend:
       case ReceiveKind.NodeRequest: {
-        return this.dispatchMeshNodeRecord(meshName, record, signal, (dispatch, onRejected) =>
-          this.admitMeshDispatch(meshName, record, 'RouteMesh node dispatch', dispatch, onRejected)
+        return this.dispatchMeshNodeRecord(
+          meshName,
+          record,
+          signal,
+          (dispatch, onRejected, receiver, context) =>
+            this.admitMeshDispatch(
+              meshName,
+              record,
+              'RouteMesh node dispatch',
+              dispatch,
+              onRejected,
+              receiver,
+              context
+            )
         );
       }
 
       case ReceiveKind.SpotSend:
       case ReceiveKind.SpotRequest:
       case ReceiveKind.SpotMulticast:
-        return this.dispatchMeshSpotRecord(meshName, owner, record, (dispatch, onRejected) =>
-          this.admitMeshDispatch(meshName, record, 'RouteMesh Spot dispatch', dispatch, onRejected)
+        return this.dispatchMeshSpotRecord(
+          meshName,
+          owner,
+          record,
+          (dispatch, onRejected, receiver, context) =>
+            this.admitMeshDispatch(
+              meshName,
+              record,
+              'RouteMesh Spot dispatch',
+              dispatch,
+              onRejected,
+              receiver,
+              context
+            )
         );
       case ReceiveKind.InstanceSpotActivation:
         if (this.spotManager === undefined) {
@@ -3016,13 +3138,15 @@ export class ZLinkFrameworkRuntimeHost
           meshName,
           owner,
           record,
-          (dispatch, onRejected) =>
+          (dispatch, onRejected, receiver, context) =>
             this.admitMeshDispatch(
               meshName,
               record,
               'RouteMesh Instance Spot dispatch',
               dispatch,
-              onRejected
+              onRejected,
+              receiver,
+              context
             )
         );
       case ReceiveKind.SpotControl:
@@ -3044,13 +3168,15 @@ export class ZLinkFrameworkRuntimeHost
           meshName,
           owner,
           record,
-          (dispatch, onRejected) =>
+          (dispatch, onRejected, receiver, context) =>
             this.admitMeshDispatch(
               meshName,
               record,
               'RouteMesh Actor dispatch',
               dispatch,
-              onRejected
+              onRejected,
+              receiver,
+              context
             )
         );
       case ReceiveKind.SendReady:
@@ -3062,18 +3188,21 @@ export class ZLinkFrameworkRuntimeHost
     }
   }
 
-  private admitMeshDispatch(
+  private admitMeshDispatch<TReceiver = void, TContext = void>(
     meshName: string,
     record: ReceiveRecord,
     operation: string,
-    dispatch: () => void | Promise<void>,
-    onRejected?: (error: unknown) => void | Promise<void>
+    dispatch: (this: TReceiver, context: TContext) => void | Promise<void>,
+    onRejected?: (this: TReceiver, error: unknown, context: TContext) => void | Promise<void>,
+    receiver?: TReceiver,
+    context?: TContext
   ): void | Promise<void> {
     // Execution gate §13 and host relocation §14 keep control progress independent
     // of new application admission. Disconnect callbacks still use their owner gate.
-    if (record.domain === ReadyDomain.Infrastructure) return dispatch();
+    if (record.domain === ReadyDomain.Infrastructure)
+      return dispatch.call(receiver as TReceiver, context as TContext);
     if (
-      !this.admission.accepts(meshName) &&
+      !this.admission.accepts(meshName, record.kind) &&
       (record.kind === ReceiveKind.NodeSend ||
         record.kind === ReceiveKind.NodeRequest ||
         record.kind === ReceiveKind.SpotSend ||
@@ -3089,13 +3218,21 @@ export class ZLinkFrameworkRuntimeHost
         decodeStreamHeader(Buffer.from(relay.header, 'base64'), this.flowCreationEnabled()).name ===
           ZLINK_REMOTE_ACTOR_SESSION_DISCONNECTED_PACKET
       ) {
-        return dispatch();
+        return dispatch.call(receiver as TReceiver, context as TContext);
       }
     }
-    return this.admission.run(meshName, operation, dispatch, onRejected);
+    return this.admission.run(
+      meshName,
+      operation,
+      dispatch,
+      record.kind,
+      onRejected,
+      receiver,
+      context
+    );
   }
 
-  private async dispatchMeshActorBinding(
+  async dispatchMeshActorBinding(
     meshName: string,
     record: ReceiveRecord,
     signal?: AbortSignal
@@ -3184,8 +3321,7 @@ export class ZLinkFrameworkRuntimeHost
       replyBindingFailure(ZLinkFrameworkInternalErrorKind.ActorLocationStale);
       return;
     }
-    replyBindingSuccess();
-    state.installBoundSessionBinding({
+    const incomingTarget = {
       ...target,
       sessionNodeRid: binding.sessionNodeRid,
       sessionRid: binding.sessionRid,
@@ -3193,24 +3329,125 @@ export class ZLinkFrameworkRuntimeHost
       sessionOwnerId: binding.sessionOwnerId,
       sessionOwnerLeaseGeneration: binding.sessionOwnerLeaseGeneration,
       bindingGeneration: binding.bindingGeneration
+    };
+    const currentBinding = state.boundSession;
+    const currentActor = state.actor;
+    if (
+      currentBinding === undefined ||
+      currentActor?.onDuplicateBind === undefined ||
+      samePhysicalSession(currentBinding, incomingTarget)
+    ) {
+      state.installBoundSessionBinding(incomingTarget);
+      replyBindingSuccess();
+      return;
+    }
+    const spotManager = this.spotManager;
+    if (spotManager === undefined || state.meshName === undefined) {
+      replyBindingFailure(ZLinkFrameworkInternalErrorKind.ActorLocationStale);
+      return;
+    }
+
+    let callbackStarted!: () => void;
+    const started = new Promise<void>((resolve) => {
+      callbackStarted = resolve;
     });
-    return Promise.resolve();
+    let callbackTimedOut = false;
+    let timeoutHandle: ReturnType<typeof setTimeout> | undefined;
+    const cancellation = new AbortController();
+    const callbackSignal =
+      signal === undefined ? cancellation.signal : AbortSignal.any([signal, cancellation.signal]);
+    const turn = spotManager.executeActorTurn(
+      state.meshName,
+      state.actor?.context.spotId,
+      state.actorId,
+      async () => {
+        const current = state.boundSession;
+        const actor = state.actor;
+        const duplicateCallback =
+          current !== undefined && !samePhysicalSession(current, incomingTarget)
+            ? actor?.onDuplicateBind
+            : undefined;
+        if (duplicateCallback === undefined || actor === undefined || current === undefined) {
+          state.installBoundSessionBinding(incomingTarget);
+          replyBindingSuccess();
+          return;
+        }
+        callbackStarted();
+        const decision = await duplicateCallback.call(
+          actor,
+          sessionBindingIdentity(current),
+          sessionBindingIdentity(incomingTarget),
+          callbackSignal
+        );
+        if (callbackTimedOut) return;
+        if (decision !== undefined && decision.length > 0) {
+          if (Buffer.byteLength(decision, 'utf8') > 0xffff) {
+            throw new RangeError('Duplicate bind rejection message exceeds Text16 capacity.');
+          }
+          if (
+            record.replyFailure?.(RequestResult.Rejected, 0, {
+              kind: 'streamBindRejected',
+              rejectionMessage: decision
+            }) !== SubmitResult.Ok
+          ) {
+            throw new ZLinkConfigurationException(
+              `MeshNode Actor binding target '${binding.actor.actorId}' rejection acknowledgement failed.`
+            );
+          }
+          return;
+        }
+        state.installBoundSessionBinding(incomingTarget);
+        replyBindingSuccess();
+      }
+    );
+    const timeout = started.then(
+      () =>
+        new Promise<'timedOut'>((resolve) => {
+          timeoutHandle = setTimeout(() => {
+            callbackTimedOut = true;
+            resolve('timedOut');
+          }, this.options.registration.sessionReplacementCallbackTimeoutMs);
+        })
+    );
+    try {
+      const outcome = await Promise.race([turn.then(() => 'completed' as const), timeout]);
+      if (outcome === 'timedOut') {
+        cancellation.abort(createDeadlineExceededError('Duplicate bind callback timed out.'));
+        if (record.replyFailure?.(RequestResult.TimedOut, 0) !== SubmitResult.Ok) {
+          throw new ZLinkConfigurationException(
+            `MeshNode Actor binding target '${binding.actor.actorId}' timeout acknowledgement failed.`
+          );
+        }
+        void turn.catch((error: unknown) =>
+          this.runtimeOrPreStartErrorSink.reportRuntimeTaskException(
+            `duplicate bind callback '${binding.actor.actorId}'`,
+            error
+          )
+        );
+      }
+    } finally {
+      if (timeoutHandle !== undefined) clearTimeout(timeoutHandle);
+    }
   }
 
-  private submitLocalMeshRoute(
-    meshName: string,
-    sourceNodeRid: string,
-    parts: readonly MessageLike[]
-  ) {
+  submitLocalMeshRoute(meshName: string, sourceNodeRid: string, parts: readonly MessageLike[]) {
     const state = this.executionState;
     const channelRuntime = this.channelRuntime;
-    if (state === undefined || channelRuntime === undefined || !this.admission.accepts(meshName)) {
+    if (
+      state === undefined ||
+      channelRuntime === undefined ||
+      !this.admission.accepts(meshName, ReceiveKind.NodeSend)
+    ) {
       return { status: ZLinkSubmitStatus.Shutdown } as const;
     }
     if (!channelRuntime.canDispatchLocalMeshRoute(meshName)) {
       return { status: ZLinkSubmitStatus.TargetNotFound } as const;
     }
-    const claim = this.admission.claim(meshName, 'RouteMesh local node dispatch');
+    const claim = this.admission.claim(
+      meshName,
+      'RouteMesh local node dispatch',
+      ReceiveKind.NodeSend
+    );
     const ownedParts: RuntimeMessage[] = [];
     try {
       for (const part of parts) {
@@ -3251,7 +3488,7 @@ export class ZLinkFrameworkRuntimeHost
     return { status: ZLinkSubmitStatus.Submitted } as const;
   }
 
-  private async dispatchMeshSpotRecord(
+  async dispatchMeshSpotRecord(
     meshName: string,
     owner: ReadyRecord,
     record: ReceiveRecord,
@@ -3268,7 +3505,8 @@ export class ZLinkFrameworkRuntimeHost
         contentType: envelope.header.contentType,
         sourceNodeRid: String(record.sourceNodeRid ?? '') as RoutingId,
         metadata: zlinkMessageMetadata(envelope.header.metadata),
-        correlationId: envelope.header.correlationId ?? undefined
+        correlationId: envelope.header.correlationId ?? undefined,
+        receivedAt: record.receivedAt ?? new Date()
       };
       try {
         const dispatch = async () => {
@@ -3299,20 +3537,12 @@ export class ZLinkFrameworkRuntimeHost
     await this.spotManager.dispatchMeshSpot(meshName, owner, record, admit);
   }
 
-  private createDispatchErrorReporter(
-    errorSink: ZLinkDispatchErrorSink
-  ): ZLinkDispatchErrorReporter {
+  createDispatchErrorReporter(errorSink: ZLinkDispatchErrorSink): ZLinkDispatchErrorReporter {
     const existing = this.dispatchErrorReporters.get(errorSink);
     if (existing !== undefined) {
       return existing;
     }
-    const reporter = new ZLinkDispatchErrorReporter(
-      undefined,
-      undefined,
-      errorSink,
-      this.diagnosticsContext(),
-      this.metrics
-    );
+    const reporter = new ZLinkDispatchErrorReporter(this.diagnosticsContext(), this.metrics);
     this.dispatchErrorReporters.set(errorSink, reporter);
     return reporter;
   }
@@ -3336,12 +3566,13 @@ export class ZLinkFrameworkRuntimeHost
     this.cachedDiagnosticsContext ??= createDiagnosticsContext(
       this.options.registration.dispatch,
       this.options.providerResolver,
-      this.messageFlowModeCell
+      this.messageFlowModeCell,
+      this.telemetryLease
     );
     return this.cachedDiagnosticsContext;
   }
 
-  private flowCreationEnabled(): boolean {
+  flowCreationEnabled(): boolean {
     const mode =
       this.executionState === undefined
         ? (this.options.registration.dispatch?.diagnostics.messageFlow ?? 'errors')
@@ -3370,7 +3601,7 @@ export class ZLinkFrameworkRuntimeHost
     return this.locationOwner.ensureRuntime(this.meshRouters.primaryMeshName());
   }
 
-  private createLocationSpotRouteResolver(): ZLinkSpotRouteResolver | undefined {
+  createLocationSpotRouteResolver(): ZLinkSpotRouteResolver | undefined {
     if (this.cachedLocationSpotRouteResolver !== undefined) {
       return this.cachedLocationSpotRouteResolver;
     }
@@ -3409,7 +3640,7 @@ export class ZLinkFrameworkRuntimeHost
     return resolver;
   }
 
-  private createActorLocationResolver(): ZLinkStoreLocationResolvers | undefined {
+  createActorLocationResolver(): ZLinkStoreLocationResolvers | undefined {
     return this.locationOwner.createActorLocationResolver(this.meshRouters.spotLocationMeshNames());
   }
 }
@@ -3451,6 +3682,50 @@ export function hasUnsupportedManualTopology(registration: ZLinkFrameworkRegistr
   return false;
 }
 
+function samePhysicalSession(
+  left: {
+    readonly sessionNodeRid?: RoutingId;
+    readonly sessionRid?: RoutingId;
+    readonly sessionOwnerNodeGeneration?: bigint;
+  },
+  right: {
+    readonly sessionNodeRid?: RoutingId;
+    readonly sessionRid?: RoutingId;
+    readonly sessionOwnerNodeGeneration?: bigint;
+  }
+): boolean {
+  return (
+    left.sessionNodeRid !== undefined &&
+    left.sessionRid !== undefined &&
+    left.sessionOwnerNodeGeneration !== undefined &&
+    right.sessionNodeRid !== undefined &&
+    right.sessionRid !== undefined &&
+    right.sessionOwnerNodeGeneration !== undefined &&
+    routingIdsEqual(left.sessionNodeRid, right.sessionNodeRid) &&
+    left.sessionRid === right.sessionRid &&
+    left.sessionOwnerNodeGeneration === right.sessionOwnerNodeGeneration
+  );
+}
+
+function sessionBindingIdentity(value: {
+  readonly sessionNodeRid?: RoutingId;
+  readonly sessionOwnerNodeGeneration?: bigint;
+  readonly bindingGeneration?: bigint;
+}): ZLinkSessionBindingIdentity {
+  if (
+    value.sessionNodeRid === undefined ||
+    value.sessionOwnerNodeGeneration === undefined ||
+    value.bindingGeneration === undefined
+  ) {
+    throw new ZLinkConfigurationException('Duplicate bind callback requires a complete identity.');
+  }
+  return {
+    sessionOwnerNodeRid: value.sessionNodeRid,
+    sessionOwnerNodeGeneration: value.sessionOwnerNodeGeneration,
+    bindingGeneration: value.bindingGeneration
+  };
+}
+
 function hasConfiguredLocationStore(registration: ZLinkFrameworkRegistration): boolean {
   return (
     registration.locations.useInMemoryStores || registration.locations.storeInstance !== undefined