@zlink-systems/framework 0.25.0 → 0.27.0

package/src/runtime/host/index.ts

@@ -1,78 +1,98 @@
-import {
-  ZLinkFrameworkInternalErrorKind,
-  createInternalFrameworkException,
-  internalFrameworkWireReply
-} from '../framework-errors-internal';
+import { listAllMeshNodeDescriptors } from '../locations/location-store-pages';
+import { ZlinkStreamContentType } from '@zlink-systems/stream-wire';
 import { randomBytes, randomUUID } from 'node:crypto';
-import { ZLinkBufferMessage as RuntimeMessage } from '../backend/runtime-message';
-import {
-  RequestResult,
-  SubmitResult,
-  type ZLinkBackendMessageLike as MessageLike
-} from '../backend/runtime-values';
-import {
-  OperationKind,
-  ReceiveKind,
-  type ReadyRecord,
-  type ReceiveRecord
-} from '../foundation/service-runtime-contracts';
-import type { ServiceMessageFollowRecord } from '../foundation/service-stateful-wire-codec';
-import { runtimeAcceptsWork, runtimeStateIsReady } from '../foundation/runtime-state-projections';
-import { createDeadlineExceededError, isDeadlineExceededError } from '../abort';
-import { meshActorSessionNodeAdapter, ZLinkNodeBackendAdapterFactory } from '../backend';
-import type {
-  ZLinkBackendAdapterFactory,
-  ZLinkBackendContext,
-  ZLinkBackendMeshNode
-} from '../backend';
-
-const LEGACY_MESH_SEND_TIMEOUT_MS = 1000;
-import type { ZLinkFrameworkRegistration } from '../configuration';
-import type {
-  ActorRef,
-  RoutingId,
-  ZLinkMeshNodeDescriptor,
-  ZLinkClientServerRuntime,
-  ZLinkFanoutRuntime,
-  ZLinkFrameworkRuntime,
-  ZLinkFrameworkLifecycleOptions,
-  ZLinkFrameworkRelocationOptions,
-  ZLinkFrameworkRelocationResult,
-  ZLinkObservedStatus,
-  ZLinkFrameworkRuntimeStatus,
-  ZLinkFrameworkTerminationResult,
-  ZLinkRouteMeshRuntime
-} from '../../contracts';
-import type { ZLinkLocationOwnerToken } from '../../contracts/Locations';
-import type { ZLinkProviderResolver } from '../../contracts/Common/ZLinkProviderResolver';
-import type { Type } from '../../contracts/Common/CoreTypes';
-import type { ZLinkRuntimeEventPublisher } from '../diagnostics';
-import type { ZLinkSpotRouteResolver } from '../spots/spot-routing-internal';
 import {
+  type ActorRef,
+  type RoutingId,
+  type ZLinkClientServerRuntime,
+  type ZLinkFanoutRuntime,
+  type ZLinkFrameworkLifecycleOptions,
+  type ZLinkFrameworkRelocationOptions,
+  type ZLinkFrameworkRelocationResult,
+  type ZLinkFrameworkRuntime,
+  type ZLinkFrameworkRuntimeStatus,
+  type ZLinkFrameworkTerminationResult,
+  type ZLinkListenerKind,
+  type ZLinkListenerStatus,
+  type ZLinkMeshNodeDescriptor,
+  type ZLinkMessageFlowControl,
+  type ZLinkMessageFlowLogMode,
+  type ZLinkObservedStatus,
+  type ZLinkRouteMeshRuntime,
   ZLinkFrameworkRelocationMode,
   ZLinkFrameworkRelocationOutcome,
   ZLinkFrameworkRelocationReason,
   ZLinkFrameworkRuntimeState,
   ZLinkFrameworkTerminationOutcome,
   ZLinkFrameworkTerminationReason,
+  zlinkMessageMetadata,
   ZLinkObjectRole,
   ZLinkPeerState,
-  ZLinkSpotKind,
-  zlinkMessageMetadata,
-  ZLinkSpotCreateState
+  ZLinkSpotCreateState,
+  ZLinkSpotKind
 } from '../../contracts';
-import { ZLinkSubmitStatus } from '../messaging/submission-result';
+import type { Type } from '../../contracts/Common/CoreTypes';
+import type { ZLinkProviderResolver } from '../../contracts/Common/ZLinkProviderResolver';
+import { ZLinkConfigurationException } from '../../contracts/Configuration/ConfigurationException';
+import { requireMessageFlowLogMode } from '../../contracts/Configuration/DiagnosticsValidation';
+import { DEFAULT_ACTIVATION_CONCURRENCY_LIMIT } from '../../contracts/Configuration/InternalDefaults';
+import { DEFAULT_REQUEST_TIMEOUT_MS } from '../../contracts/Configuration/Registration';
 import {
-  ZLinkRuntimeMessageFlowOutcome as ZLinkMessageFlowOutcome,
   ZLinkDispatchErrorSurface,
-  ZLinkDispatchMessageKind
+  ZLinkDispatchMessageKind,
+  ZLinkRuntimeMessageFlowOutcome as ZLinkMessageFlowOutcome
 } from '../../contracts/Dispatch/ZLinkDispatchOptions';
-import type { ZLinkMessageFlowControl, ZLinkMessageFlowLogMode } from '../../contracts';
-import { requireMessageFlowLogMode } from '../../contracts/Configuration/DiagnosticsValidation';
+import {
+  type ZLinkLocationOwnerToken,
+  zlinkDefaultLocationOptions,
+  type ZLinkLocationRuntimeQuery
+} from '../../contracts/Locations';
+import { ZLINK_PROVIDER_MAX_PAGE_SIZE } from '../../contracts/Locations/Stores';
+import { createDeadlineExceededError, isAbortError, isDeadlineExceededError } from '../abort';
+import { ZLinkActivationAdmission } from '../activation-admission';
+import {
+  type DefaultZLinkActorManager,
+  type ZLinkActorManagerOptions,
+  decodeRemoteActorSourceLeaveTerminal,
+  DefaultZLinkActorClient,
+  isActorAuthorityPayload,
+  publishInitialActorAuthority,
+  rewriteActorAuthorityOwner,
+  ZLINK_REMOTE_ACTOR_PACKET_RELAY_PACKET,
+  ZLinkActorHandoffCoordinator,
+  ZLinkActorTransferRegistry
+} from '../actors';
+import { messageFollowOwnerNodeRid, ownerFence } from '../actors/actor-message-follow-context';
+import {
+  ZLINK_INTERNAL_ACTOR_TRANSPORT_DELIVERY_GATE,
+  type ZLinkInternalActorTransportDeliveryGate
+} from '../actors/actor-transport-delivery-gate';
+import { ZLinkRuntimeAdmissionGate } from '../admission';
+import {
+  ZLINK_INTERNAL_APPLICATION_JOB_QUEUE_HANDLER_START_GATE,
+  type ZLinkInternalApplicationJobQueueHandlerStartGate
+} from '../application-jobs/application-job-queue-handler-start-gate';
+import {
+  releaseApplicationJobPermitBeforeHandler,
+  runWithApplicationJobPermit
+} from '../application-jobs/application-job-queue-scope';
+import {
+  type ZLinkBackendAdapterFactory,
+  type ZLinkBackendContext,
+  type ZLinkBackendMeshNode,
+  meshActorSessionNodeAdapter,
+  ZLinkNodeBackendAdapterFactory
+} from '../backend';
+import { ZLinkBufferMessage as RuntimeMessage } from '../backend/runtime-message';
+import {
+  RequestResult,
+  SubmitResult,
+  type ZLinkBackendMessageLike as MessageLike
+} from '../backend/runtime-values';
 import {
   DefaultZLinkChannelRuntimeOptions,
-  ZLinkDispatchErrorReporter,
   ZLinkChannelRuntimeManager,
+  ZLinkDispatchErrorReporter,
   ZLinkRuntimeRouteTransport,
   type ZLinkDispatchErrorSink
 } from '../channels';
@@ -82,33 +102,72 @@ import {
   encodeChannelErrorReplyParts,
   encodeChannelReplyParts
 } from '../channels/channel-envelope';
-import { ZLINK_REMOTE_ACTOR_PACKET_RELAY_PACKET } from '../actors';
-import {
-  ZLINK_INTERNAL_ACTOR_TRANSPORT_DELIVERY_GATE,
-  type ZLinkInternalActorTransportDeliveryGate
-} from '../actors/actor-transport-delivery-gate';
-import {
-  ZLINK_INTERNAL_APPLICATION_JOB_QUEUE_HANDLER_START_GATE,
-  type ZLinkInternalApplicationJobQueueHandlerStartGate
-} from '../application-jobs/application-job-queue-handler-start-gate';
-import { ZLinkFrameworkExecutionState, ZLinkRuntimeTaskErrorSink } from '../execution';
+import type { ZLinkFrameworkRegistration } from '../configuration';
 import {
+  type ZLinkRuntimeEventPublisher,
   createDiagnosticsContext,
   createInboundFlow,
   currentOrCreateFlow,
   DefaultZLinkRuntimeEventPublisher,
   flowIfEnabled,
   runWithFlow,
+  ZLinkRuntimeMetrics,
   type ZLinkDiagnosticsContext,
   type ZLinkMessageFlowModeCell,
-  type ZLinkRuntimeMetricMeshSnapshot,
-  ZLinkRuntimeMetrics
+  type ZLinkRuntimeMetricMeshSnapshot
 } from '../diagnostics';
+import { METRIC_NAMES, MILLISECONDS_PER_SECOND } from '../diagnostics/runtime-metrics';
+import { RuntimeEventQueue } from '../diagnostics/runtime-observation-queue';
 import {
   ZLinkClientServerRuntimeProjection,
   ZLinkFanoutRuntimeProjection
 } from '../diagnostics/topology-runtime-projections';
-import { RuntimeEventQueue } from '../diagnostics/runtime-observation-queue';
+import { ZLinkFrameworkExecutionState, ZLinkRuntimeTaskErrorSink } from '../execution';
+import { ZLinkListenerRecords } from '../foundation/listener-records';
+import {
+  runtimeAcceptsWork,
+  runtimeObservationIsTerminal,
+  runtimeStateIsReady
+} from '../foundation/runtime-state-projections';
+import { rewriteServiceAuthorityOwner } from '../foundation/service-authority-payload-codec';
+import { ServiceRelocationAuthorityError } from '../foundation/service-relocation-coordinator';
+import {
+  replaceServiceRelocationAuthorityApplicationPayload,
+  serviceRelocationAuthorityApplicationPayload
+} from '../foundation/service-relocation-runtime';
+import {
+  MeshPeerRuntimeState,
+  OperationKind,
+  ReceiveKind,
+  type ReadyRecord,
+  type ReceiveRecord
+} from '../foundation/service-runtime-contracts';
+import type {
+  ServiceAsyncInstanceActivationAuthority,
+  ServiceInstanceApplicationLifecycle
+} from '../foundation/service-stateful-runtime';
+import type { ServiceMessageFollowRecord } from '../foundation/service-stateful-wire-codec';
+import {
+  createInternalFrameworkException,
+  internalFrameworkWireReply,
+  ZLinkFrameworkInternalErrorKind
+} from '../framework-errors-internal';
+import {
+  ZLinkAuthoritySpotRouteResolver,
+  ZLinkOwnerCleanupError,
+  type ZLinkLocationRuntime,
+  type ZLinkStoreLocationResolvers
+} from '../locations';
+import {
+  decodeFrameworkCreationPayload,
+  encodeFrameworkCreationPayload
+} from '../messaging/creation-payload-codec';
+import {
+  decodeFrameworkPayloadMessage,
+  encodeFrameworkPayloadMessage
+} from '../messaging/payload-codec';
+import { ZLinkSubmitStatus } from '../messaging/submission-result';
+import { decodeRoutingId, encodeRoutingIdStorageHex, routingIdsEqual } from '../routing-id';
 import {
   DefaultZLinkSpotManager,
   ZLinkPublicSpotManager,
@@ -117,83 +176,52 @@ import {
   type ZLinkDetachedTaskRunner,
   type ZLinkSpotManagerOptions
 } from '../spots';
-import type { DefaultZLinkActorManager, ZLinkActorManagerOptions } from '../actors';
-import { messageFollowOwnerNodeRid, ownerFence } from '../actors/actor-message-follow-context';
-import {
-  DefaultZLinkActorClient,
-  ZLinkActorHandoffCoordinator,
-  ZLinkActorTransferRegistry,
-  decodeRemoteActorSourceLeaveTerminal,
-  isActorAuthorityPayload,
-  publishInitialActorAuthority,
-  rewriteActorAuthorityOwner
-} from '../actors';
+import type { ZLinkSpotRouteResolver } from '../spots/spot-routing-internal';
 import {
   DefaultZLinkBoundSessionFactory,
-  type DefaultZLinkBoundSession,
-  type ZLinkStreamPayloadCodec,
   ZLinkStreamBindingRuntime,
-  ZLinkStreamRuntimeManager
+  ZLinkStreamRuntimeManager,
+  type DefaultZLinkBoundSession,
+  type ZLinkStreamPayloadCodec
 } from '../streams';
-import {
-  ZLinkOwnerCleanupError,
-  ZLinkAuthoritySpotRouteResolver,
-  type ZLinkLocationRuntime,
-  type ZLinkStoreLocationResolvers
-} from '../locations';
-import {
-  zlinkDefaultLocationOptions,
-  type ZLinkLocationRuntimeQuery
-} from '../../contracts/Locations';
+import { ZLinkActorPlacementCoordinator } from './actor-placement-coordinator';
 import { ZLinkActorRuntimeOptionsFactory } from './actor-runtime-options-factory';
-import { ZLinkActorTransferRuntime } from './actor-transfer-runtime';
 import { ZLinkActorTransferAuthorityRuntime } from './actor-transfer-authority-runtime';
-import { ZLinkEntryActorRuntimeService } from './entry-actor-runtime';
-import { ZLinkLocationRuntimeOwner } from './location-runtime-owner';
-import { MeshRouterResolver } from './mesh-router-resolver';
-import { ZLinkBoundSessionRelay } from './bound-session-relay';
-import { decodeRoutingId, encodeRoutingIdStorageHex, routingIdsEqual } from '../routing-id';
-import { rewriteServiceAuthorityOwner } from '../foundation/service-authority-payload-codec';
+import { ZLinkActorTransferRuntime } from './actor-transfer-runtime';
 import {
-  replaceServiceRelocationAuthorityApplicationPayload,
-  serviceRelocationAuthorityApplicationPayload
-} from '../foundation/service-relocation-runtime';
-import { ZLinkSpotRuntimeOptionsFactory } from './spot-runtime-options-factory';
+  ApplicationJobQueue,
+  nodeEffectiveProcessorCount,
+  resolveApplicationJobQueueConfiguration
+} from './application-job-queue';
+import { ZLinkBoundSessionRelay } from './bound-session-relay';
 import { ZLinkChannelRuntimeOptionsFactory } from './channel-runtime-options-factory';
-import { ZLinkSpotNodeRuntimeOptionsFactory } from './spot-node-runtime-options-factory';
-import { rollbackRuntimeStart, stopRuntimeParts } from './runtime-shutdown';
-import { ZLinkRuntimeAdmissionGate } from '../admission';
-import { ZLinkRetiringRollbackError, ZLinkRouteMeshRuntimeCoordinator } from './route-mesh-runtime';
-import { ZLinkConfigurationException } from '../../contracts/Configuration/ConfigurationException';
-import { ZLinkStatefulAuthorityRouteRuntime } from './stateful-authority-route-runtime';
+import { ZLinkEntryActorRuntimeService } from './entry-actor-runtime';
+import { HostCapacityStatusProjection } from './host-capacity-status';
 import { ZLinkInstanceActivationAuthority } from './instance-activation-authority';
-import type {
-  ServiceAsyncInstanceActivationAuthority,
-  ServiceInstanceApplicationLifecycle
-} from '../foundation/service-stateful-runtime';
-import { hasObjectClientCapability, ZLinkHostSpotAddressTransport } from './spot-address-transport';
-import { ZLinkUserSpotCreationCoordinator } from './user-spot-creation-coordinator';
-import { ZLinkActorPlacementCoordinator } from './actor-placement-coordinator';
-import {
-  decodeFrameworkPayloadMessage,
-  encodeFrameworkPayloadMessage
-} from '../messaging/payload-codec';
+import { ZLinkLocationRuntimeOwner } from './location-runtime-owner';
+import { MeshRouterResolver } from './mesh-router-resolver';
 import {
-  decodeFrameworkCreationPayload,
-  encodeFrameworkCreationPayload
-} from '../messaging/creation-payload-codec';
+  ZLinkDrainingStatePublishError,
+  ZLinkRetiringRollbackError,
+  ZLinkRouteMeshRuntimeCoordinator
+} from './route-mesh-runtime';
 import { DefaultZLinkRouteMeshRuntimeOptions } from './route-mesh-runtime-options';
+import { rollbackRuntimeStart, stopRuntimeParts } from './runtime-shutdown';
 import {
   ZLinkHostServiceRelocationRuntime,
   ZLinkRelocationStateIncompatibleError
 } from './service-relocation-host-runtime';
-import {
-  ApplicationJobQueue,
-  nodeEffectiveProcessorCount,
-  resolveApplicationJobQueueConfiguration
-} from './application-job-queue';
-import { runWithApplicationJobPermit } from '../application-jobs/application-job-queue-scope';
-import { HostCapacityStatusProjection } from './host-capacity-status';
+import { hasObjectClientCapability, ZLinkHostSpotAddressTransport } from './spot-address-transport';
+import { ZLinkSpotNodeRuntimeOptionsFactory } from './spot-node-runtime-options-factory';
+import { ZLinkSpotRuntimeOptionsFactory } from './spot-runtime-options-factory';
+import { ZLinkStatefulAuthorityRouteRuntime } from './stateful-authority-route-runtime';
+import { ZLinkUserSpotCreationCoordinator } from './user-spot-creation-coordinator';
+
+const HOST_SHUTDOWN_POLL_INTERVAL_MS = 100;
+const HANDOFF_ACCEPTANCE_POLL_INTERVAL_MS = 10;
+const DEFAULT_HOST_CONTROL_TIMEOUT_MS = 30_000;
+
+const LEGACY_MESH_SEND_TIMEOUT_MS = 1000;
 
 export interface ZLinkFrameworkRuntimeLifecycle {
   readonly isStarted: boolean;
@@ -219,6 +247,7 @@ export class ZLinkFrameworkRuntimeHost
   private runtimeSequence = 0n;
   private readonly runtimeObservationSource = Symbol('zlink.framework-runtime');
   private runtimeDeadline?: Date;
+  private shutdownDeadline?: AbortController;
   private runtimeRelocationResult?: ZLinkFrameworkRelocationResult;
   private runtimeTerminationResult?: ZLinkFrameworkTerminationResult;
   private relocationOperation?: Promise<ZLinkFrameworkRelocationResult>;
@@ -245,6 +274,7 @@ export class ZLinkFrameworkRuntimeHost
   private actorManager?: DefaultZLinkActorManager;
   private actorPlacement?: ZLinkActorPlacementCoordinator;
   private spotManager?: DefaultZLinkSpotManager;
+  private userSpotCoordinator?: ZLinkUserSpotCreationCoordinator;
   private ownerLeaseRecoveryRuntime?: ZLinkLocationRuntime;
   private ownerLeaseRecoveryHandler?: () => void;
   private registerUserSpotHandlers?: (runtime: ZLinkSpotNodeRuntimeManager) => void;
@@ -256,6 +286,11 @@ export class ZLinkFrameworkRuntimeHost
   private readonly metricRegistrations: import('../diagnostics').ZLinkRuntimeMetricRegistration[] =
     [];
   private readonly admission = new ZLinkRuntimeAdmissionGate(() => this.ownerAdmissionOpen());
+  private readonly activationAdmission = new ZLinkActivationAdmission(
+    (meshName) =>
+      this.options.registration.spotNodes.get(meshName)?.activationConcurrencyLimit ??
+      DEFAULT_ACTIVATION_CONCURRENCY_LIMIT
+  );
   private cachedLocationSpotRouteResolver?: ZLinkSpotRouteResolver;
   private actorClientLocationResolver?: ZLinkStoreLocationResolvers;
   // Shared, runtime-mutable message-flow mode cell — installed once so
@@ -360,7 +395,8 @@ export class ZLinkFrameworkRuntimeHost
         this.meshRouters.classifyManualNodeTarget(meshName, targetNodeRid),
       (meshName, sourceNodeRid, parts) => this.submitLocalMeshRoute(meshName, sourceNodeRid, parts),
       this.metrics,
-      () => this.flowCreationEnabled()
+      () => this.flowCreationEnabled(),
+      () => this.createDispatchErrorReporter(this.runtimeOrPreStartErrorSink).flow
     );
     this.spotAddressTransport = new ZLinkHostSpotAddressTransport({
       resolver: () => this.createLocationSpotRouteResolver(),
@@ -373,7 +409,7 @@ export class ZLinkFrameworkRuntimeHost
       meshNode: (meshName) => this.spotNodeRuntime?.meshNode(meshName),
       completions: (meshName) => this.spotNodeRuntime?.meshCompletionTable(meshName),
       codecs: { serializers: options.registration.messageSerializers },
-      defaultRequestTimeoutMs: options.registration.requestTimeoutMs ?? 30_000,
+      defaultRequestTimeoutMs: options.registration.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS,
       defaultSendTimeoutMs: defaultSpotSendTimeoutMs,
       sendTimeoutMsForMesh: sendTimeoutMsForSpotMesh,
       sendTimeoutMsForRouteChannel: sendTimeoutMsForSpotRouteChannel,
@@ -387,6 +423,7 @@ export class ZLinkFrameworkRuntimeHost
       rewriteAuthorityPayloadForOwner
     });
     this.streamBindingRuntime = new ZLinkStreamBindingRuntime({
+      dispatchErrors: this.createDispatchErrorReporter(this.runtimeOrPreStartErrorSink),
       streamPayloadCodec: resolveStreamPayloadCodec(options.registration),
       streamCompression: options.registration.streamCompression,
       messageSerializers: options.registration.messageSerializers,
@@ -698,7 +735,6 @@ export class ZLinkFrameworkRuntimeHost
       actorTransferRegistry: this.actorTransferRegistry,
       authorityStore: () => this.locationOwner.currentStores?.locationStore,
       currentOwner: () => this.locationOwner.currentRuntime?.currentOwnerToken,
-      relocationStore: () => this.options.registration.locations.relocationStoreInstance,
       liveDescriptors: (meshName, signal) => {
         const runtime = this.locationOwner.currentRuntime;
         if (runtime === undefined) {
@@ -766,6 +802,7 @@ export class ZLinkFrameworkRuntimeHost
     });
     this.serviceRelocation = new ZLinkHostServiceRelocationRuntime({
       registration: options.registration,
+      targetAdmissionSealed: () => this.shutdownOperationStartedAt !== undefined,
       providerResolver: options.providerResolver,
       locationStore: () => this.locationOwner.currentStores?.locationStore,
       currentOwner: () => this.locationOwner.currentRuntime?.currentOwnerToken,
@@ -795,7 +832,8 @@ export class ZLinkFrameworkRuntimeHost
       reconcileStatefulAuthorityRoutes: (signal) =>
         this.statefulAuthorityRoutes?.reconcile(signal) ?? Promise.resolve(),
       runtimeEventPublisher: this.runtimeEventPublisher,
-      metrics: this.metrics
+      metrics: this.metrics,
+      activationAdmission: this.activationAdmission
     });
     this.actorTransferAuthorityRuntime = new ZLinkActorTransferAuthorityRuntime({
       store: () => this.locationOwner.actorTransferStore() as never,
@@ -828,6 +866,7 @@ export class ZLinkFrameworkRuntimeHost
       },
       hostState: () => this.runtimeState,
       admission: this.admission,
+      activationAdmission: this.activationAdmission,
       publishRetiring: (meshName, signal) => this.publishMeshRetiring(meshName, signal),
       rollbackRetiring: (meshName, signal) => this.publishMeshServing(meshName, signal),
       publishDraining: (meshName, signal) => this.publishMeshDraining(meshName, signal),
@@ -850,7 +889,7 @@ export class ZLinkFrameworkRuntimeHost
   }
 
   get isStarted(): boolean {
-    return this.executionState !== undefined;
+    return this.executionState !== undefined && !this.executionState.abortController.signal.aborted;
   }
 
   get status(): ZLinkFrameworkRuntimeStatus {
@@ -868,10 +907,24 @@ export class ZLinkFrameworkRuntimeHost
     };
   }
 
+  // Bound listener records of the current runtime generation.
+  private listenerRecords = new ZLinkListenerRecords();
+
+  getListenerStatus(kind: ZLinkListenerKind, name: string): ZLinkListenerStatus {
+    const endpoint = this.listenerRecords.endpoint(kind, name);
+    if (endpoint === undefined || endpoint.length === 0) {
+      throw new ZLinkConfigurationException(
+        `Listener '${kind}:${name}' is not configured or has not bound.`
+      );
+    }
+    return { kind, name, endpoint, observedAt: new Date() };
+  }
+
   observe(signal?: AbortSignal): AsyncIterable<ZLinkObservedStatus<ZLinkFrameworkRuntimeStatus>> {
     const queue = new RuntimeEventQueue<ZLinkFrameworkRuntimeStatus>(undefined, signal);
     this.runtimeObservers.add(queue);
     queue.onClose(() => this.runtimeObservers.delete(queue));
+    this.publishRuntimeStatus(queue, this.status);
     return queue;
   }
 
@@ -888,7 +941,7 @@ export class ZLinkFrameworkRuntimeHost
       options,
       this.options.registration.applicationVersion
     );
-    const deadlineMs = options.deadlineMs ?? 30_000;
+    const deadlineMs = options.deadlineMs ?? DEFAULT_HOST_CONTROL_TIMEOUT_MS;
     if (!Number.isFinite(deadlineMs) || deadlineMs <= 0) {
       return Promise.reject(new RangeError('Relocation deadlineMs must be greater than zero.'));
     }
@@ -957,7 +1010,7 @@ export class ZLinkFrameworkRuntimeHost
   }
 
   shutdown(options?: ZLinkFrameworkLifecycleOptions): Promise<ZLinkFrameworkTerminationResult> {
-    const deadlineMs = options?.deadlineMs ?? 30_000;
+    const deadlineMs = options?.deadlineMs ?? DEFAULT_HOST_CONTROL_TIMEOUT_MS;
     if (!Number.isFinite(deadlineMs) || deadlineMs <= 0) {
       return Promise.reject(new RangeError('Shutdown deadlineMs must be greater than zero.'));
     }
@@ -1021,6 +1074,17 @@ export class ZLinkFrameworkRuntimeHost
         stopStartingSignal
       );
       if (drained.kind === 'forceStopped') {
+        // Settled authority on both sides or an expired source owner lease
+        // ends the host in Error (spec 30 §13).
+        if (drained.error instanceof ServiceRelocationAuthorityError) {
+          return this.completeRelocation(
+            blockedRelocation(
+              mode,
+              effectiveTargetApplicationVersion,
+              ZLinkFrameworkRelocationReason.RelocationFailed
+            )
+          );
+        }
         return this.resetBlockedRelocation(
           blockedRelocation(
             mode,
@@ -1059,52 +1123,78 @@ export class ZLinkFrameworkRuntimeHost
   }
 
   private async runShutdown(deadlineMs: number): Promise<ZLinkFrameworkTerminationResult> {
-    const deadlineAtMs = performance.now() + deadlineMs;
-    try {
-      this.admission.close();
-      this.setRuntimeState(ZLinkFrameworkRuntimeState.Draining);
-      if (
-        this.relocationOperation !== undefined &&
-        this.relocationStopStarting?.signal.aborted === true
-      ) {
-        await this.relocationOperation;
-      }
-      // Seal application admission before the public status reports that the
-      // host no longer accepts work. This prevents status polling from racing
-      // with the coordinator's synchronous seal step.
-      const unscopedStreamDrain = this.streamRuntime?.notifyUnscopedServerDrain();
-      const shutdown = this.routeMeshCoordinator.shutdownHost(
-        this.runtimeDeadline!,
-        undefined,
-        Math.max(0, deadlineAtMs - performance.now())
-      );
-      const drain = await shutdown;
-      await unscopedStreamDrain;
-      await this.stop();
-      if (drain.kind === 'forceStopped') {
+    return await this.runWithShutdownDeadline(deadlineMs, async () => {
+      const deadlineAtMs = performance.now() + deadlineMs;
+      try {
+        this.admission.close();
+        if (this.runtimeState === ZLinkFrameworkRuntimeState.Preparing) {
+          this.executionState?.abortController.abort();
+        }
+        this.setRuntimeState(ZLinkFrameworkRuntimeState.Draining);
+        if (
+          this.relocationOperation !== undefined &&
+          this.relocationStopStarting?.signal.aborted === true
+        ) {
+          await this.relocationOperation;
+        }
+        // Seal application admission before the public status reports that the
+        // host no longer accepts work. This prevents status polling from racing
+        // with the coordinator's synchronous seal step.
+        const unscopedStreamDrain = this.streamRuntime?.notifyUnscopedServerDrain();
+        const shutdown = this.routeMeshCoordinator.shutdownHost(
+          this.runtimeDeadline!,
+          this.shutdownDeadline!.signal
+        );
+        const drain = await shutdown;
+        await unscopedStreamDrain;
+        await this.stop();
+        if (drain.kind === 'forceStopped') {
+          return this.completeTermination({
+            outcome: ZLinkFrameworkTerminationOutcome.ForceStopped,
+            reason:
+              drain.reason === 'deadline_exceeded'
+                ? ZLinkFrameworkTerminationReason.DeadlineExceeded
+                : ZLinkFrameworkTerminationReason.TeardownFailed
+          });
+        }
+        if (performance.now() >= deadlineAtMs) {
+          throw createDeadlineExceededError('Shutdown resource cleanup exceeded its deadline.');
+        }
+        return this.completeTermination({
+          outcome: ZLinkFrameworkTerminationOutcome.Stopped,
+          reason: ZLinkFrameworkTerminationReason.None
+        });
+      } catch (error) {
+        await this.stop().catch(() => undefined);
         return this.completeTermination({
           outcome: ZLinkFrameworkTerminationOutcome.ForceStopped,
-          reason:
-            drain.reason === 'deadline_exceeded'
-              ? ZLinkFrameworkTerminationReason.DeadlineExceeded
-              : ZLinkFrameworkTerminationReason.TeardownFailed
+          reason: isDeadlineExceededError(error)
+            ? ZLinkFrameworkTerminationReason.DeadlineExceeded
+            : ZLinkFrameworkTerminationReason.TeardownFailed
         });
       }
-      if (performance.now() >= deadlineAtMs) {
-        throw createDeadlineExceededError('Shutdown resource cleanup exceeded its deadline.');
-      }
-      return this.completeTermination({
-        outcome: ZLinkFrameworkTerminationOutcome.Stopped,
-        reason: ZLinkFrameworkTerminationReason.None
-      });
-    } catch (error) {
-      await this.stop().catch(() => undefined);
-      return this.completeTermination({
-        outcome: ZLinkFrameworkTerminationOutcome.ForceStopped,
-        reason: isDeadlineExceededError(error)
-          ? ZLinkFrameworkTerminationReason.DeadlineExceeded
-          : ZLinkFrameworkTerminationReason.TeardownFailed
-      });
+    });
+  }
+
+  private async runWithShutdownDeadline<T>(
+    deadlineMs: number,
+    operation: () => Promise<T>
+  ): Promise<T> {
+    if (this.shutdownDeadline !== undefined) return await operation();
+    const deadline = new AbortController();
+    this.shutdownDeadline = deadline;
+    const ownsRuntimeDeadline = this.runtimeDeadline === undefined;
+    this.runtimeDeadline ??= new Date(Date.now() + deadlineMs);
+    const timer = setTimeout(
+      () => deadline.abort(createDeadlineExceededError('Shutdown deadline exceeded.')),
+      Math.max(0, this.runtimeDeadline.getTime() - Date.now())
+    );
+    try {
+      return await operation();
+    } finally {
+      clearTimeout(timer);
+      this.shutdownDeadline = undefined;
+      if (ownsRuntimeDeadline) this.runtimeDeadline = undefined;
     }
   }
 
@@ -1141,8 +1231,9 @@ export class ZLinkFrameworkRuntimeHost
   private recordRelocationResult(result: ZLinkFrameworkRelocationResult): void {
     if (this.relocationOperationStartedAt !== undefined) {
       this.metrics.duration(
-        'zlink.host.relocation.duration',
-        Math.max(0, performance.now() - this.relocationOperationStartedAt) / 1000,
+        METRIC_NAMES.HostRelocationDuration,
+        Math.max(0, performance.now() - this.relocationOperationStartedAt) /
+          MILLISECONDS_PER_SECOND,
         {
           mode: relocationModeMetricName(result.mode),
           outcome:
@@ -1151,7 +1242,7 @@ export class ZLinkFrameworkRuntimeHost
       );
     }
     if (result.outcome === ZLinkFrameworkRelocationOutcome.Blocked) {
-      this.metrics.count('zlink.host.relocation.blocked', 1, {
+      this.metrics.count(METRIC_NAMES.HostRelocationBlocked, 1, {
         mode: relocationModeMetricName(result.mode),
         reason: relocationReasonMetricName(result.reason)
       });
@@ -1174,8 +1265,8 @@ export class ZLinkFrameworkRuntimeHost
     this.runtimeDeadline = undefined;
     if (this.shutdownOperationStartedAt !== undefined) {
       this.metrics.duration(
-        'zlink.host.shutdown.duration',
-        Math.max(0, performance.now() - this.shutdownOperationStartedAt) / 1000,
+        METRIC_NAMES.HostShutdownDuration,
+        Math.max(0, performance.now() - this.shutdownOperationStartedAt) / MILLISECONDS_PER_SECOND,
         {
           outcome:
             result.outcome === ZLinkFrameworkTerminationOutcome.Stopped
@@ -1185,7 +1276,7 @@ export class ZLinkFrameworkRuntimeHost
       );
     }
     if (result.outcome === ZLinkFrameworkTerminationOutcome.ForceStopped) {
-      this.metrics.count('zlink.host.shutdown.forced', 1, {
+      this.metrics.count(METRIC_NAMES.HostShutdownForced, 1, {
         reason: terminationReasonMetricName(result.reason)
       });
     }
@@ -1207,14 +1298,18 @@ export class ZLinkFrameworkRuntimeHost
     this.notifyTopologyHostStateChanged();
     const status = this.status;
     for (const observer of this.runtimeObservers) {
-      if (state === ZLinkFrameworkRuntimeState.Stopped) {
-        observer.seal(status, this.runtimeObservationSource);
-      } else {
-        observer.push(status, this.runtimeObservationSource);
-      }
+      this.publishRuntimeStatus(observer, status);
     }
-    if (state === ZLinkFrameworkRuntimeState.Stopped) {
-      this.runtimeObservers.clear();
+  }
+
+  private publishRuntimeStatus(
+    observer: RuntimeEventQueue<ZLinkFrameworkRuntimeStatus>,
+    status: ZLinkFrameworkRuntimeStatus
+  ): void {
+    if (runtimeObservationIsTerminal(status.state)) {
+      observer.complete(status, this.runtimeObservationSource);
+    } else {
+      observer.push(status, this.runtimeObservationSource);
     }
   }
 
@@ -1322,6 +1417,8 @@ export class ZLinkFrameworkRuntimeHost
 
     this.setRuntimeState(ZLinkFrameworkRuntimeState.Preparing);
     this.lifecycleSink?.push('framework:start');
+    const listenerRecords = new ZLinkListenerRecords();
+    this.listenerRecords = listenerRecords;
     const channelAdapter = this.backendAdapterFactory.createChannelAdapter();
     const context = channelAdapter.createContext();
     const coreHwm = this.options.registration.coreHwm;
@@ -1344,13 +1441,14 @@ export class ZLinkFrameworkRuntimeHost
         channelAdapter,
         context,
         this.options.providerResolver,
-        this.createChannelRuntimeOptions()
+        { ...this.createChannelRuntimeOptions(), listenerRecords }
       );
       this.channelRuntime = channelRuntime;
       channelRuntime.prepareMeshDispatch(this.executionState.taskRunner);
-      spotNodeRuntime = new ZLinkSpotNodeRuntimeManager(
-        this.createSpotNodeRuntimeOptions(context, dispatchErrors)
-      );
+      spotNodeRuntime = new ZLinkSpotNodeRuntimeManager({
+        ...this.createSpotNodeRuntimeOptions(context, dispatchErrors),
+        listenerRecords
+      });
       await spotNodeRuntime.start();
       this.spotNodeRuntime = spotNodeRuntime;
       this.registerUserSpotHandlers?.(spotNodeRuntime);
@@ -1363,7 +1461,9 @@ export class ZLinkFrameworkRuntimeHost
       const locationRuntime = await this.locationOwner.startForRuntime(
         this.meshRouters.primaryMeshName(),
         spotNodeRuntime,
-        channelRuntime
+        channelRuntime,
+        this.executionState.abortController.signal,
+        () => this.shutdownDeadline!.signal
       );
       startedLocationRuntime = locationRuntime;
       if (locationRuntime !== undefined) {
@@ -1386,6 +1486,7 @@ export class ZLinkFrameworkRuntimeHost
         }
       }
       streamRuntime = new ZLinkStreamRuntimeManager({
+        listenerRecords,
         registration: this.options.registration,
         backendAdapterFactory: this.backendAdapterFactory,
         context,
@@ -1414,15 +1515,27 @@ export class ZLinkFrameworkRuntimeHost
       this.setRuntimeState(ZLinkFrameworkRuntimeState.Serving);
       this.lifecycleSink?.push('framework:started');
     } catch (error) {
+      if (this.shutdownDeadline !== undefined && isAbortError(error)) throw error;
       await statefulAuthorityRoutes?.stop();
-      await rollbackRuntimeStart({
-        context,
-        startedLocationRuntime,
-        streamRuntime,
-        spotNodeRuntime,
-        channelRuntime,
-        ownedStores: registeredRuntimeStores(this.options.registration)
-      });
+      try {
+        await this.runWithShutdownDeadline(DEFAULT_HOST_CONTROL_TIMEOUT_MS, async () =>
+          rollbackRuntimeStart({
+            context,
+            startedLocationRuntime: startedLocationRuntime ?? this.locationOwner.currentRuntime,
+            shutdownSignal: this.shutdownDeadline!.signal,
+            streamRuntime,
+            spotNodeRuntime,
+            channelRuntime,
+            ownedStores: registeredRuntimeStores(this.options.registration)
+          })
+        );
+      } catch (cleanupError) {
+        this.runtimeOrPreStartErrorSink.reportRuntimeTaskException(
+          'framework start rollback',
+          cleanupError
+        );
+        throw new AggregateError([error, cleanupError], 'Framework start and rollback failed.');
+      }
       this.executionState = undefined;
       this.channelRuntime = undefined;
       this.spotNodeRuntime = undefined;
@@ -1436,7 +1549,9 @@ export class ZLinkFrameworkRuntimeHost
   }
 
   async stop(): Promise<void> {
-    await this.runLifecycle(() => this.stopCore());
+    await this.runWithShutdownDeadline(DEFAULT_HOST_CONTROL_TIMEOUT_MS, async () =>
+      this.runLifecycle(() => this.stopCore())
+    );
   }
 
   private runtimeMetricMeshSnapshots(): readonly ZLinkRuntimeMetricMeshSnapshot[] {
@@ -1477,7 +1592,7 @@ export class ZLinkFrameworkRuntimeHost
           reserved: capacity.reserved,
           limit: capacity.limit
         })),
-        activation: descriptor?.activationConcurrency ?? { active: 0, limit: 0 },
+        activation: this.activationAdmission.current(meshName),
         instanceSpots: []
       });
     }
@@ -1489,6 +1604,7 @@ export class ZLinkFrameworkRuntimeHost
     if (state === undefined) {
       return;
     }
+    state.abortController.abort();
 
     const channelRuntime = this.channelRuntime;
     const spotNodeRuntime = this.spotNodeRuntime;
@@ -1497,24 +1613,26 @@ export class ZLinkFrameworkRuntimeHost
     this.stopTopologyObservers();
     this.removeOwnerLeaseRecoveryPublication();
     const locationSnapshot = this.locationOwner.clearForStop();
-    this.executionState = undefined;
-    this.channelRuntime = undefined;
-    this.spotNodeRuntime = undefined;
-    this.streamRuntime = undefined;
-    this.statefulAuthorityRoutes = undefined;
-    this.cachedLocationSpotRouteResolver = undefined;
     this.lifecycleSink?.push('framework:stop');
     await statefulAuthorityRoutes?.stop();
     await stopRuntimeParts({
       state,
       locationSnapshot,
       cleanupDeadline: this.runtimeDeadline,
+      shutdownSignal: this.shutdownDeadline!.signal,
       streamRuntime,
       spotNodeRuntime,
       channelRuntime,
       serviceRelocation: this.serviceRelocation,
       ownedStores: registeredRuntimeStores(this.options.registration)
     });
+    this.executionState = undefined;
+    this.channelRuntime = undefined;
+    this.spotNodeRuntime = undefined;
+    this.streamRuntime = undefined;
+    this.statefulAuthorityRoutes = undefined;
+    this.cachedLocationSpotRouteResolver = undefined;
+    this.listenerRecords.clear();
     this.lifecycleSink?.push('framework:stopped');
     if (this.shutdownOperation === undefined) {
       this.setRuntimeState(ZLinkFrameworkRuntimeState.Stopped);
@@ -1579,7 +1697,7 @@ export class ZLinkFrameworkRuntimeHost
       pollingIntervalMs:
         this.options.registration.locations.options.pollingIntervalMs ??
         zlinkDefaultLocationOptions.pollingIntervalMs,
-      pageSize: 1000,
+      pageSize: ZLINK_PROVIDER_MAX_PAGE_SIZE,
       reportError: (error) =>
         this.runtimeOrPreStartErrorSink.reportRuntimeTaskException(
           'stateful authority route reconciliation',
@@ -1834,6 +1952,7 @@ export class ZLinkFrameworkRuntimeHost
     } catch (error) {
       throw new ZLinkDrainingStatePublishError(error);
     }
+    await awaitWithDrainSignal(this.serviceRelocation.drainTargetAttempts(), signal);
   }
 
   private async cleanupOwnerForDrain(signal: AbortSignal): Promise<void> {
@@ -1870,7 +1989,6 @@ export class ZLinkFrameworkRuntimeHost
         ),
       isMaterializing: (target) =>
         spotManager.isInstanceMaterializing(meshName, target.targetSpotId as never),
-      isClosing: (target) => spotManager.isInstanceClosing(meshName, target.targetSpotId as never),
       isIdleEvicting: (target) =>
         spotManager.isInstanceSpotIdleEvicting(meshName, target.targetSpotId as never),
       beginIdleEviction: (target) =>
@@ -1926,11 +2044,13 @@ export class ZLinkFrameworkRuntimeHost
     | 'boundSessionFactory'
     | 'shutdownSignal'
     | 'admission'
+    | 'activationAdmission'
     | 'placementCreate'
   > {
     this.ensureLocationRuntime();
     return {
       ...this.actorRuntimeOptionsFactory().createActorManagerOptions(),
+      activationAdmission: this.activationAdmission,
       placementCreate: async (actorId, actorType, createOnly, call, signal) => {
         const placement = this.actorPlacement;
         if (placement === undefined) {
@@ -1992,12 +2112,31 @@ export class ZLinkFrameworkRuntimeHost
             ?.meshNode(meshName)
             ?.completeClosedInstance?.(spotId, objectGeneration);
         },
-        beginInstanceIdleClosingAuthority: (meshName, spotId) =>
-          this.locationOwner.currentLifecycle?.beginInstanceSpotClosing(meshName, spotId) ??
-          Promise.resolve(undefined),
-        beginInstanceClosingAuthority: (meshName, spotId) =>
-          this.locationOwner.currentLifecycle?.beginInstanceSpotClosing(meshName, spotId) ??
-          Promise.resolve(undefined),
+        beginInstanceIdleClosingAuthority: (meshName, spotId, onCommitted) =>
+          this.locationOwner.currentLifecycle?.beginInstanceSpotClosing(
+            meshName,
+            spotId,
+            onCommitted
+          ) ?? Promise.resolve(undefined),
+        beginInstanceClosingAuthority: (meshName, spotId, onCommitted) =>
+          this.locationOwner.currentLifecycle?.beginInstanceSpotClosing(
+            meshName,
+            spotId,
+            onCommitted
+          ) ?? Promise.resolve(undefined),
+        beginUserClosingAuthority: async (meshName, spotId, objectGeneration, onCommitted) => {
+          const coordinator = this.userSpotCoordinator;
+          const nodeRid = this.spotNodeRuntime?.meshNode(meshName)?.status().routingId;
+          if (coordinator === undefined || nodeRid === undefined) {
+            throw new ZLinkConfigurationException(
+              'User Spot context Close requires its authority coordinator.'
+            );
+          }
+          return await coordinator.beginOwnerClose(
+            { spotId: spotId as never, objectGeneration, meshName, nodeRid },
+            onCommitted
+          );
+        },
         createLocationSpotRouteResolver: () => this.createLocationSpotRouteResolver(),
         boundSessionRelay: this.boundSessionRelay,
         actorHandoff: this.actorHandoff,
@@ -2006,19 +2145,12 @@ export class ZLinkFrameworkRuntimeHost
         detachedTaskRunner: this.detachedTaskRunner(),
         metrics: this.metrics,
         admission: this.admission,
-        statefulExecutionAllowed: () => this.ownerAdmissionOpen()
+        statefulExecution: {
+          admissionOpen: () => this.ownerAdmissionOpen(),
+          hostState: () => this.runtimeState
+        }
       }).create(this.actorTransferRuntime),
-      activationConcurrencyLimitProvider: (meshName: string) =>
-        this.options.registration.spotNodes.get(meshName)?.activationConcurrencyLimit ?? 128,
-      onInstanceActivationConcurrencyChanged: (meshName: string) => {
-        void this.spotNodeRuntime
-          ?.publishMeshNodeState(
-            this.runtimeState,
-            this.executionState?.abortController.signal,
-            meshName
-          )
-          .catch(() => undefined);
-      }
+      activationAdmission: this.activationAdmission
     };
   }
 
@@ -2077,7 +2209,7 @@ export class ZLinkFrameworkRuntimeHost
           );
         }
         const meshNode = this.spotNodeRuntime?.meshNode(meshName);
-        const descriptors = (await locationStore.listMeshNodes(meshName, undefined, signal)).items;
+        const descriptors = await listAllMeshNodeDescriptors(locationStore, meshName, signal);
         const selected = await selectReadyLocationPlacementDescriptor(
           descriptors,
           meshNode,
@@ -2123,7 +2255,7 @@ export class ZLinkFrameworkRuntimeHost
         };
       }
     });
-    const coordinator = new ZLinkUserSpotCreationCoordinator({
+    const coordinator = (this.userSpotCoordinator = new ZLinkUserSpotCreationCoordinator({
       store: locationStore,
       publishReadyRoute: (meshName, route) => {
         this.cachedLocationSpotRouteResolver?.invalidate?.(route.spot.spotId);
@@ -2187,7 +2319,7 @@ export class ZLinkFrameworkRuntimeHost
           );
         }
         const meshNode = this.spotNodeRuntime?.meshNode(meshName);
-        const descriptors = (await locationStore.listMeshNodes(meshName, undefined, signal)).items;
+        const descriptors = await listAllMeshNodeDescriptors(locationStore, meshName, signal);
         const selected = await selectReadyLocationPlacementDescriptor(
           descriptors,
           meshNode,
@@ -2231,7 +2363,7 @@ export class ZLinkFrameworkRuntimeHost
             localStatus.lifecycleGeneration === selected.lifecycleGeneration
         };
       }
-    });
+    }));
     const factories = new Map(
       [...this.options.registration.spotNodes].map(([meshName, node]) => [
         meshName,
@@ -2298,7 +2430,7 @@ export class ZLinkFrameworkRuntimeHost
               try {
                 payload = {
                   packetName: 'ZLinkFrameworkUserSpotReply',
-                  contentType: 'application/octet-stream',
+                  contentType: ZlinkStreamContentType.Raw,
                   payload: Buffer.from(message.data())
                 };
               } finally {
@@ -2428,23 +2560,6 @@ export class ZLinkFrameworkRuntimeHost
             );
           },
           close: async (record, signal) => {
-            if (!local.hasActiveSpot(record.target.spotId as never)) {
-              throw createInternalFrameworkException(
-                ZLinkFrameworkInternalErrorKind.SpotMoving,
-                `User Spot '${record.target.spotId}' is not materialized on its authority owner.`,
-                true
-              );
-            }
-            if (!local.canCloseUserSpot(meshName, record.target.spotId as never)) {
-              return {
-                terminalResult: RequestResult.Ok,
-                failureCode: 0,
-                tail: {
-                  kind: 'userSpotClose' as const,
-                  closed: false
-                }
-              };
-            }
             return {
               terminalResult: RequestResult.Ok,
               failureCode: 0,
@@ -2452,8 +2567,15 @@ export class ZLinkFrameworkRuntimeHost
                 kind: 'userSpotClose' as const,
                 closed: await coordinator.handleRemoteClose(
                   record,
-                  (spot, closeSignal) => local.close(spot.meshName, spot.spotId, closeSignal),
-                  signal
+                  (spot, beginAuthority, closeSignal) =>
+                    local.closeUserWithAuthority(
+                      spot.meshName,
+                      spot.spotId,
+                      beginAuthority,
+                      closeSignal
+                    ),
+                  signal,
+                  (spot) => local.isSpotClosing(spot.meshName, spot.spotId)
                 )
               }
             };
@@ -2487,7 +2609,7 @@ export class ZLinkFrameworkRuntimeHost
         }
         return node.requestUserSpotClose(targetNodeRid, request, timeoutMs);
       },
-      defaultTimeoutMs: this.options.registration.requestTimeoutMs ?? 30_000,
+      defaultTimeoutMs: this.options.registration.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS,
       messageSerializers: this.options.registration.messageSerializers
     });
   }
@@ -2509,6 +2631,10 @@ export class ZLinkFrameworkRuntimeHost
       notifyEntrySpotActorCreated: (nodeRid, actor, createRequest, signal) =>
         this.spotNodeRuntime?.notifyEntrySpotActorCreated(nodeRid, actor, createRequest, signal) ??
         Promise.resolve(undefined),
+      notifyEntrySpotActorJoined: (actor, signal) =>
+        this.spotNodeRuntime?.notifyPrimaryEntrySpotActorJoined(actor, signal) ?? Promise.resolve(),
+      notifyEntrySpotActorLeft: (actor, signal) =>
+        this.spotNodeRuntime?.notifyPrimaryEntrySpotActorLeft(actor, signal) ?? Promise.resolve(),
       locationLifecycle: () => this.locationOwner.currentLifecycle,
       primaryMeshName: () => this.meshRouters.primaryMeshName(),
       actorMeshName: (actorType) => this.meshRouters.actorMeshName(actorType),
@@ -2626,6 +2752,7 @@ export class ZLinkFrameworkRuntimeHost
     }).create();
     return {
       ...options,
+      errorSink: this.runtimeOrPreStartErrorSink,
       peerAdmissionSealed: (meshName: string) => !this.admission.accepts(meshName),
       messageFollowReceiver: (record: ServiceMessageFollowRecord) => {
         if (record.source.kind === 'actor') {
@@ -2655,15 +2782,7 @@ export class ZLinkFrameworkRuntimeHost
           record,
           this.executionState?.abortController.signal
         ),
-      instanceActivationConcurrencyProvider: (meshName: string) => {
-        return (
-          this.spotManager?.instanceActivationConcurrency(meshName) ?? {
-            active: 0,
-            limit:
-              this.options.registration.spotNodes.get(meshName)?.activationConcurrencyLimit ?? 128
-          }
-        );
-      }
+      activationConcurrency: (meshName: string) => this.activationAdmission.current(meshName)
     };
   }
 
@@ -2688,14 +2807,8 @@ export class ZLinkFrameworkRuntimeHost
         );
       }
       const binding = record.kindData;
-      const replyBindingFailure = (
-        kind: ZLinkFrameworkInternalErrorKind,
-        message: string
-      ): void => {
-        const terminal =
-          kind === ZLinkFrameworkInternalErrorKind.ActorGenerationStale
-            ? { terminalResult: RequestResult.InvalidState, failureCode: 0 }
-            : internalFrameworkWireReply(createInternalFrameworkException(kind, message));
+      const replyBindingFailure = (kind: ZLinkFrameworkInternalErrorKind): void => {
+        const terminal = internalFrameworkWireReply(kind);
         if (
           record.replyFailure?.(terminal.terminalResult, terminal.failureCode) !== SubmitResult.Ok
         ) {
@@ -2719,17 +2832,11 @@ export class ZLinkFrameworkRuntimeHost
         nativeActorRef.actorId !== binding.actor.actorId ||
         !routingIdsEqual(nativeActorRef.nodeRid, binding.actor.nodeRid)
       ) {
-        replyBindingFailure(
-          ZLinkFrameworkInternalErrorKind.ActorLocationStale,
-          `Actor '${binding.actor.actorId}' binding target is no longer current.`
-        );
+        replyBindingFailure(ZLinkFrameworkInternalErrorKind.ActorLocationStale);
         return;
       }
       if (nativeActorRef.generation !== binding.actor.generation) {
-        replyBindingFailure(
-          ZLinkFrameworkInternalErrorKind.ActorGenerationStale,
-          `Actor '${binding.actor.actorId}' binding object generation is stale.`
-        );
+        replyBindingFailure(ZLinkFrameworkInternalErrorKind.ActorGenerationStale);
         return;
       }
       const node =
@@ -2744,10 +2851,7 @@ export class ZLinkFrameworkRuntimeHost
         state.locationGeneration !== binding.authorityOwnerGeneration ||
         state.isMoving
       ) {
-        replyBindingFailure(
-          ZLinkFrameworkInternalErrorKind.ActorLocationStale,
-          `Actor '${binding.actor.actorId}' binding authority fence is stale.`
-        );
+        replyBindingFailure(ZLinkFrameworkInternalErrorKind.ActorLocationStale);
         return;
       }
       const sessionOwnerIsLocal =
@@ -2761,6 +2865,7 @@ export class ZLinkFrameworkRuntimeHost
           nodeRid: binding.actor.nodeRid
         };
         if (sessionOwnerIsLocal) {
+          releaseApplicationJobPermitBeforeHandler();
           await this.streamBindingRuntime.retireRemoteBinding(
             actorRef,
             binding.sessionRid,
@@ -2777,10 +2882,7 @@ export class ZLinkFrameworkRuntimeHost
         binding.sessionRid
       );
       if (target === undefined) {
-        replyBindingFailure(
-          ZLinkFrameworkInternalErrorKind.ActorLocationStale,
-          `Actor '${binding.actor.actorId}' Session route is unavailable.`
-        );
+        replyBindingFailure(ZLinkFrameworkInternalErrorKind.ActorLocationStale);
         return;
       }
       const installed = state.installBoundSessionBinding({
@@ -2793,10 +2895,7 @@ export class ZLinkFrameworkRuntimeHost
         bindingGeneration: binding.bindingGeneration
       });
       if (!installed) {
-        replyBindingFailure(
-          ZLinkFrameworkInternalErrorKind.ActorLocationStale,
-          `Actor '${binding.actor.actorId}' Session binding generation is stale.`
-        );
+        replyBindingFailure(ZLinkFrameworkInternalErrorKind.ActorLocationStale);
         return;
       }
       replyBindingSuccess();
@@ -3017,7 +3116,7 @@ export class ZLinkFrameworkRuntimeHost
     return reporter;
   }
 
-  private detachedTaskRunner(): ZLinkDetachedTaskRunner {
+  detachedTaskRunner(): ZLinkDetachedTaskRunner {
     return {
       runDetached: (taskName, callback) => {
         const runner = this.executionState?.taskRunner;
@@ -3093,8 +3192,8 @@ export class ZLinkFrameworkRuntimeHost
             undefined,
             async (meshName, nodeRid, expectedLifecycleGeneration, signal) => {
               const descriptor = (
-                await authority.listMeshNodes(meshName, undefined, signal)
-              ).items.find(
+                await listAllMeshNodeDescriptors(authority, meshName, signal)
+              ).find(
                 (candidate) =>
                   routingIdsEqual(candidate.rid, nodeRid) &&
                   candidate.lifecycleGeneration === expectedLifecycleGeneration
@@ -3191,7 +3290,7 @@ export function hasExactPeerReadiness(
           peer.routingId !== null &&
           String(peer.routingId) === String(descriptor.rid) &&
           peer.lifecycleGeneration === descriptor.lifecycleGeneration &&
-          peer.state === 3
+          peer.state === MeshPeerRuntimeState.Serving
       )
     )
   );
@@ -3201,7 +3300,11 @@ export {
   ZLinkActorTransferAuthorityRuntime,
   transferIdString
 } from './actor-transfer-authority-runtime';
-export { ZLinkRetiringRollbackError, ZLinkRouteMeshRuntimeCoordinator } from './route-mesh-runtime';
+export {
+  ZLinkDrainingStatePublishError,
+  ZLinkRetiringRollbackError,
+  ZLinkRouteMeshRuntimeCoordinator
+} from './route-mesh-runtime';
 export { ZLinkRelocationStateIncompatibleError } from './service-relocation-host-runtime';
 export {
   ZLinkHostSpotAddressTransport,
@@ -3220,7 +3323,7 @@ async function waitForForcedSessionNotification(
   if (operation === undefined) return;
   let timeoutHandle: ReturnType<typeof setTimeout> | undefined;
   const timeout = new Promise<void>((resolve) => {
-    timeoutHandle = setTimeout(resolve, 100);
+    timeoutHandle = setTimeout(resolve, HOST_SHUTDOWN_POLL_INTERVAL_MS);
   });
   try {
     await Promise.race([operation.catch(() => undefined), timeout]);
@@ -3253,13 +3356,6 @@ async function awaitWithDrainSignal(
   });
 }
 
-class ZLinkDrainingStatePublishError extends Error {
-  constructor(cause: unknown) {
-    super('Failed to publish draining peer rows.', { cause });
-    this.name = 'ZLinkDrainingStatePublishError';
-  }
-}
-
 function resolveBackendAdapterFactory(internalOptions: unknown): ZLinkBackendAdapterFactory {
   if (
     typeof internalOptions === 'object' &&
@@ -3418,7 +3514,7 @@ function waitForPlacementReadiness(signal?: AbortSignal): Promise<void> {
     const timer = setTimeout(() => {
       signal?.removeEventListener('abort', abort);
       resolve();
-    }, 10);
+    }, HANDOFF_ACCEPTANCE_POLL_INTERVAL_MS);
     const abort = () => {
       clearTimeout(timer);
       signal?.removeEventListener('abort', abort);