@zlink-systems/framework 0.26.0 → 0.27.0

package/dist/runtime/host/index.js

@@ -1,68 +1,76 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ZLinkUserSpotCreationCoordinator = exports.ZLinkHostSpotAddressTransport = exports.ZLinkRelocationStateIncompatibleError = exports.ZLinkRouteMeshRuntimeCoordinator = exports.ZLinkRetiringRollbackError = exports.transferIdString = exports.ZLinkActorTransferAuthorityRuntime = exports.ZLinkFrameworkRuntimeHost = void 0;
+exports.ZLinkUserSpotCreationCoordinator = exports.ZLinkHostSpotAddressTransport = exports.ZLinkRelocationStateIncompatibleError = exports.ZLinkRouteMeshRuntimeCoordinator = exports.ZLinkRetiringRollbackError = exports.ZLinkDrainingStatePublishError = exports.transferIdString = exports.ZLinkActorTransferAuthorityRuntime = exports.ZLinkFrameworkRuntimeHost = void 0;
 exports.hasUnsupportedManualTopology = hasUnsupportedManualTopology;
 exports.hasExactPeerReadiness = hasExactPeerReadiness;
-const listener_records_1 = require("../foundation/listener-records");
-const framework_errors_internal_1 = require("../framework-errors-internal");
+const location_store_pages_1 = require("../locations/location-store-pages");
+const stream_wire_1 = require("@zlink-systems/stream-wire");
 const node_crypto_1 = require("node:crypto");
-const runtime_message_1 = require("../backend/runtime-message");
-const runtime_values_1 = require("../backend/runtime-values");
-const service_runtime_contracts_1 = require("../foundation/service-runtime-contracts");
-const runtime_state_projections_1 = require("../foundation/runtime-state-projections");
-const abort_1 = require("../abort");
-const backend_1 = require("../backend");
-const LEGACY_MESH_SEND_TIMEOUT_MS = 1000;
 const contracts_1 = require("../../contracts");
-const submission_result_1 = require("../messaging/submission-result");
-const ZLinkDispatchOptions_1 = require("../../contracts/Dispatch/ZLinkDispatchOptions");
+const ConfigurationException_1 = require("../../contracts/Configuration/ConfigurationException");
 const DiagnosticsValidation_1 = require("../../contracts/Configuration/DiagnosticsValidation");
-const channels_1 = require("../channels");
-const channel_envelope_1 = require("../channels/channel-envelope");
+const InternalDefaults_1 = require("../../contracts/Configuration/InternalDefaults");
+const Registration_1 = require("../../contracts/Configuration/Registration");
+const ZLinkDispatchOptions_1 = require("../../contracts/Dispatch/ZLinkDispatchOptions");
+const Locations_1 = require("../../contracts/Locations");
+const Stores_1 = require("../../contracts/Locations/Stores");
+const abort_1 = require("../abort");
+const activation_admission_1 = require("../activation-admission");
 const actors_1 = require("../actors");
+const actor_message_follow_context_1 = require("../actors/actor-message-follow-context");
 const actor_transport_delivery_gate_1 = require("../actors/actor-transport-delivery-gate");
+const admission_1 = require("../admission");
 const application_job_queue_handler_start_gate_1 = require("../application-jobs/application-job-queue-handler-start-gate");
-const execution_1 = require("../execution");
+const application_job_queue_scope_1 = require("../application-jobs/application-job-queue-scope");
+const backend_1 = require("../backend");
+const runtime_message_1 = require("../backend/runtime-message");
+const runtime_values_1 = require("../backend/runtime-values");
+const channels_1 = require("../channels");
+const channel_envelope_1 = require("../channels/channel-envelope");
 const diagnostics_1 = require("../diagnostics");
-const topology_runtime_projections_1 = require("../diagnostics/topology-runtime-projections");
+const runtime_metrics_1 = require("../diagnostics/runtime-metrics");
 const runtime_observation_queue_1 = require("../diagnostics/runtime-observation-queue");
+const topology_runtime_projections_1 = require("../diagnostics/topology-runtime-projections");
+const execution_1 = require("../execution");
+const listener_records_1 = require("../foundation/listener-records");
+const runtime_state_projections_1 = require("../foundation/runtime-state-projections");
+const service_authority_payload_codec_1 = require("../foundation/service-authority-payload-codec");
+const service_relocation_coordinator_1 = require("../foundation/service-relocation-coordinator");
+const service_relocation_runtime_1 = require("../foundation/service-relocation-runtime");
+const service_runtime_contracts_1 = require("../foundation/service-runtime-contracts");
+const framework_errors_internal_1 = require("../framework-errors-internal");
+const locations_1 = require("../locations");
+const creation_payload_codec_1 = require("../messaging/creation-payload-codec");
+const payload_codec_1 = require("../messaging/payload-codec");
+const submission_result_1 = require("../messaging/submission-result");
+const routing_id_1 = require("../routing-id");
 const spots_1 = require("../spots");
-const actor_message_follow_context_1 = require("../actors/actor-message-follow-context");
-const actors_2 = require("../actors");
 const streams_1 = require("../streams");
-const locations_1 = require("../locations");
-const Locations_1 = require("../../contracts/Locations");
+const actor_placement_coordinator_1 = require("./actor-placement-coordinator");
 const actor_runtime_options_factory_1 = require("./actor-runtime-options-factory");
-const actor_transfer_runtime_1 = require("./actor-transfer-runtime");
 const actor_transfer_authority_runtime_1 = require("./actor-transfer-authority-runtime");
+const actor_transfer_runtime_1 = require("./actor-transfer-runtime");
+const application_job_queue_1 = require("./application-job-queue");
+const bound_session_relay_1 = require("./bound-session-relay");
+const channel_runtime_options_factory_1 = require("./channel-runtime-options-factory");
 const entry_actor_runtime_1 = require("./entry-actor-runtime");
+const host_capacity_status_1 = require("./host-capacity-status");
+const instance_activation_authority_1 = require("./instance-activation-authority");
 const location_runtime_owner_1 = require("./location-runtime-owner");
 const mesh_router_resolver_1 = require("./mesh-router-resolver");
-const bound_session_relay_1 = require("./bound-session-relay");
-const routing_id_1 = require("../routing-id");
-const service_authority_payload_codec_1 = require("../foundation/service-authority-payload-codec");
-const service_relocation_runtime_1 = require("../foundation/service-relocation-runtime");
-const spot_runtime_options_factory_1 = require("./spot-runtime-options-factory");
-const channel_runtime_options_factory_1 = require("./channel-runtime-options-factory");
-const spot_node_runtime_options_factory_1 = require("./spot-node-runtime-options-factory");
-const runtime_shutdown_1 = require("./runtime-shutdown");
-const admission_1 = require("../admission");
-const activation_admission_1 = require("../activation-admission");
 const route_mesh_runtime_1 = require("./route-mesh-runtime");
-const service_relocation_coordinator_1 = require("../foundation/service-relocation-coordinator");
-const ConfigurationException_1 = require("../../contracts/Configuration/ConfigurationException");
-const stateful_authority_route_runtime_1 = require("./stateful-authority-route-runtime");
-const instance_activation_authority_1 = require("./instance-activation-authority");
-const spot_address_transport_1 = require("./spot-address-transport");
-const user_spot_creation_coordinator_1 = require("./user-spot-creation-coordinator");
-const actor_placement_coordinator_1 = require("./actor-placement-coordinator");
-const payload_codec_1 = require("../messaging/payload-codec");
-const creation_payload_codec_1 = require("../messaging/creation-payload-codec");
 const route_mesh_runtime_options_1 = require("./route-mesh-runtime-options");
+const runtime_shutdown_1 = require("./runtime-shutdown");
 const service_relocation_host_runtime_1 = require("./service-relocation-host-runtime");
-const application_job_queue_1 = require("./application-job-queue");
-const application_job_queue_scope_1 = require("../application-jobs/application-job-queue-scope");
-const host_capacity_status_1 = require("./host-capacity-status");
+const spot_address_transport_1 = require("./spot-address-transport");
+const spot_node_runtime_options_factory_1 = require("./spot-node-runtime-options-factory");
+const spot_runtime_options_factory_1 = require("./spot-runtime-options-factory");
+const stateful_authority_route_runtime_1 = require("./stateful-authority-route-runtime");
+const user_spot_creation_coordinator_1 = require("./user-spot-creation-coordinator");
+const HOST_SHUTDOWN_POLL_INTERVAL_MS = 100;
+const HANDOFF_ACCEPTANCE_POLL_INTERVAL_MS = 10;
+const DEFAULT_HOST_CONTROL_TIMEOUT_MS = 30_000;
+const LEGACY_MESH_SEND_TIMEOUT_MS = 1000;
 class ZLinkFrameworkRuntimeHost {
     options;
     backendAdapterFactory;
@@ -72,6 +80,7 @@ class ZLinkFrameworkRuntimeHost {
     runtimeSequence = 0n;
     runtimeObservationSource = Symbol('zlink.framework-runtime');
     runtimeDeadline;
+    shutdownDeadline;
     runtimeRelocationResult;
     runtimeTerminationResult;
     relocationOperation;
@@ -109,11 +118,8 @@ class ZLinkFrameworkRuntimeHost {
     capacityStatus;
     metricRegistrations = [];
     admission = new admission_1.ZLinkRuntimeAdmissionGate(() => this.ownerAdmissionOpen());
-    activationAdmission = new activation_admission_1.ZLinkActivationAdmission((meshName) => this.options.registration.spotNodes.get(meshName)?.activationConcurrencyLimit ?? 128, (meshName) => {
-        void this.spotNodeRuntime
-            ?.publishMeshNodeState(this.runtimeState, this.executionState?.abortController.signal, meshName)
-            .catch(() => undefined);
-    });
+    activationAdmission = new activation_admission_1.ZLinkActivationAdmission((meshName) => this.options.registration.spotNodes.get(meshName)?.activationConcurrencyLimit ??
+        InternalDefaults_1.DEFAULT_ACTIVATION_CONCURRENCY_LIMIT);
     cachedLocationSpotRouteResolver;
     actorClientLocationResolver;
     // Shared, runtime-mutable message-flow mode cell — installed once so
@@ -172,7 +178,7 @@ class ZLinkFrameworkRuntimeHost {
             return sendTimeoutMsForSpotMesh(routeChannelId);
         };
         const defaultSpotSendTimeoutMs = Math.max(LEGACY_MESH_SEND_TIMEOUT_MS, ...[...options.registration.spotNodes.values()].map((node) => node.router?.sendTimeoutMs ?? LEGACY_MESH_SEND_TIMEOUT_MS), ...[...options.registration.routeChannelOptions.values()].map((route) => route.sendTimeoutMs ?? LEGACY_MESH_SEND_TIMEOUT_MS));
-        this.routeTransport = new channels_1.ZLinkRuntimeRouteTransport(() => this.channelRuntime, (routerChannelId) => this.meshRouters.canUseRouterChannel(routerChannelId), () => this.spotNodeRuntime, { serializers: options.registration.messageSerializers }, (meshName, targetNodeRid) => this.meshRouters.classifyManualNodeTarget(meshName, targetNodeRid), (meshName, sourceNodeRid, parts) => this.submitLocalMeshRoute(meshName, sourceNodeRid, parts), this.metrics, () => this.flowCreationEnabled());
+        this.routeTransport = new channels_1.ZLinkRuntimeRouteTransport(() => this.channelRuntime, (routerChannelId) => this.meshRouters.canUseRouterChannel(routerChannelId), () => this.spotNodeRuntime, { serializers: options.registration.messageSerializers }, (meshName, targetNodeRid) => this.meshRouters.classifyManualNodeTarget(meshName, targetNodeRid), (meshName, sourceNodeRid, parts) => this.submitLocalMeshRoute(meshName, sourceNodeRid, parts), this.metrics, () => this.flowCreationEnabled(), () => this.createDispatchErrorReporter(this.runtimeOrPreStartErrorSink).flow);
         this.spotAddressTransport = new spot_address_transport_1.ZLinkHostSpotAddressTransport({
             resolver: () => this.createLocationSpotRouteResolver(),
             routed: this.routeTransport,
@@ -183,7 +189,7 @@ class ZLinkFrameworkRuntimeHost {
             meshNode: (meshName) => this.spotNodeRuntime?.meshNode(meshName),
             completions: (meshName) => this.spotNodeRuntime?.meshCompletionTable(meshName),
             codecs: { serializers: options.registration.messageSerializers },
-            defaultRequestTimeoutMs: options.registration.requestTimeoutMs ?? 30_000,
+            defaultRequestTimeoutMs: options.registration.requestTimeoutMs ?? Registration_1.DEFAULT_REQUEST_TIMEOUT_MS,
             defaultSendTimeoutMs: defaultSpotSendTimeoutMs,
             sendTimeoutMsForMesh: sendTimeoutMsForSpotMesh,
             sendTimeoutMsForRouteChannel: sendTimeoutMsForSpotRouteChannel,
@@ -197,6 +203,7 @@ class ZLinkFrameworkRuntimeHost {
             rewriteAuthorityPayloadForOwner
         });
         this.streamBindingRuntime = new streams_1.ZLinkStreamBindingRuntime({
+            dispatchErrors: this.createDispatchErrorReporter(this.runtimeOrPreStartErrorSink),
             streamPayloadCodec: resolveStreamPayloadCodec(options.registration),
             streamCompression: options.registration.streamCompression,
             messageSerializers: options.registration.messageSerializers,
@@ -229,7 +236,7 @@ class ZLinkFrameworkRuntimeHost {
             relay: (actor, header, payload, signal) => this.boundSessionRelay.actorPackets.relayActorPacket(actor, header, payload, signal),
             notifyDisconnected: (actor, signal) => this.boundSessionRelay.actorPackets.notifyBoundActorDisconnected(actor, signal)
         });
-        this.actorHandoff = new actors_2.ZLinkActorHandoffCoordinator({
+        this.actorHandoff = new actors_1.ZLinkActorHandoffCoordinator({
             routedTransport: this.routeTransport,
             messageFollowDurationMs: options.registration.locations.options.messageFollowDurationMs,
             requestTimeoutMs: options.registration.requestTimeoutMs,
@@ -385,7 +392,7 @@ class ZLinkFrameworkRuntimeHost {
                 });
             }
         });
-        this.actorTransferRegistry = new actors_2.ZLinkActorTransferRegistry(options.registration.spotNodes, options.providerResolver, options.registration.messageSerializers);
+        this.actorTransferRegistry = new actors_1.ZLinkActorTransferRegistry(options.registration.spotNodes, options.providerResolver, options.registration.messageSerializers);
         this.boundSessionFactory = new streams_1.DefaultZLinkBoundSessionFactory(this.streamBindingRuntime);
         this.boundSessionRelay = new bound_session_relay_1.ZLinkBoundSessionRelay({
             requestTimeoutMs: options.registration.requestTimeoutMs,
@@ -514,6 +521,7 @@ class ZLinkFrameworkRuntimeHost {
         });
         this.serviceRelocation = new service_relocation_host_runtime_1.ZLinkHostServiceRelocationRuntime({
             registration: options.registration,
+            targetAdmissionSealed: () => this.shutdownOperationStartedAt !== undefined,
             providerResolver: options.providerResolver,
             locationStore: () => this.locationOwner.currentStores?.locationStore,
             currentOwner: () => this.locationOwner.currentRuntime?.currentOwnerToken,
@@ -584,7 +592,7 @@ class ZLinkFrameworkRuntimeHost {
         this.routeMeshRuntime = this.routeMeshCoordinator;
     }
     get isStarted() {
-        return this.executionState !== undefined;
+        return this.executionState !== undefined && !this.executionState.abortController.signal.aborted;
     }
     get status() {
         return {
@@ -613,6 +621,7 @@ class ZLinkFrameworkRuntimeHost {
         const queue = new runtime_observation_queue_1.RuntimeEventQueue(undefined, signal);
         this.runtimeObservers.add(queue);
         queue.onClose(() => this.runtimeObservers.delete(queue));
+        this.publishRuntimeStatus(queue, this.status);
         return queue;
     }
     resetCapacityMetrics() {
@@ -624,7 +633,7 @@ class ZLinkFrameworkRuntimeHost {
     }
     relocate(options) {
         const effectiveTargetApplicationVersion = validateRelocationOptions(options, this.options.registration.applicationVersion);
-        const deadlineMs = options.deadlineMs ?? 30_000;
+        const deadlineMs = options.deadlineMs ?? DEFAULT_HOST_CONTROL_TIMEOUT_MS;
         if (!Number.isFinite(deadlineMs) || deadlineMs <= 0) {
             return Promise.reject(new RangeError('Relocation deadlineMs must be greater than zero.'));
         }
@@ -686,7 +695,7 @@ class ZLinkFrameworkRuntimeHost {
         return waitForRuntimeOperation(this.relocationOperation, options.signal);
     }
     shutdown(options) {
-        const deadlineMs = options?.deadlineMs ?? 30_000;
+        const deadlineMs = options?.deadlineMs ?? DEFAULT_HOST_CONTROL_TIMEOUT_MS;
         if (!Number.isFinite(deadlineMs) || deadlineMs <= 0) {
             return Promise.reject(new RangeError('Shutdown deadlineMs must be greater than zero.'));
         }
@@ -756,46 +765,69 @@ class ZLinkFrameworkRuntimeHost {
         }
     }
     async runShutdown(deadlineMs) {
-        const deadlineAtMs = performance.now() + deadlineMs;
-        try {
-            this.admission.close();
-            this.setRuntimeState(contracts_1.ZLinkFrameworkRuntimeState.Draining);
-            if (this.relocationOperation !== undefined &&
-                this.relocationStopStarting?.signal.aborted === true) {
-                await this.relocationOperation;
+        return await this.runWithShutdownDeadline(deadlineMs, async () => {
+            const deadlineAtMs = performance.now() + deadlineMs;
+            try {
+                this.admission.close();
+                if (this.runtimeState === contracts_1.ZLinkFrameworkRuntimeState.Preparing) {
+                    this.executionState?.abortController.abort();
+                }
+                this.setRuntimeState(contracts_1.ZLinkFrameworkRuntimeState.Draining);
+                if (this.relocationOperation !== undefined &&
+                    this.relocationStopStarting?.signal.aborted === true) {
+                    await this.relocationOperation;
+                }
+                // Seal application admission before the public status reports that the
+                // host no longer accepts work. This prevents status polling from racing
+                // with the coordinator's synchronous seal step.
+                const unscopedStreamDrain = this.streamRuntime?.notifyUnscopedServerDrain();
+                const shutdown = this.routeMeshCoordinator.shutdownHost(this.runtimeDeadline, this.shutdownDeadline.signal);
+                const drain = await shutdown;
+                await unscopedStreamDrain;
+                await this.stop();
+                if (drain.kind === 'forceStopped') {
+                    return this.completeTermination({
+                        outcome: contracts_1.ZLinkFrameworkTerminationOutcome.ForceStopped,
+                        reason: drain.reason === 'deadline_exceeded'
+                            ? contracts_1.ZLinkFrameworkTerminationReason.DeadlineExceeded
+                            : contracts_1.ZLinkFrameworkTerminationReason.TeardownFailed
+                    });
+                }
+                if (performance.now() >= deadlineAtMs) {
+                    throw (0, abort_1.createDeadlineExceededError)('Shutdown resource cleanup exceeded its deadline.');
+                }
+                return this.completeTermination({
+                    outcome: contracts_1.ZLinkFrameworkTerminationOutcome.Stopped,
+                    reason: contracts_1.ZLinkFrameworkTerminationReason.None
+                });
             }
-            // Seal application admission before the public status reports that the
-            // host no longer accepts work. This prevents status polling from racing
-            // with the coordinator's synchronous seal step.
-            const unscopedStreamDrain = this.streamRuntime?.notifyUnscopedServerDrain();
-            const shutdown = this.routeMeshCoordinator.shutdownHost(this.runtimeDeadline, undefined, Math.max(0, deadlineAtMs - performance.now()));
-            const drain = await shutdown;
-            await unscopedStreamDrain;
-            await this.stop();
-            if (drain.kind === 'forceStopped') {
+            catch (error) {
+                await this.stop().catch(() => undefined);
                 return this.completeTermination({
                     outcome: contracts_1.ZLinkFrameworkTerminationOutcome.ForceStopped,
-                    reason: drain.reason === 'deadline_exceeded'
+                    reason: (0, abort_1.isDeadlineExceededError)(error)
                         ? contracts_1.ZLinkFrameworkTerminationReason.DeadlineExceeded
                         : contracts_1.ZLinkFrameworkTerminationReason.TeardownFailed
                 });
             }
-            if (performance.now() >= deadlineAtMs) {
-                throw (0, abort_1.createDeadlineExceededError)('Shutdown resource cleanup exceeded its deadline.');
-            }
-            return this.completeTermination({
-                outcome: contracts_1.ZLinkFrameworkTerminationOutcome.Stopped,
-                reason: contracts_1.ZLinkFrameworkTerminationReason.None
-            });
+        });
+    }
+    async runWithShutdownDeadline(deadlineMs, operation) {
+        if (this.shutdownDeadline !== undefined)
+            return await operation();
+        const deadline = new AbortController();
+        this.shutdownDeadline = deadline;
+        const ownsRuntimeDeadline = this.runtimeDeadline === undefined;
+        this.runtimeDeadline ??= new Date(Date.now() + deadlineMs);
+        const timer = setTimeout(() => deadline.abort((0, abort_1.createDeadlineExceededError)('Shutdown deadline exceeded.')), Math.max(0, this.runtimeDeadline.getTime() - Date.now()));
+        try {
+            return await operation();
         }
-        catch (error) {
-            await this.stop().catch(() => undefined);
-            return this.completeTermination({
-                outcome: contracts_1.ZLinkFrameworkTerminationOutcome.ForceStopped,
-                reason: (0, abort_1.isDeadlineExceededError)(error)
-                    ? contracts_1.ZLinkFrameworkTerminationReason.DeadlineExceeded
-                    : contracts_1.ZLinkFrameworkTerminationReason.TeardownFailed
-            });
+        finally {
+            clearTimeout(timer);
+            this.shutdownDeadline = undefined;
+            if (ownsRuntimeDeadline)
+                this.runtimeDeadline = undefined;
         }
     }
     resetBlockedRelocation(result) {
@@ -822,13 +854,14 @@ class ZLinkFrameworkRuntimeHost {
     }
     recordRelocationResult(result) {
         if (this.relocationOperationStartedAt !== undefined) {
-            this.metrics.duration('zlink.host.relocation.duration', Math.max(0, performance.now() - this.relocationOperationStartedAt) / 1000, {
+            this.metrics.duration(runtime_metrics_1.METRIC_NAMES.HostRelocationDuration, Math.max(0, performance.now() - this.relocationOperationStartedAt) /
+                runtime_metrics_1.MILLISECONDS_PER_SECOND, {
                 mode: relocationModeMetricName(result.mode),
                 outcome: result.outcome === contracts_1.ZLinkFrameworkRelocationOutcome.Relocated ? 'relocated' : 'blocked'
             });
         }
         if (result.outcome === contracts_1.ZLinkFrameworkRelocationOutcome.Blocked) {
-            this.metrics.count('zlink.host.relocation.blocked', 1, {
+            this.metrics.count(runtime_metrics_1.METRIC_NAMES.HostRelocationBlocked, 1, {
                 mode: relocationModeMetricName(result.mode),
                 reason: relocationReasonMetricName(result.reason)
             });
@@ -847,14 +880,14 @@ class ZLinkFrameworkRuntimeHost {
         this.runtimeTerminationResult = result;
         this.runtimeDeadline = undefined;
         if (this.shutdownOperationStartedAt !== undefined) {
-            this.metrics.duration('zlink.host.shutdown.duration', Math.max(0, performance.now() - this.shutdownOperationStartedAt) / 1000, {
+            this.metrics.duration(runtime_metrics_1.METRIC_NAMES.HostShutdownDuration, Math.max(0, performance.now() - this.shutdownOperationStartedAt) / runtime_metrics_1.MILLISECONDS_PER_SECOND, {
                 outcome: result.outcome === contracts_1.ZLinkFrameworkTerminationOutcome.Stopped
                     ? 'stopped'
                     : 'force_stopped'
             });
         }
         if (result.outcome === contracts_1.ZLinkFrameworkTerminationOutcome.ForceStopped) {
-            this.metrics.count('zlink.host.shutdown.forced', 1, {
+            this.metrics.count(runtime_metrics_1.METRIC_NAMES.HostShutdownForced, 1, {
                 reason: terminationReasonMetricName(result.reason)
             });
         }
@@ -876,15 +909,15 @@ class ZLinkFrameworkRuntimeHost {
         this.notifyTopologyHostStateChanged();
         const status = this.status;
         for (const observer of this.runtimeObservers) {
-            if (state === contracts_1.ZLinkFrameworkRuntimeState.Stopped) {
-                observer.seal(status, this.runtimeObservationSource);
-            }
-            else {
-                observer.push(status, this.runtimeObservationSource);
-            }
+            this.publishRuntimeStatus(observer, status);
+        }
+    }
+    publishRuntimeStatus(observer, status) {
+        if ((0, runtime_state_projections_1.runtimeObservationIsTerminal)(status.state)) {
+            observer.complete(status, this.runtimeObservationSource);
         }
-        if (state === contracts_1.ZLinkFrameworkRuntimeState.Stopped) {
-            this.runtimeObservers.clear();
+        else {
+            observer.push(status, this.runtimeObservationSource);
         }
     }
     notifyTopologyHostStateChanged() {
@@ -1004,7 +1037,7 @@ class ZLinkFrameworkRuntimeHost {
             // Start bound receivers before publishing Serving descriptors. A
             // discovered ClientServer endpoint must already be able to dispatch.
             this.executionState.listenerTasks.push(...channelRuntime.start(this.executionState.taskRunner));
-            const locationRuntime = await this.locationOwner.startForRuntime(this.meshRouters.primaryMeshName(), spotNodeRuntime, channelRuntime);
+            const locationRuntime = await this.locationOwner.startForRuntime(this.meshRouters.primaryMeshName(), spotNodeRuntime, channelRuntime, this.executionState.abortController.signal, () => this.shutdownDeadline.signal);
             startedLocationRuntime = locationRuntime;
             if (locationRuntime !== undefined) {
                 this.installOwnerLeaseRecoveryPublication(locationRuntime, spotNodeRuntime, channelRuntime);
@@ -1046,15 +1079,24 @@ class ZLinkFrameworkRuntimeHost {
             this.lifecycleSink?.push('framework:started');
         }
         catch (error) {
+            if (this.shutdownDeadline !== undefined && (0, abort_1.isAbortError)(error))
+                throw error;
             await statefulAuthorityRoutes?.stop();
-            await (0, runtime_shutdown_1.rollbackRuntimeStart)({
-                context,
-                startedLocationRuntime,
-                streamRuntime,
-                spotNodeRuntime,
-                channelRuntime,
-                ownedStores: registeredRuntimeStores(this.options.registration)
-            });
+            try {
+                await this.runWithShutdownDeadline(DEFAULT_HOST_CONTROL_TIMEOUT_MS, async () => (0, runtime_shutdown_1.rollbackRuntimeStart)({
+                    context,
+                    startedLocationRuntime: startedLocationRuntime ?? this.locationOwner.currentRuntime,
+                    shutdownSignal: this.shutdownDeadline.signal,
+                    streamRuntime,
+                    spotNodeRuntime,
+                    channelRuntime,
+                    ownedStores: registeredRuntimeStores(this.options.registration)
+                }));
+            }
+            catch (cleanupError) {
+                this.runtimeOrPreStartErrorSink.reportRuntimeTaskException('framework start rollback', cleanupError);
+                throw new AggregateError([error, cleanupError], 'Framework start and rollback failed.');
+            }
             this.executionState = undefined;
             this.channelRuntime = undefined;
             this.spotNodeRuntime = undefined;
@@ -1067,7 +1109,7 @@ class ZLinkFrameworkRuntimeHost {
         }
     }
     async stop() {
-        await this.runLifecycle(() => this.stopCore());
+        await this.runWithShutdownDeadline(DEFAULT_HOST_CONTROL_TIMEOUT_MS, async () => this.runLifecycle(() => this.stopCore()));
     }
     runtimeMetricMeshSnapshots() {
         const snapshots = [];
@@ -1117,6 +1159,7 @@ class ZLinkFrameworkRuntimeHost {
         if (state === undefined) {
             return;
         }
+        state.abortController.abort();
         const channelRuntime = this.channelRuntime;
         const spotNodeRuntime = this.spotNodeRuntime;
         const streamRuntime = this.streamRuntime;
@@ -1124,24 +1167,25 @@ class ZLinkFrameworkRuntimeHost {
         this.stopTopologyObservers();
         this.removeOwnerLeaseRecoveryPublication();
         const locationSnapshot = this.locationOwner.clearForStop();
-        this.executionState = undefined;
-        this.channelRuntime = undefined;
-        this.spotNodeRuntime = undefined;
-        this.streamRuntime = undefined;
-        this.statefulAuthorityRoutes = undefined;
-        this.cachedLocationSpotRouteResolver = undefined;
         this.lifecycleSink?.push('framework:stop');
         await statefulAuthorityRoutes?.stop();
         await (0, runtime_shutdown_1.stopRuntimeParts)({
             state,
             locationSnapshot,
             cleanupDeadline: this.runtimeDeadline,
+            shutdownSignal: this.shutdownDeadline.signal,
             streamRuntime,
             spotNodeRuntime,
             channelRuntime,
             serviceRelocation: this.serviceRelocation,
             ownedStores: registeredRuntimeStores(this.options.registration)
         });
+        this.executionState = undefined;
+        this.channelRuntime = undefined;
+        this.spotNodeRuntime = undefined;
+        this.streamRuntime = undefined;
+        this.statefulAuthorityRoutes = undefined;
+        this.cachedLocationSpotRouteResolver = undefined;
         this.listenerRecords.clear();
         this.lifecycleSink?.push('framework:stopped');
         if (this.shutdownOperation === undefined) {
@@ -1191,7 +1235,7 @@ class ZLinkFrameworkRuntimeHost {
             meshNodes: spotNodeRuntime.meshNodesByName,
             pollingIntervalMs: this.options.registration.locations.options.pollingIntervalMs ??
                 Locations_1.zlinkDefaultLocationOptions.pollingIntervalMs,
-            pageSize: 1000,
+            pageSize: Stores_1.ZLINK_PROVIDER_MAX_PAGE_SIZE,
             reportError: (error) => this.runtimeOrPreStartErrorSink.reportRuntimeTaskException('stateful authority route reconciliation', error),
             onSpotRouteChanged: (spotId) => this.cachedLocationSpotRouteResolver?.invalidate?.(spotId)
         });
@@ -1211,11 +1255,6 @@ class ZLinkFrameworkRuntimeHost {
             const activationNode = node;
             const spotManager = this.spotManager;
             if (spotManager !== undefined) {
-                activationNode.setSpotAdmissionProvider?.({
-                    isClosing: (spotId) => spotManager.isSpotClosing(meshName, spotId),
-                    awaitCloseDecision: (spotId) => spotManager.pendingSpotCloseDecision(meshName, spotId),
-                    runtimeState: () => this.runtimeState
-                });
                 this.registerInstanceApplicationLifecycle(meshName, activationNode, spotManager);
             }
             activationNode.registerAsyncInstanceActivationAuthority?.(new instance_activation_authority_1.ZLinkInstanceActivationAuthority({
@@ -1364,8 +1403,9 @@ class ZLinkFrameworkRuntimeHost {
             }
         }
         catch (error) {
-            throw new ZLinkDrainingStatePublishError(error);
+            throw new route_mesh_runtime_1.ZLinkDrainingStatePublishError(error);
         }
+        await awaitWithDrainSignal(this.serviceRelocation.drainTargetAttempts(), signal);
     }
     async cleanupOwnerForDrain(signal) {
         const runtime = this.locationOwner.currentRuntime;
@@ -1387,7 +1427,6 @@ class ZLinkFrameworkRuntimeHost {
         activationNode.registerInstanceApplicationLifecycle?.({
             isMaterialized: (target) => spotManager.isInstanceMaterialized(meshName, target.stableType, target.targetSpotId),
             isMaterializing: (target) => spotManager.isInstanceMaterializing(meshName, target.targetSpotId),
-            isClosing: (target) => spotManager.isInstanceClosing(meshName, target.targetSpotId),
             isIdleEvicting: (target) => spotManager.isInstanceSpotIdleEvicting(meshName, target.targetSpotId),
             beginIdleEviction: (target) => spotManager.beginInstanceSpotIdleEviction(meshName, target.targetSpotId),
             materialize: (target, objectGeneration) => spotManager.materializeInstance(meshName, target.stableType, target.targetSpotId, objectGeneration, this.executionState?.abortController.signal),
@@ -1402,12 +1441,6 @@ class ZLinkFrameworkRuntimeHost {
     setSpotManager(spotManager) {
         this.spotManager = spotManager;
         for (const [meshName, node] of this.spotNodeRuntime?.meshNodesByName ?? []) {
-            const admissionNode = node;
-            admissionNode.setSpotAdmissionProvider?.({
-                isClosing: (spotId) => spotManager.isSpotClosing(meshName, spotId),
-                awaitCloseDecision: (spotId) => spotManager.pendingSpotCloseDecision(meshName, spotId),
-                runtimeState: () => this.runtimeState
-            });
             this.registerInstanceApplicationLifecycle(meshName, node, spotManager);
         }
     }
@@ -1477,7 +1510,10 @@ class ZLinkFrameworkRuntimeHost {
                 detachedTaskRunner: this.detachedTaskRunner(),
                 metrics: this.metrics,
                 admission: this.admission,
-                statefulExecutionAllowed: () => this.ownerAdmissionOpen()
+                statefulExecution: {
+                    admissionOpen: () => this.ownerAdmissionOpen(),
+                    hostState: () => this.runtimeState
+                }
             }).create(this.actorTransferRuntime),
             activationAdmission: this.activationAdmission
         };
@@ -1521,7 +1557,7 @@ class ZLinkFrameworkRuntimeHost {
                     throw (0, framework_errors_internal_1.createInternalFrameworkException)(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ObjectClientNotConfigured, `RouteMesh '${meshName}' has no object-client role.`);
                 }
                 const meshNode = this.spotNodeRuntime?.meshNode(meshName);
-                const descriptors = (await locationStore.listMeshNodes(meshName, undefined, signal)).items;
+                const descriptors = await (0, location_store_pages_1.listAllMeshNodeDescriptors)(locationStore, meshName, signal);
                 const selected = await selectReadyLocationPlacementDescriptor(descriptors, meshNode, signal, (descriptor) => {
                     const capability = descriptor.objectCapabilities.find((candidate) => candidate.objectKind === 'actor' && candidate.stableType === stableType);
                     const actors = descriptor.populationCapacity.actors;
@@ -1602,7 +1638,7 @@ class ZLinkFrameworkRuntimeHost {
                     throw (0, framework_errors_internal_1.createInternalFrameworkException)(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ObjectClientNotConfigured, `RouteMesh '${meshName}' has no object-client role.`);
                 }
                 const meshNode = this.spotNodeRuntime?.meshNode(meshName);
-                const descriptors = (await locationStore.listMeshNodes(meshName, undefined, signal)).items;
+                const descriptors = await (0, location_store_pages_1.listAllMeshNodeDescriptors)(locationStore, meshName, signal);
                 const selected = await selectReadyLocationPlacementDescriptor(descriptors, meshNode, signal, (descriptor) => {
                     const capability = descriptor.objectCapabilities.find((candidate) => candidate.objectKind === 'user_spot' && candidate.stableType === request.stableType);
                     const spotTypeCapacity = descriptor.populationCapacity.spotTypes.find((candidate) => candidate.objectKind === 'user_spot' && candidate.stableType === request.stableType);
@@ -1680,7 +1716,7 @@ class ZLinkFrameworkRuntimeHost {
                             try {
                                 payload = {
                                     packetName: 'ZLinkFrameworkUserSpotReply',
-                                    contentType: 'application/octet-stream',
+                                    contentType: stream_wire_1.ZlinkStreamContentType.Raw,
                                     payload: Buffer.from(message.data())
                                 };
                             }
@@ -1804,7 +1840,7 @@ class ZLinkFrameworkRuntimeHost {
                 }
                 return node.requestUserSpotClose(targetNodeRid, request, timeoutMs);
             },
-            defaultTimeoutMs: this.options.registration.requestTimeoutMs ?? 30_000,
+            defaultTimeoutMs: this.options.registration.requestTimeoutMs ?? Registration_1.DEFAULT_REQUEST_TIMEOUT_MS,
             messageSerializers: this.options.registration.messageSerializers
         });
     }
@@ -1824,6 +1860,8 @@ class ZLinkFrameworkRuntimeHost {
             meshCompletions: (meshName) => this.spotNodeRuntime?.meshCompletionTable(meshName),
             notifyEntrySpotActorCreated: (nodeRid, actor, createRequest, signal) => this.spotNodeRuntime?.notifyEntrySpotActorCreated(nodeRid, actor, createRequest, signal) ??
                 Promise.resolve(undefined),
+            notifyEntrySpotActorJoined: (actor, signal) => this.spotNodeRuntime?.notifyPrimaryEntrySpotActorJoined(actor, signal) ?? Promise.resolve(),
+            notifyEntrySpotActorLeft: (actor, signal) => this.spotNodeRuntime?.notifyPrimaryEntrySpotActorLeft(actor, signal) ?? Promise.resolve(),
             locationLifecycle: () => this.locationOwner.currentLifecycle,
             primaryMeshName: () => this.meshRouters.primaryMeshName(),
             actorMeshName: (actorType) => this.meshRouters.actorMeshName(actorType),
@@ -1845,7 +1883,7 @@ class ZLinkFrameworkRuntimeHost {
                 if (entrySpotId === undefined) {
                     throw new ConfigurationException_1.ZLinkConfigurationException(`Actor '${actorRef.actorId}' authority requires its RouteMesh Entry Spot.`);
                 }
-                await (0, actors_2.publishInitialActorAuthority)(store, {
+                await (0, actors_1.publishInitialActorAuthority)(store, {
                     actorType,
                     actor: actorRef,
                     meshName,
@@ -1913,6 +1951,7 @@ class ZLinkFrameworkRuntimeHost {
         }).create();
         return {
             ...options,
+            errorSink: this.runtimeOrPreStartErrorSink,
             peerAdmissionSealed: (meshName) => !this.admission.accepts(meshName),
             messageFollowReceiver: (record) => {
                 if (record.source.kind === 'actor') {
@@ -1951,10 +1990,8 @@ class ZLinkFrameworkRuntimeHost {
                 throw new ConfigurationException_1.ZLinkConfigurationException('MeshNode Actor binding record has no binding generation.');
             }
             const binding = record.kindData;
-            const replyBindingFailure = (kind, message) => {
-                const terminal = kind === framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ActorGenerationStale
-                    ? { terminalResult: runtime_values_1.RequestResult.InvalidState, failureCode: 0 }
-                    : (0, framework_errors_internal_1.internalFrameworkWireReply)((0, framework_errors_internal_1.createInternalFrameworkException)(kind, message));
+            const replyBindingFailure = (kind) => {
+                const terminal = (0, framework_errors_internal_1.internalFrameworkWireReply)(kind);
                 if (record.replyFailure?.(terminal.terminalResult, terminal.failureCode) !== runtime_values_1.SubmitResult.Ok) {
                     throw new ConfigurationException_1.ZLinkConfigurationException(`MeshNode Actor binding target '${binding.actor.actorId}' failure acknowledgement failed.`);
                 }
@@ -1970,11 +2007,11 @@ class ZLinkFrameworkRuntimeHost {
                 nativeActorRef === undefined ||
                 nativeActorRef.actorId !== binding.actor.actorId ||
                 !(0, routing_id_1.routingIdsEqual)(nativeActorRef.nodeRid, binding.actor.nodeRid)) {
-                replyBindingFailure(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ActorLocationStale, `Actor '${binding.actor.actorId}' binding target is no longer current.`);
+                replyBindingFailure(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ActorLocationStale);
                 return;
             }
             if (nativeActorRef.generation !== binding.actor.generation) {
-                replyBindingFailure(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ActorGenerationStale, `Actor '${binding.actor.actorId}' binding object generation is stale.`);
+                replyBindingFailure(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ActorGenerationStale);
                 return;
             }
             const node = state.meshName === meshName ? this.spotNodeRuntime?.meshNode(meshName) : undefined;
@@ -1986,7 +2023,7 @@ class ZLinkFrameworkRuntimeHost {
                 localStatus.lifecycleGeneration !== binding.actorNodeGeneration ||
                 state.locationGeneration !== binding.authorityOwnerGeneration ||
                 state.isMoving) {
-                replyBindingFailure(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ActorLocationStale, `Actor '${binding.actor.actorId}' binding authority fence is stale.`);
+                replyBindingFailure(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ActorLocationStale);
                 return;
             }
             const sessionOwnerIsLocal = (0, routing_id_1.routingIdsEqual)(localStatus.routingId, binding.sessionNodeRid) &&
@@ -1999,6 +2036,7 @@ class ZLinkFrameworkRuntimeHost {
                     nodeRid: binding.actor.nodeRid
                 };
                 if (sessionOwnerIsLocal) {
+                    (0, application_job_queue_scope_1.releaseApplicationJobPermitBeforeHandler)();
                     await this.streamBindingRuntime.retireRemoteBinding(actorRef, binding.sessionRid, binding.bindingGeneration, signal);
                 }
                 state.retireBoundSessionBinding(binding);
@@ -2007,7 +2045,7 @@ class ZLinkFrameworkRuntimeHost {
             }
             const target = this.boundSessionRelay.boundSessions.resolveRemoteBoundSessionTarget(binding.sessionNodeRid, binding.sessionRid);
             if (target === undefined) {
-                replyBindingFailure(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ActorLocationStale, `Actor '${binding.actor.actorId}' Session route is unavailable.`);
+                replyBindingFailure(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ActorLocationStale);
                 return;
             }
             const installed = state.installBoundSessionBinding({
@@ -2020,7 +2058,7 @@ class ZLinkFrameworkRuntimeHost {
                 bindingGeneration: binding.bindingGeneration
             });
             if (!installed) {
-                replyBindingFailure(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ActorLocationStale, `Actor '${binding.actor.actorId}' Session binding generation is stale.`);
+                replyBindingFailure(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ActorLocationStale);
                 return;
             }
             replyBindingSuccess();
@@ -2047,7 +2085,7 @@ class ZLinkFrameworkRuntimeHost {
                         return;
                     }
                     if (record.kind === service_runtime_contracts_1.ReceiveKind.NodeRequest && record.parts.length === 1) {
-                        const terminal = (0, actors_2.decodeRemoteActorSourceLeaveTerminal)(record.parts[0].data());
+                        const terminal = (0, actors_1.decodeRemoteActorSourceLeaveTerminal)(record.parts[0].data());
                         if (terminal !== undefined) {
                             if ((await this.spotManager?.completeFormalSourceLeaveTerminal(terminal.actorId, terminal.transferId, terminal.succeeded)) !== true) {
                                 throw new ConfigurationException_1.ZLinkConfigurationException(`Actor '${terminal.actorId}' has no matching formal transfer terminal gate.`);
@@ -2231,7 +2269,7 @@ class ZLinkFrameworkRuntimeHost {
         const resolver = authority === undefined
             ? legacy
             : new locations_1.ZLinkAuthoritySpotRouteResolver(authority, this.meshRouters.spotRouterChannelIdByMesh(), legacy, this.locationOwner.currentLeaseTracker, this.options.registration.locations.options.routeCacheMaxAgeMs, undefined, async (meshName, nodeRid, expectedLifecycleGeneration, signal) => {
-                const descriptor = (await authority.listMeshNodes(meshName, undefined, signal)).items.find((candidate) => (0, routing_id_1.routingIdsEqual)(candidate.rid, nodeRid) &&
+                const descriptor = (await (0, location_store_pages_1.listAllMeshNodeDescriptors)(authority, meshName, signal)).find((candidate) => (0, routing_id_1.routingIdsEqual)(candidate.rid, nodeRid) &&
                     candidate.lifecycleGeneration === expectedLifecycleGeneration);
                 return descriptor?.state;
             });
@@ -2282,12 +2320,13 @@ function hasExactPeerReadiness(descriptors, local, peers) {
         replacementDescriptors.some((descriptor) => peers.some((peer) => peer.routingId !== null &&
             String(peer.routingId) === String(descriptor.rid) &&
             peer.lifecycleGeneration === descriptor.lifecycleGeneration &&
-            peer.state === 3)));
+            peer.state === service_runtime_contracts_1.MeshPeerRuntimeState.Serving)));
 }
 var actor_transfer_authority_runtime_2 = require("./actor-transfer-authority-runtime");
 Object.defineProperty(exports, "ZLinkActorTransferAuthorityRuntime", { enumerable: true, get: function () { return actor_transfer_authority_runtime_2.ZLinkActorTransferAuthorityRuntime; } });
 Object.defineProperty(exports, "transferIdString", { enumerable: true, get: function () { return actor_transfer_authority_runtime_2.transferIdString; } });
 var route_mesh_runtime_2 = require("./route-mesh-runtime");
+Object.defineProperty(exports, "ZLinkDrainingStatePublishError", { enumerable: true, get: function () { return route_mesh_runtime_2.ZLinkDrainingStatePublishError; } });
 Object.defineProperty(exports, "ZLinkRetiringRollbackError", { enumerable: true, get: function () { return route_mesh_runtime_2.ZLinkRetiringRollbackError; } });
 Object.defineProperty(exports, "ZLinkRouteMeshRuntimeCoordinator", { enumerable: true, get: function () { return route_mesh_runtime_2.ZLinkRouteMeshRuntimeCoordinator; } });
 var service_relocation_host_runtime_2 = require("./service-relocation-host-runtime");
@@ -2301,7 +2340,7 @@ async function waitForForcedSessionNotification(operation) {
         return;
     let timeoutHandle;
     const timeout = new Promise((resolve) => {
-        timeoutHandle = setTimeout(resolve, 100);
+        timeoutHandle = setTimeout(resolve, HOST_SHUTDOWN_POLL_INTERVAL_MS);
     });
     try {
         await Promise.race([operation.catch(() => undefined), timeout]);
@@ -2330,12 +2369,6 @@ async function awaitWithDrainSignal(operation, signal) {
         });
     });
 }
-class ZLinkDrainingStatePublishError extends Error {
-    constructor(cause) {
-        super('Failed to publish draining peer rows.', { cause });
-        this.name = 'ZLinkDrainingStatePublishError';
-    }
-}
 function resolveBackendAdapterFactory(internalOptions) {
     if (typeof internalOptions === 'object' &&
         internalOptions !== null &&
@@ -2465,7 +2498,7 @@ function waitForPlacementReadiness(signal) {
         const timer = setTimeout(() => {
             signal?.removeEventListener('abort', abort);
             resolve();
-        }, 10);
+        }, HANDOFF_ACCEPTANCE_POLL_INTERVAL_MS);
         const abort = () => {
             clearTimeout(timer);
             signal?.removeEventListener('abort', abort);
@@ -2503,8 +2536,8 @@ function relocationReason(reason) {
 }
 function rewriteAuthorityPayloadForOwner(payload, owner) {
     const applicationPayload = (0, service_relocation_runtime_1.serviceRelocationAuthorityApplicationPayload)(payload);
-    const rewritten = (0, actors_2.isActorAuthorityPayload)(applicationPayload)
-        ? (0, actors_2.rewriteActorAuthorityOwner)(applicationPayload, owner)
+    const rewritten = (0, actors_1.isActorAuthorityPayload)(applicationPayload)
+        ? (0, actors_1.rewriteActorAuthorityOwner)(applicationPayload, owner)
         : ((0, service_authority_payload_codec_1.rewriteServiceAuthorityOwner)(applicationPayload, owner) ?? applicationPayload);
     return (0, service_relocation_runtime_1.replaceServiceRelocationAuthorityApplicationPayload)(payload, rewritten);
 }