@zlink-systems/framework 0.29.0 → 0.30.0

package/dist/runtime/host/application-job-queue.d.ts

@@ -1,4 +1,4 @@
-import type { ApplicationJobPermitPort, ApplicationJobQueuePort } from '../application-jobs/contracts';
+import type { ApplicationJobOrigin, ApplicationJobPermitPort, ApplicationJobQueuePort } from '../application-jobs/contracts';
 export type ApplicationJobQueueProfileValue = 'compact' | 'low_latency' | 'balanced' | 'throughput';
 export interface ApplicationJobQueueOptions {
     readonly profile?: ApplicationJobQueueProfileValue;
@@ -39,8 +39,9 @@ export interface ApplicationJobQueueSnapshot extends ApplicationJobQueueConfigur
 export declare function resolveApplicationJobQueueConfiguration(options?: ApplicationJobQueueOptions, effectiveProcessorCount?: () => bigint): ApplicationJobQueueConfiguration;
 export declare class ApplicationJobQueuePermit implements ApplicationJobPermitPort {
     private readonly owner;
+    readonly origin: ApplicationJobOrigin;
     private state;
-    constructor(owner: ApplicationJobQueue);
+    constructor(owner: ApplicationJobQueue, origin: ApplicationJobOrigin);
     markApplicationQueued(): void;
     releaseBeforeHandler(): void;
     releaseAfterInternalProcessing(): void;
@@ -55,6 +56,9 @@ export declare class ApplicationJobQueue implements ApplicationJobQueuePort {
     private readonly configuration;
     private readonly nowMs;
     private readonly handlerStartGate?;
+    private readonly loggerFailureReporter?;
+    private localWaiters;
+    private localBacklogLogged;
     private reservedSupplyPermits;
     private queuedApplicationJobs;
     private peakPermitsInUse;
@@ -73,11 +77,12 @@ export declare class ApplicationJobQueue implements ApplicationJobQueuePort {
     private readonly waiters;
     private readonly receiveFlowTargets;
     private readonly receiveFlowController;
-    constructor(configuration: ApplicationJobQueueConfiguration, nowMs?: () => number, handlerStartGate?: (() => boolean) | undefined);
+    constructor(configuration: ApplicationJobQueueConfiguration, nowMs?: () => number, handlerStartGate?: (() => boolean) | undefined, loggerFailureReporter?: ((error: unknown) => void) | undefined);
     shouldHoldPermitBeforeHandler(): boolean;
     registerReceiveFlowTarget(identity: object, applyState: (state: ApplicationJobQueuePressureState) => void, failureSink?: (error: unknown) => void): boolean;
     unregisterReceiveFlowTarget(identity: object): void;
-    acquire(signal?: AbortSignal): Promise<ApplicationJobQueuePermit>;
+    acquire(signal: AbortSignal | undefined, origin: ApplicationJobOrigin): Promise<ApplicationJobQueuePermit>;
+    acquireOrResume(resume: (permit: ApplicationJobQueuePermit) => void, reject: (reason: unknown) => void, signal: AbortSignal | undefined, origin: ApplicationJobOrigin): ApplicationJobQueuePermit | undefined;
     snapshot(): ApplicationJobQueueSnapshot;
     resetMetrics(): void;
     pressureState(): ApplicationJobQueuePressureState;
@@ -92,6 +97,8 @@ export declare class ApplicationJobQueue implements ApplicationJobQueuePort {
     private drainWaiters;
     private cancel;
     private recordCompletedWait;
+    private updateLocalWaiters;
+    private logLocalBacklog;
     private evaluatePressure;
     private pauseDurationAt;
     private cumulativePauseDurationAt;

package/dist/runtime/host/application-job-queue.js

@@ -3,6 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.ApplicationJobQueue = exports.ApplicationJobQueuePermit = void 0;
 exports.resolveApplicationJobQueueConfiguration = resolveApplicationJobQueueConfiguration;
 exports.nodeEffectiveProcessorCount = nodeEffectiveProcessorCount;
+const api_logs_1 = require("@opentelemetry/api-logs");
+const message_flow_1 = require("../diagnostics/message-flow");
 const runtime_metrics_1 = require("../diagnostics/runtime-metrics");
 const Dispatch_1 = require("../../contracts/Dispatch");
 const Registration_1 = require("../../contracts/Configuration/Registration");
@@ -50,9 +52,11 @@ function resolveApplicationJobQueueConfiguration(options = {}, effectiveProcesso
 }
 class ApplicationJobQueuePermit {
     owner;
+    origin;
     state = 'reserved';
-    constructor(owner) {
+    constructor(owner, origin) {
         this.owner = owner;
+        this.origin = origin;
     }
     markApplicationQueued() {
         if (this.state !== 'reserved') {
@@ -86,6 +90,9 @@ class ApplicationJobQueue {
     configuration;
     nowMs;
     handlerStartGate;
+    loggerFailureReporter;
+    localWaiters = 0;
+    localBacklogLogged = false;
     reservedSupplyPermits = 0n;
     queuedApplicationJobs = 0n;
     peakPermitsInUse = 0n;
@@ -104,10 +111,11 @@ class ApplicationJobQueue {
     waiters = [];
     receiveFlowTargets = new Map();
     receiveFlowController;
-    constructor(configuration, nowMs = () => performance.now(), handlerStartGate) {
+    constructor(configuration, nowMs = () => performance.now(), handlerStartGate, loggerFailureReporter) {
         this.configuration = configuration;
         this.nowMs = nowMs;
         this.handlerStartGate = handlerStartGate;
+        this.loggerFailureReporter = loggerFailureReporter;
         this.receiveFlowController = new receive_flow_controller_1.ApplicationJobReceiveFlowController(this, (state) => state);
     }
     shouldHoldPermitBeforeHandler() {
@@ -153,15 +161,24 @@ class ApplicationJobQueue {
         this.receiveFlowTargets.delete(identity);
         this.receiveFlowController.unregister(target);
     }
-    acquire(signal) {
+    acquire(signal, origin) {
+        return new Promise((resolve, reject) => {
+            const permit = this.acquireOrResume(resolve, reject, signal, origin);
+            if (permit !== undefined)
+                resolve(permit);
+        });
+    }
+    acquireOrResume(resume, reject, signal, origin) {
         if (signal?.aborted === true) {
-            return Promise.reject(abortReason(signal));
+            reject(abortReason(signal));
+            return undefined;
         }
         if (this.waiters.length === 0 && this.permitsInUse() < this.limit()) {
-            return Promise.resolve(this.reserve());
+            return this.reserve(origin);
         }
-        return new Promise((resolve, reject) => {
+        new Promise((resolve, reject) => {
             const waiter = {
+                origin,
                 resolve,
                 reject,
                 signal,
@@ -172,9 +189,15 @@ class ApplicationJobQueue {
             };
             this.waiters.push(waiter);
             this.capacityWaitCount += 1n;
+            const warningCount = this.updateLocalWaiters(origin, 1);
             signal?.addEventListener('abort', waiter.abort, { once: true });
             this.drainWaiters();
-        });
+            if (warningCount !== undefined)
+                this.logLocalBacklog(warningCount);
+        })
+            .then(resume)
+            .catch(reject);
+        return undefined;
     }
     snapshot() {
         const permitsInUse = this.permitsInUse();
@@ -253,14 +276,14 @@ class ApplicationJobQueue {
     permitsInUse() {
         return this.reservedSupplyPermits + this.queuedApplicationJobs;
     }
-    reserve(evaluatePressure = true) {
+    reserve(origin, evaluatePressure = true) {
         this.reservedSupplyPermits += 1n;
         const current = this.permitsInUse();
         if (current > this.peakPermitsInUse)
             this.peakPermitsInUse = current;
         if (evaluatePressure)
             this.evaluatePressure();
-        return new ApplicationJobQueuePermit(this);
+        return new ApplicationJobQueuePermit(this, origin);
     }
     drainWaiters() {
         while (this.waiters.length > 0 && this.permitsInUse() < this.limit()) {
@@ -270,7 +293,7 @@ class ApplicationJobQueue {
             waiter.active = false;
             waiter.signal?.removeEventListener('abort', waiter.abort);
             this.recordCompletedWait(waiter);
-            waiter.resolve(this.reserve(false));
+            waiter.resolve(this.reserve(waiter.origin, false));
         }
     }
     cancel(waiter) {
@@ -286,11 +309,44 @@ class ApplicationJobQueue {
         this.drainWaiters();
     }
     recordCompletedWait(waiter) {
+        this.updateLocalWaiters(waiter.origin, -1);
         if (waiter.metricsEpoch !== this.metricsEpoch)
             return;
         const durationMs = Math.max(0, this.nowMs() - waiter.startedAtMs);
         this.capacityWaitDurationSeconds += durationMs / runtime_metrics_1.MILLISECONDS_PER_SECOND;
     }
+    updateLocalWaiters(origin, delta) {
+        if (origin !== 'local')
+            return undefined;
+        this.localWaiters += delta;
+        if (this.localWaiters === 0)
+            this.localBacklogLogged = false;
+        if (!this.localBacklogLogged && this.localWaiters > Number(this.limit())) {
+            this.localBacklogLogged = true;
+            return this.localWaiters;
+        }
+        return undefined;
+    }
+    logLocalBacklog(localWaiters) {
+        try {
+            message_flow_1.telemetryLogger.emit({
+                eventName: 'zlink.runtime.host.local_job_backlog_exceeded',
+                timestamp: new Date(),
+                severityNumber: api_logs_1.SeverityNumber.WARN,
+                severityText: 'WARN',
+                body: 'Same-host jobs waiting for a host permit exceeded the application job queue maximum.',
+                attributes: {
+                    source_kind: 'host',
+                    source_name: 'zlink.runtime.host',
+                    local_waiters: localWaiters,
+                    effective_maximum: Number(this.limit())
+                }
+            });
+        }
+        catch (error) {
+            this.loggerFailureReporter?.(error);
+        }
+    }
     evaluatePressure() {
         const permitsInUse = this.permitsInUse();
         const next = this.pressureStateValue === 'running'

package/dist/runtime/host/entry-actor-runtime.js

@@ -66,7 +66,7 @@ class ZLinkEntryActorRuntimeService {
         }
         return {
             handled: true,
-            response: await spotManager.dispatchRoutedActorPacket(spotId, actorId, parts, returnResponse, remoteBoundSessionTarget, fallbackActorRef)
+            response: await spotManager.dispatchRoutedActorPacket('remote', spotId, actorId, parts, returnResponse, remoteBoundSessionTarget, fallbackActorRef)
         };
     }
     requireActorManager() {

package/dist/runtime/host/index.d.ts

@@ -155,7 +155,7 @@ export declare class ZLinkFrameworkRuntimeHost implements ZLinkFrameworkRuntimeL
     private registerInstanceApplicationLifecycle;
     setActorManager(actorManager: DefaultZLinkActorManager): void;
     setSpotManager(spotManager: DefaultZLinkSpotManager): void;
-    createActorManagerOptions(): Pick<ZLinkActorManagerOptions, 'joinCoordinator' | 'actorMeshNameProvider' | 'actorLeaveSpot' | 'messageSerializers' | 'nativeActorNode' | 'nativeActorNodeProvider' | 'actorCreatedNodeRidProvider' | 'actorRefResolver' | 'actorCreatedNotifier' | 'actorDestroyedCleanup' | 'locationLifecycle' | 'boundSessionFactory' | 'shutdownSignal' | 'admission' | 'activationAdmission' | 'placementCreate'>;
+    createActorManagerOptions(): Pick<ZLinkActorManagerOptions, 'joinCoordinator' | 'actorMeshNameProvider' | 'actorLeaveSpot' | 'messageSerializers' | 'nativeActorNode' | 'nativeActorNodeProvider' | 'actorCreatedNodeRidProvider' | 'actorRefResolver' | 'actorCreatedNotifier' | 'actorDestroyedCleanup' | 'terminateActorActivation' | 'locationLifecycle' | 'boundSessionFactory' | 'shutdownSignal' | 'admission' | 'activationAdmission' | 'placementCreate'>;
     createActorClientOptions(): ConstructorParameters<typeof DefaultZLinkActorClient>[0];
     createLocationHandleResolver(): ZLinkStoreLocationResolvers | undefined;
     createSpotManagerOptions(): Partial<ZLinkSpotManagerOptions>;

package/dist/runtime/host/index.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.ZLinkUserSpotCreationCoordinator = exports.ZLinkHostSpotAddressTransport = exports.ZLinkRelocationStateIncompatibleError = exports.ZLinkRouteMeshRuntimeCoordinator = exports.ZLinkRetiringRollbackError = exports.ZLinkDrainingStatePublishError = exports.transferIdString = exports.ZLinkActorTransferAuthorityRuntime = exports.ZLinkFrameworkRuntimeHost = void 0;
 exports.hasUnsupportedManualTopology = hasUnsupportedManualTopology;
 exports.hasExactPeerReadiness = hasExactPeerReadiness;
+const application_job_queue_scope_1 = require("../application-jobs/application-job-queue-scope");
 const location_store_pages_1 = require("../locations/location-store-pages");
 const stream_wire_1 = require("@zlink-systems/stream-wire");
 const node_crypto_1 = require("node:crypto");
@@ -21,7 +22,7 @@ const actor_message_follow_context_1 = require("../actors/actor-message-follow-c
 const actor_transport_delivery_gate_1 = require("../actors/actor-transport-delivery-gate");
 const admission_1 = require("../admission");
 const application_job_queue_handler_start_gate_1 = require("../application-jobs/application-job-queue-handler-start-gate");
-const application_job_queue_scope_1 = require("../application-jobs/application-job-queue-scope");
+const application_job_queue_scope_2 = require("../application-jobs/application-job-queue-scope");
 const backend_1 = require("../backend");
 const runtime_message_1 = require("../backend/runtime-message");
 const runtime_values_1 = require("../backend/runtime-values");
@@ -161,7 +162,7 @@ class ZLinkFrameworkRuntimeHost {
             catch {
                 return false;
             }
-        });
+        }, (error) => (this.errorSink ?? this.preStartErrorSink).reportRuntimeTaskException('logger-provider', error));
         this.capacityStatus = new host_capacity_status_1.HostCapacityStatusProjection(options.registration.coreHwm, this.applicationJobQueue);
         this.clientServerRuntime = new topology_runtime_projections_1.ZLinkClientServerRuntimeProjection(() => this.channelRuntime, () => this.runtimeState);
         this.fanoutRuntime = new topology_runtime_projections_1.ZLinkFanoutRuntimeProjection(() => this.channelRuntime, () => this.runtimeState);
@@ -452,12 +453,12 @@ class ZLinkFrameworkRuntimeHost {
             },
             sessionRelocationWire: () => this.serviceRelocation,
             clearRemoteActorPacketTarget: (actorId) => this.boundSessionRelay.clearRemoteActorPacketTarget(actorId),
-            prepareApplicationJob: async (preparationSignal) => {
+            prepareApplicationJob: async (preparationSignal, origin) => {
                 const runtimeSignal = this.executionState?.abortController.signal;
                 const signal = runtimeSignal === undefined
                     ? preparationSignal
                     : AbortSignal.any([runtimeSignal, preparationSignal]);
-                const permit = await this.applicationJobQueue.acquire(signal);
+                const permit = await this.applicationJobQueue.acquire(signal, origin);
                 let state = 'ready';
                 let closedReason;
                 let abort;
@@ -491,7 +492,7 @@ class ZLinkFrameworkRuntimeHost {
                         state = 'running';
                         signal.removeEventListener('abort', abort);
                         try {
-                            return await (0, application_job_queue_scope_1.runWithApplicationJobPermit)(permit, operation);
+                            return await (0, application_job_queue_scope_2.runWithApplicationJobPermit)(permit, operation);
                         }
                         finally {
                             state = 'closed';
@@ -1245,6 +1246,7 @@ class ZLinkFrameworkRuntimeHost {
             }
             activationNode.registerAsyncInstanceActivationAuthority?.(new instance_activation_authority_1.ZLinkInstanceActivationAuthority({
                 store: locationStore,
+                relocationPolicy: (stableType) => this.options.registration.spotNodes.get(meshName)?.instanceSpotFactoryRegistrations?.[stableType]?.relocation.kind,
                 relocationStore: this.options.registration.locations.relocationStoreInstance,
                 meshName,
                 owner: () => this.locationOwner.currentRuntime?.currentOwnerToken,
@@ -1511,6 +1513,8 @@ class ZLinkFrameworkRuntimeHost {
         }
         this.actorPlacement ??= new actor_placement_coordinator_1.ZLinkActorPlacementCoordinator({
             store: locationStore,
+            actorRelocationPolicy: (meshName, stableType) => this.options.registration.spotNodes.get(meshName)?.actorFactoryRegistrations?.[stableType]
+                ?.relocation.kind,
             remoteCreate: (meshName, targetNodeRid, request, timeoutMs) => {
                 const node = this.spotNodeRuntime?.meshNode(meshName);
                 if (node === undefined) {
@@ -1546,13 +1550,11 @@ class ZLinkFrameworkRuntimeHost {
                 const descriptors = await (0, location_store_pages_1.listAllMeshNodeDescriptors)(locationStore, meshName, signal);
                 const selected = await selectReadyLocationPlacementDescriptor(descriptors, meshNode, signal, (descriptor) => {
                     const capability = descriptor.objectCapabilities.find((candidate) => candidate.objectKind === 'actor' && candidate.stableType === stableType);
-                    const actors = descriptor.populationCapacity.actors;
                     return (descriptor.state === contracts_1.ZLinkFrameworkRuntimeState.Serving &&
                         descriptor.objectRole === 'server' &&
                         descriptor.placementWeight > 0 &&
                         descriptor.entrySpotId !== undefined &&
                         excludedNodeRids?.has(String(descriptor.rid)) !== true &&
-                        (actors.limit === 0 || actors.active + actors.reserved < actors.limit) &&
                         capability !== undefined);
                 });
                 if (selected === undefined)
@@ -1579,6 +1581,8 @@ class ZLinkFrameworkRuntimeHost {
         });
         const coordinator = (this.userSpotCoordinator = new user_spot_creation_coordinator_1.ZLinkUserSpotCreationCoordinator({
             store: locationStore,
+            relocationPolicy: (meshName, stableType) => this.options.registration.spotNodes.get(meshName)?.spotFactoryRegistrations?.[stableType]
+                ?.relocation.kind,
             publishReadyRoute: (meshName, route) => {
                 this.cachedLocationSpotRouteResolver?.invalidate?.(route.spot.spotId);
                 this.spotNodeRuntime?.meshNode(meshName)?.rememberSpotRoute?.(route);
@@ -1832,6 +1836,7 @@ class ZLinkFrameworkRuntimeHost {
     }
     actorRuntimeOptionsFactory() {
         return new actor_runtime_options_factory_1.ZLinkActorRuntimeOptionsFactory({
+            terminateActorActivation: (actor, terminal) => this.spotNodeRuntime?.terminateActorActivation(actor, terminal) ?? terminal(),
             registration: this.options.registration,
             messageFlow: () => this.createDispatchErrorReporter(this.runtimeOrPreStartErrorSink).flow,
             routeTransport: this.routeTransport,
@@ -1960,11 +1965,11 @@ class ZLinkFrameworkRuntimeHost {
                     ownerLeaseGeneration: record.source.ownerLeaseGeneration
                 });
             },
-            meshRecordDispatcher: (meshName, owner, record) => this.dispatchMeshRecord(meshName, owner, record, this.executionState?.abortController.signal),
+            meshRecordDispatcher: (meshName, owner, record) => this.dispatchMeshRecord(application_job_queue_scope_1.currentApplicationJobOrigin, meshName, owner, record, this.executionState?.abortController.signal),
             activationConcurrency: (meshName) => this.activationAdmission.current(meshName)
         };
     }
-    async dispatchMeshRecord(meshName, owner, record, signal) {
+    async dispatchMeshRecord(applicationJobOrigin, meshName, owner, record, signal) {
         if (record.operationKind === service_runtime_contracts_1.OperationKind.ActorJoin) {
             if (this.spotManager === undefined) {
                 throw new ConfigurationException_1.ZLinkConfigurationException('MeshNode Actor join dispatch requires the Spot manager.');
@@ -2022,7 +2027,7 @@ class ZLinkFrameworkRuntimeHost {
                     nodeRid: binding.actor.nodeRid
                 };
                 if (sessionOwnerIsLocal) {
-                    (0, application_job_queue_scope_1.releaseApplicationJobPermitBeforeHandler)();
+                    (0, application_job_queue_scope_2.releaseApplicationJobPermitBeforeHandler)();
                     await this.streamBindingRuntime.retireRemoteBinding(actorRef, binding.sessionRid, binding.bindingGeneration, signal);
                 }
                 state.retireBoundSessionBinding(binding);
@@ -2108,7 +2113,7 @@ class ZLinkFrameworkRuntimeHost {
                 if (this.spotManager === undefined) {
                     throw new ConfigurationException_1.ZLinkConfigurationException('MeshNode Actor dispatch requires the Spot manager.');
                 }
-                return this.admission.run(meshName, 'RouteMesh Actor dispatch', () => this.spotManager.dispatchMeshActor(meshName, owner, record));
+                return this.admission.run(meshName, 'RouteMesh Actor dispatch', () => this.spotManager.dispatchMeshActor(applicationJobOrigin(), meshName, owner, record));
             case service_runtime_contracts_1.ReceiveKind.SendReady:
                 return Promise.resolve();
             default:
@@ -2142,9 +2147,9 @@ class ZLinkFrameworkRuntimeHost {
         try {
             state.taskRunner.runDetached('mesh-node-local-route-dispatch', async () => {
                 try {
-                    const permit = await this.applicationJobQueue.acquire(state.abortController.signal);
+                    const permit = await this.applicationJobQueue.acquire(state.abortController.signal, 'local');
                     permit.markApplicationQueued();
-                    await (0, application_job_queue_scope_1.runWithApplicationJobPermit)(permit, () => channelRuntime.dispatchLocalMeshRoute(meshName, sourceNodeRid, ownedParts));
+                    await (0, application_job_queue_scope_2.runWithApplicationJobPermit)(permit, () => channelRuntime.dispatchLocalMeshRoute(meshName, sourceNodeRid, ownedParts));
                 }
                 finally {
                     for (const part of ownedParts)

package/dist/runtime/host/instance-activation-authority.d.ts

@@ -5,6 +5,7 @@ import type { ServiceSpotState } from '../foundation/service-stateful-registry';
 import type { ServiceInstanceActivationTarget, ServiceInstanceRouteFence } from '../foundation/service-stateful-wire-codec';
 import { type ServiceInstanceActivationRecoveryEnvelope } from '../foundation/service-instance-activation-recovery-codec';
 export interface ZLinkInstanceActivationAuthorityOptions {
+    readonly relocationPolicy?: (stableType: string) => import('../locations/internal-location-contracts').ZLinkObjectReserveRequest['actorRelocationPolicy'];
     readonly store: ZLinkObjectCreationStore & ZLinkAuthorityStore;
     readonly relocationStore?: ZLinkRelocationStore;
     readonly meshName: string;
@@ -16,7 +17,7 @@ export declare class ZLinkInstanceActivationAuthority implements ServiceAsyncIns
     private readonly options;
     private readonly pending;
     constructor(options: ZLinkInstanceActivationAuthorityOptions);
-    read(target: ServiceInstanceActivationTarget): Promise<ServiceInstanceAuthorityRead>;
+    read(target: ServiceInstanceActivationTarget, instanceIntent?: true, activation?: ServiceInstanceActivationRecoveryEnvelope, acceptsLocal?: (route: ServiceInstanceRouteFence) => boolean): Promise<ServiceInstanceAuthorityRead>;
     reserve(activation: Omit<ServiceInstanceActivationRecoveryEnvelope, 'targetMeshName'>, signal?: AbortSignal): Promise<ServiceInstanceAuthorityReserve>;
     resume(target: ServiceInstanceActivationTarget, pending: ServicePendingInstanceActivation): Promise<ServiceInstanceActivationReservation>;
     commit(target: ServiceInstanceActivationTarget, reservation: ServiceInstanceActivationReservation, spot: ServiceSpotState, deadlineUnixMs?: bigint): Promise<{

package/dist/runtime/host/instance-activation-authority.js

@@ -17,9 +17,71 @@ class ZLinkInstanceActivationAuthority {
     constructor(options) {
         this.options = options;
     }
-    async read(target) {
-        const current = await this.options.store.readAuthority(authorityKey(target.targetSpotId));
-        return current.kind === 'snapshot' ? readyRead(current, target) : { kind: 'missing' };
+    async read(target, instanceIntent, activation, acceptsLocal) {
+        const key = authorityKey(target.targetSpotId);
+        const current = await this.options.store.readAuthority(key);
+        if (current.kind !== 'snapshot')
+            return { kind: 'missing' };
+        const ready = (0, service_authority_payload_codec_1.decodeServiceInstanceAuthorityPayload)(current.payload);
+        const resolved = readyRead(current, target, ready);
+        if (ready?.state === 'ready' &&
+            resolved.kind === 'ready' &&
+            acceptsLocal?.(resolved.route) === true) {
+            return resolved;
+        }
+        if (activation !== undefined &&
+            current.allocation.objectKind === 'instance_spot' &&
+            (0, routing_id_1.routingIdsEqual)(current.allocation.descriptor.rid, target.targetNodeRid) &&
+            current.allocation.descriptorLifecycleGeneration === target.targetNodeGeneration) {
+            const reference = current.pendingCreation?.requestContentReference ?? ready?.activationRecovery?.reference;
+            if (reference !== undefined) {
+                const root = await this.options.relocationStore?.read((0, relocation_blob_1.relocationBlobReference)(reference));
+                if (root?.kind !== 'found') {
+                    throw new contracts_1.ZLinkFrameworkException(contracts_1.ZLinkFrameworkErrorKind.DataLost, 'Stored Instance activation is missing.');
+                }
+                let stored;
+                try {
+                    stored = (0, service_instance_activation_recovery_codec_1.decodeInstanceActivationRecoveryEnvelope)(root.bytes);
+                }
+                catch (error) {
+                    if (!(error instanceof TypeError || error instanceof RangeError))
+                        throw error;
+                    throw new contracts_1.ZLinkFrameworkException(contracts_1.ZLinkFrameworkErrorKind.ProtocolError, 'Stored Instance activation is invalid.');
+                }
+                if (stored.operation.high === activation.operation.high &&
+                    stored.operation.low === activation.operation.low &&
+                    (stored.targetMeshName !== activation.targetMeshName ||
+                        stored.target.stableType !== target.stableType ||
+                        stored.target.descriptorVersion !== target.descriptorVersion ||
+                        stored.deadlineUnixMs !== activation.deadlineUnixMs ||
+                        (stored.metadataFrame === undefined) !== (activation.metadataFrame === undefined) ||
+                        (stored.metadataFrame !== undefined &&
+                            activation.metadataFrame !== undefined &&
+                            Buffer.compare(stored.metadataFrame, activation.metadataFrame) !== 0))) {
+                    throw new contracts_1.ZLinkFrameworkException(contracts_1.ZLinkFrameworkErrorKind.ProtocolError, 'Instance route does not match the stored activation.');
+                }
+            }
+        }
+        if (instanceIntent === true &&
+            current.allocation.state === 'active' &&
+            ready?.state === 'ready') {
+            if (current.allocation.objectKind !== 'instance_spot' ||
+                current.allocation.stableType !== target.stableType) {
+                throw (0, framework_errors_internal_1.createInternalFrameworkException)(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.SpotTypeMismatch, 'Instance activation type does not match the current authority.');
+            }
+            const released = await this.options.store.releaseEndedReservation?.(key, current.storeVersion.value, undefined, {
+                key: { kind: 'instance_spot', globalId: target.targetSpotId },
+                intent: { stableType: target.stableType },
+                actorRelocationPolicy: this.options.relocationPolicy?.(target.stableType)
+            });
+            if (released === true)
+                return { kind: 'missing' };
+            const refreshed = await this.options.store.readAuthority(key);
+            return refreshed.kind === 'snapshot'
+                ? readyRead(refreshed, target, (0, service_authority_payload_codec_1.decodeServiceInstanceAuthorityPayload)(refreshed.payload))
+                : { kind: 'missing' };
+        }
+        return resolved;
     }
     async reserve(activation, signal) {
         const target = activation.target;
@@ -49,6 +111,7 @@ class ZLinkInstanceActivationAuthority {
         try {
             reserved = await this.options.store.reserve({
                 key: { kind: 'instance_spot', globalId: target.targetSpotId },
+                actorRelocationPolicy: this.options.relocationPolicy?.(target.stableType),
                 intent: {
                     stableType: target.stableType,
                     requestContentReference: stored.reference.value,
@@ -207,7 +270,7 @@ class ZLinkInstanceActivationAuthority {
         }
         const current = await this.options.store.readAuthority(authorityKey(target.targetSpotId));
         if (current.kind === 'snapshot') {
-            const ready = readyRead(current, target);
+            const ready = readyRead(current, target, (0, service_authority_payload_codec_1.decodeServiceInstanceAuthorityPayload)(current.payload));
             if (ready.kind === 'ready') {
                 this.pending.delete(reservation.token);
                 return { kind: 'lost', route: ready.route };
@@ -236,7 +299,7 @@ class ZLinkInstanceActivationAuthority {
         }
         const recovery = decoded.activationRecovery;
         if (recovery === undefined)
-            return routeFromSnapshot(current);
+            return routeFromSnapshot(current, decoded.spotId);
         let completed = current;
         if (recovery.replayCursor < recovery.inboxSequence) {
             const cursorStored = await this.options.store.compareExchangeAuthority(key, current.storeVersion, {
@@ -338,19 +401,16 @@ class ZLinkInstanceActivationAuthority {
     }
 }
 exports.ZLinkInstanceActivationAuthority = ZLinkInstanceActivationAuthority;
-function readyRead(snapshot, target) {
-    const decoded = (0, service_authority_payload_codec_1.decodeServiceReadySpotAuthority)(snapshot.payload) ??
-        (0, service_authority_payload_codec_1.decodeServiceClosingSpotAuthority)(snapshot.payload);
-    const creating = (0, service_authority_payload_codec_1.decodeServiceInstanceAuthorityPayload)(snapshot.payload);
-    if (creating?.state === 'coldActivating' &&
+function readyRead(snapshot, target, decoded) {
+    if (decoded?.state === 'coldActivating' &&
         snapshot.allocation.objectKind === 'instance_spot' &&
-        creating.stableType === target.stableType &&
-        creating.spotId === target.targetSpotId &&
-        creating.ownerId === snapshot.ownerId &&
-        creating.ownerLeaseGeneration === snapshot.ownerLeaseGeneration &&
-        creating.ownerMeshName === snapshot.allocation.descriptor.meshName &&
-        creating.ownerNodeGeneration === snapshot.allocation.descriptorLifecycleGeneration &&
-        (0, routing_id_1.routingIdsEqual)(creating.ownerNodeRid, snapshot.allocation.descriptor.rid) &&
+        decoded.stableType === target.stableType &&
+        decoded.spotId === target.targetSpotId &&
+        decoded.ownerId === snapshot.ownerId &&
+        decoded.ownerLeaseGeneration === snapshot.ownerLeaseGeneration &&
+        decoded.ownerMeshName === snapshot.allocation.descriptor.meshName &&
+        decoded.ownerNodeGeneration === snapshot.allocation.descriptorLifecycleGeneration &&
+        (0, routing_id_1.routingIdsEqual)(decoded.ownerNodeRid, snapshot.allocation.descriptor.rid) &&
         (0, routing_id_1.routingIdsEqual)(target.targetNodeRid, snapshot.allocation.descriptor.rid) &&
         target.targetNodeGeneration === snapshot.allocation.descriptorLifecycleGeneration) {
         return {
@@ -360,7 +420,8 @@ function readyRead(snapshot, target) {
             authority: snapshot
         };
     }
-    if (decoded?.kind !== 'instance_spot' ||
+    if (decoded === undefined ||
+        decoded.state === 'coldActivating' ||
         decoded.stableType !== target.stableType ||
         decoded.spotId !== target.targetSpotId ||
         snapshot.allocation.objectKind !== 'instance_spot' ||
@@ -374,7 +435,7 @@ function readyRead(snapshot, target) {
         target.targetNodeGeneration !== snapshot.allocation.descriptorLifecycleGeneration) {
         return { kind: 'missing' };
     }
-    return { kind: 'ready', route: routeFromSnapshot(snapshot), authority: snapshot };
+    return { kind: 'ready', route: routeFromSnapshot(snapshot, decoded.spotId), authority: snapshot };
 }
 function requireCommitIdentity(target, reservation, creating, spot, meshName) {
     if (creating.allocation.state !== 'reserved' ||
@@ -392,16 +453,19 @@ function requireCommitIdentity(target, reservation, creating, spot, meshName) {
         throw new Error('Instance activation Ready identity does not match its exact reservation.');
     }
 }
-function routeFromSnapshot(snapshot) {
-    const decoded = (0, service_authority_payload_codec_1.decodeServiceReadySpotAuthority)(snapshot.payload) ??
-        (0, service_authority_payload_codec_1.decodeServiceClosingSpotAuthority)(snapshot.payload);
-    if (decoded?.kind !== 'instance_spot') {
-        throw new Error('Instance authority is not Ready.');
+function routeFromSnapshot(snapshot, spotId) {
+    if (spotId === undefined) {
+        const decoded = (0, service_authority_payload_codec_1.decodeServiceReadySpotAuthority)(snapshot.payload) ??
+            (0, service_authority_payload_codec_1.decodeServiceClosingSpotAuthority)(snapshot.payload);
+        if (decoded?.kind !== 'instance_spot') {
+            throw new Error('Instance authority is not Ready.');
+        }
+        spotId = decoded.spotId;
     }
     return {
         targetNodeRid: String(snapshot.allocation.descriptor.rid),
         targetNodeGeneration: snapshot.allocation.descriptorLifecycleGeneration,
-        targetSpotId: decoded.spotId,
+        targetSpotId: spotId,
         objectGeneration: snapshot.objectGeneration,
         ownerId: snapshot.ownerId,
         authorityOwnerGeneration: snapshot.authorityOwnerGeneration,

package/dist/runtime/host/service-relocation-host-runtime.js

@@ -3227,9 +3227,9 @@ class LocalTargetPort {
                 if (runtime === undefined) {
                     throw new Error('Relocated Entry Spot Actor dispatch is unavailable.');
                 }
-                return runtime.dispatchEntryActorPacket(hidden.actor.context.actorId, parts, returnResponse, remoteBoundSessionTarget, fallbackActorRef);
+                return runtime.dispatchEntryActorPacket('remote', hidden.actor.context.actorId, parts, returnResponse, remoteBoundSessionTarget, fallbackActorRef);
             }
-            return this.requireSpotManager().dispatchRoutedActorPacket(state.spotId, hidden.actor.context.actorId, parts, returnResponse, remoteBoundSessionTarget, fallbackActorRef);
+            return this.requireSpotManager().dispatchRoutedActorPacket('remote', state.spotId, hidden.actor.context.actorId, parts, returnResponse, remoteBoundSessionTarget, fallbackActorRef);
         });
         hidden.replayResults.push(...results);
     }

package/dist/runtime/host/spot-address-transport.d.ts

@@ -31,6 +31,7 @@ export declare class ZLinkHostSpotAddressTransport implements ZLinkSpotAddressTr
     private reportInstanceRequestError;
     private traceInstanceAddress;
     private resolveExisting;
+    private reactivationCall;
     private selectMissingTarget;
     private validateExisting;
     private encode;