@zlink-systems/framework 0.18.1 → 0.20.0

package/dist/runtime/host/actor-placement-coordinator.js

@@ -6,8 +6,15 @@ const node_crypto_1 = require("node:crypto");
 const runtime_values_1 = require("../backend/runtime-values");
 const contracts_1 = require("../../contracts");
 const authority_key_codec_1 = require("../locations/authority-key-codec");
+const creation_operation_id_1 = require("../locations/creation-operation-id");
 const actor_authority_publication_1 = require("../actors/actor-authority-publication");
 const user_spot_creation_coordinator_1 = require("./user-spot-creation-coordinator");
+const service_wire_codec_generated_1 = require("../protocol/service_wire_codec.generated");
+const CREATION_TERMINAL_CODEC_CONTEXT = {
+    effectiveCompleteMessageBytesMinusActualEnvelopeOverhead: 1024 * 1024,
+    effectiveCompleteMessageBytes: 1024 * 1024,
+    runtimePredicates: {}
+};
 class ZLinkActorPlacementCoordinator {
     options;
     constructor(options) {
@@ -20,6 +27,7 @@ class ZLinkActorPlacementCoordinator {
         const contentReference = (0, user_spot_creation_coordinator_1.encodeLocationCreationContent)(requestPayload);
         const requestSha256 = (0, node_crypto_1.createHash)('sha256').update(requestPayload).digest();
         const excluded = new Set();
+        const operationId = (0, creation_operation_id_1.randomOperationId)();
         try {
             for (;;) {
                 const target = await this.options.target(meshName, stableType, deadline.signal, excluded);
@@ -75,7 +83,27 @@ class ZLinkActorPlacementCoordinator {
                 if (reserved.kind !== 'reserved') {
                     throw (0, framework_errors_internal_1.createInternalFrameworkException)(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ActorCreateFailed, `Actor '${actorId}' reservation failed with '${reserved.kind}'.`, true);
                 }
-                const remote = await this.options.remoteCreate(target.meshName, String(target.nodeRid), actorCreateRecord(actorId, stableType, reserved.creating, target, deadlineUnixMs), Math.max(1, Math.ceil(deadlineMs - performance.now())));
+                const operation = {
+                    sourceNodeRid: target.sourceNodeRid,
+                    sourceNodeGeneration: target.sourceNodeGeneration,
+                    operationId
+                };
+                let remote;
+                let transportFailure;
+                try {
+                    remote = await this.options.remoteCreate(target.meshName, String(target.nodeRid), actorCreateRecord(actorId, stableType, reserved.creating, target, deadlineUnixMs, operation), Math.max(1, Math.ceil(deadlineMs - performance.now())));
+                }
+                catch (error) {
+                    transportFailure = error;
+                }
+                if (remote === undefined || !isCompletedActorCreate(remote)) {
+                    const retained = await this.options.store.readCreationTerminal(operation);
+                    if (retained.kind === 'found') {
+                        remote = remoteActorCreateResult(retained.terminalEnvelope, target);
+                    }
+                }
+                if (remote === undefined)
+                    throw transportFailure;
                 if (remote.terminalResult !== runtime_values_1.RequestResult.Ok
                     || remote.failureCode !== 0) {
                     //  Classify the (terminal, fine failure code) pair via the shared
@@ -151,35 +179,51 @@ class ZLinkActorPlacementCoordinator {
         let completion;
         try {
             local = await materialize(requestPayload, current, signal);
-            const terminal = encodeActorTerminal(record.correlation, local.result, local.actor);
-            completion = await this.options.store.completeCreation({
-                key,
-                reservationId: record.reservation.reservationId,
-                expectedStoreVersion: current.storeVersion.value,
-                target: creationTarget(current),
-                completion: local.result === 'created'
-                    ? {
-                        kind: 'created',
-                        readyPayload: (0, actor_authority_publication_1.encodeActorAuthorityIdentity)({
-                            actorType: record.stableType,
-                            actor: local.actor,
-                            meshName: current.allocation.descriptor.meshName,
-                            ownerNodeGeneration: current.allocation.descriptorLifecycleGeneration,
-                            owner: {
-                                ownerId: current.ownerId,
-                                leaseGeneration: current.ownerLeaseGeneration
-                            },
-                            spotId: local.entrySpotId,
-                            spotGeneration: local.entrySpotGeneration,
-                            spotKind: contracts_1.ZLinkSpotKind.Entry
-                        }),
-                        terminal: terminalPublication(record, terminal)
-                    }
-                    : {
-                        kind: 'rejected',
-                        terminal: terminalPublication(record, terminal)
+            if (local.result === 'failed') {
+                completion = await this.options.store.completeCreation({
+                    key,
+                    reservationId: record.reservation.reservationId,
+                    expectedStoreVersion: current.storeVersion.value,
+                    target: creationTarget(current),
+                    completion: {
+                        kind: 'failed',
+                        terminal: terminalPublication(record, encodeActorTerminal({ result: 'failed' }))
                     }
-            }, signal);
+                }, signal);
+            }
+            else {
+                const terminal = encodeActorTerminal(local.result === 'created'
+                    ? { result: 'created', actor: local.actor, reply: local.reply }
+                    : { result: 'rejected', reply: local.reply });
+                completion = await this.options.store.completeCreation({
+                    key,
+                    reservationId: record.reservation.reservationId,
+                    expectedStoreVersion: current.storeVersion.value,
+                    target: creationTarget(current),
+                    completion: local.result === 'created'
+                        ? {
+                            kind: 'created',
+                            readyPayload: (0, actor_authority_publication_1.encodeActorAuthorityIdentity)({
+                                actorType: record.stableType,
+                                actor: local.actor,
+                                meshName: current.allocation.descriptor.meshName,
+                                ownerNodeGeneration: current.allocation.descriptorLifecycleGeneration,
+                                owner: {
+                                    ownerId: current.ownerId,
+                                    leaseGeneration: current.ownerLeaseGeneration
+                                },
+                                spotId: local.entrySpotId,
+                                spotGeneration: local.entrySpotGeneration,
+                                spotKind: contracts_1.ZLinkSpotKind.Entry
+                            }),
+                            terminal: terminalPublication(record, terminal)
+                        }
+                        : {
+                            kind: 'rejected',
+                            terminal: terminalPublication(record, terminal)
+                        }
+                }, signal);
+            }
         }
         catch (error) {
             const cleanup = createDeadline(1_000);
@@ -199,14 +243,22 @@ class ZLinkActorPlacementCoordinator {
             }
             throw error;
         }
+        if (local.result === 'failed') {
+            if (completion.kind !== 'failed' && completion.kind !== 'alreadyCompleted') {
+                throw (0, framework_errors_internal_1.createInternalFrameworkException)(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ActorCreateFailed, `Actor '${record.actorId}' failed creation did not record its terminal.`);
+            }
+            throw local.error;
+        }
         if (local.result === 'created'
             ? completion.kind !== 'created' && completion.kind !== 'alreadyCompleted'
             : completion.kind !== 'rejected' && completion.kind !== 'alreadyCompleted') {
             throw (0, framework_errors_internal_1.createInternalFrameworkException)(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ActorCreateFailed, `Actor '${record.actorId}' creation completion lost its reservation.`);
         }
-        if (local.result === 'created')
+        if (local.result === 'created') {
             local.onPublished?.();
-        return actorResult(local.result, local.actor, local.reply);
+            return actorResult('created', local.actor, local.reply);
+        }
+        return actorResult('rejected', undefined, local.reply);
     }
 }
 exports.ZLinkActorPlacementCoordinator = ZLinkActorPlacementCoordinator;
@@ -253,13 +305,14 @@ function withActorAuthorityFence(actor, authority) {
     });
     return actor;
 }
-function actorCreateRecord(actorId, stableType, snapshot, target, deadlineUnixMs) {
+function actorCreateRecord(actorId, stableType, snapshot, target, deadlineUnixMs, operation) {
     const pending = snapshot.pendingCreation;
     if (pending === undefined)
         throw new Error('Actor reservation omitted Pending creation.');
     return {
-        sourceNodeRid: '',
-        sourceNodeGeneration: 1n,
+        sourceNodeRid: String(operation.sourceNodeRid),
+        sourceNodeGeneration: operation.sourceNodeGeneration,
+        operation: operation.operationId,
         actorId,
         stableType,
         reservation: reservationFence(snapshot, target, pending.reservationId),
@@ -332,19 +385,79 @@ function actorResult(createResult, actor, reply) {
             })
     };
 }
-function encodeActorTerminal(correlation, result, actor) {
-    return Buffer.from(JSON.stringify({
-        correlation: correlation.toString(),
-        result,
-        actor: actor === undefined
-            ? undefined
+function isCompletedActorCreate(result) {
+    return result.terminalResult === runtime_values_1.RequestResult.Ok
+        && result.failureCode === 0
+        && result.tail?.kind === 'actorCreate';
+}
+function encodeActorTerminal(terminal) {
+    if (terminal.result === 'failed') {
+        return Buffer.from((0, service_wire_codec_generated_1.encodeCreationOperationTerminalV1)({
+            terminalResult: 'internalError',
+            failureCode: 'actorCreateFailed',
+            hasCreation: 'false',
+            hasApplicationPayload: 'false'
+        }, CREATION_TERMINAL_CODEC_CONTEXT));
+    }
+    return Buffer.from((0, service_wire_codec_generated_1.encodeCreationOperationTerminalV1)({
+        terminalResult: 'ok',
+        failureCode: 'none',
+        hasCreation: 'true',
+        creation: terminal.result === 'rejected'
+            ? { createResult: 'rejected' }
             : {
-                actorId: actor.actorId,
-                generation: actor.objectGeneration.toString(),
-                meshName: actor.meshName,
-                nodeRid: String(actor.nodeRid)
-            }
-    }));
+                createResult: 'created',
+                actor: {
+                    actorId: terminal.actor.actorId,
+                    objectGeneration: terminal.actor.objectGeneration
+                }
+            },
+        hasApplicationPayload: terminal.reply === undefined ? 'false' : 'true',
+        ...(terminal.reply === undefined
+            ? {}
+            : {
+                applicationPayload: {
+                    packetName: 'ZLinkFrameworkActorCreateReply',
+                    contentType: 'application/octet-stream',
+                    payload: Buffer.from(terminal.reply)
+                }
+            })
+    }, CREATION_TERMINAL_CODEC_CONTEXT));
+}
+function remoteActorCreateResult(terminalEnvelope, target) {
+    const terminal = (0, service_wire_codec_generated_1.decodeCreationOperationTerminalV1)(terminalEnvelope, CREATION_TERMINAL_CODEC_CONTEXT);
+    const terminalResult = (0, service_wire_codec_generated_1.enumWireRequestTerminalResult)(terminal.terminalResult);
+    const failureCode = (0, service_wire_codec_generated_1.enumWireFrameworkErrorCode)(terminal.failureCode);
+    if (terminal.hasCreation === 'false' || terminal.creation === undefined) {
+        return { terminalResult, failureCode };
+    }
+    const creation = terminal.creation;
+    return {
+        terminalResult,
+        failureCode,
+        tail: {
+            kind: 'actorCreate',
+            createResult: creation.createResult,
+            ...(creation.createResult === 'rejected'
+                ? {}
+                : {
+                    actor: {
+                        actorId: creation.actor.actorId,
+                        generation: creation.actor.objectGeneration,
+                        nodeRid: String(target.nodeRid)
+                    }
+                })
+        },
+        ...(terminal.applicationPayload === undefined
+            ? {}
+            : {
+                payload: {
+                    packetName: terminal.applicationPayload.packetName,
+                    contentType: terminal.applicationPayload.contentType,
+                    payload: Buffer.from(terminal.applicationPayload.payload)
+                }
+            })
+    };
 }
 function terminalPublication(record, terminalEnvelope) {
     return {
@@ -354,7 +467,6 @@ function terminalPublication(record, terminalEnvelope) {
             operationId: record.operation
         },
         terminalEnvelope,
-        terminalEnvelopeSha256: (0, node_crypto_1.createHash)('sha256').update(terminalEnvelope).digest(),
         operationDeadline: new Date(Number(record.deadlineUnixMs))
     };
 }

package/dist/runtime/host/index.d.ts

@@ -63,7 +63,6 @@ export declare class ZLinkFrameworkRuntimeHost implements ZLinkFrameworkRuntimeL
     private spotManager?;
     private ownerLeaseRecoveryRuntime?;
     private ownerLeaseRecoveryHandler?;
-    private ownerLeaseFailureHandler?;
     private registerUserSpotHandlers?;
     private readonly destroyedActorRefs;
     private readonly runtimeEventPublisher;
@@ -128,10 +127,12 @@ export declare class ZLinkFrameworkRuntimeHost implements ZLinkFrameworkRuntimeL
     private runtimeMetricMeshSnapshots;
     private stopCore;
     private installOwnerLeaseRecoveryPublication;
+    private resumeOwnerLeaseOwnedWork;
     private createStatefulAuthorityRoutes;
     private requiresStatefulAuthorityRuntime;
     private installLocationBackedAuthorities;
     private removeOwnerLeaseRecoveryPublication;
+    private ownerAdmissionOpen;
     onApplicationBootstrap(): Promise<void>;
     onApplicationShutdown(): Promise<void>;
     private performMeshDrain;

package/dist/runtime/host/index.js

@@ -97,7 +97,6 @@ class ZLinkFrameworkRuntimeHost {
     spotManager;
     ownerLeaseRecoveryRuntime;
     ownerLeaseRecoveryHandler;
-    ownerLeaseFailureHandler;
     registerUserSpotHandlers;
     destroyedActorRefs = new Map();
     runtimeEventPublisher;
@@ -105,7 +104,7 @@ class ZLinkFrameworkRuntimeHost {
     applicationJobQueue;
     capacityStatus;
     metricRegistrations = [];
-    admission = new admission_1.ZLinkRuntimeAdmissionGate();
+    admission = new admission_1.ZLinkRuntimeAdmissionGate(() => this.ownerAdmissionOpen());
     cachedLocationSpotRouteResolver;
     actorClientLocationResolver;
     // Shared, runtime-mutable message-flow mode cell — installed once so
@@ -638,6 +637,14 @@ class ZLinkFrameworkRuntimeHost {
                     : contracts_1.ZLinkFrameworkRelocationReason.RuntimeNotReady
             });
         }
+        if (!this.admission.acceptsNewWork) {
+            return Promise.resolve({
+                mode: options.mode,
+                effectiveTargetApplicationVersion,
+                outcome: contracts_1.ZLinkFrameworkRelocationOutcome.Blocked,
+                reason: contracts_1.ZLinkFrameworkRelocationReason.StoreUnavailable
+            });
+        }
         if (hasUnsupportedManualTopology(this.options.registration)) {
             return Promise.resolve({
                 mode: options.mode,
@@ -1116,96 +1123,40 @@ class ZLinkFrameworkRuntimeHost {
         }
     }
     installOwnerLeaseRecoveryPublication(runtime, spotNodeRuntime, channelRuntime) {
-        let publishing = false;
-        let lastPublishedOwnerToken = runtime.currentOwnerToken;
-        let recoveryRequired = false;
-        let recoveringServices;
-        let stoppingServices;
-        let transportFence = Promise.resolve();
         const handler = () => {
-            const ownerToken = runtime.currentOwnerToken;
-            if (publishing
-                || ownerToken === undefined
-                || (ownerToken === lastPublishedOwnerToken && !recoveryRequired))
-                return;
-            if (runtime.ownerLeaseUsable === false)
+            if (!this.ownerAdmissionOpen())
                 return;
-            publishing = true;
-            const signal = this.executionState?.abortController.signal;
-            const recoverServices = async () => {
-                if (stoppingServices !== undefined) {
-                    await stoppingServices;
-                    stoppingServices = undefined;
-                }
-                if (this.statefulAuthorityRoutes !== undefined)
-                    return;
-                const store = this.locationOwner.currentStores?.locationStore;
-                if (store === undefined || this.executionState === undefined)
-                    return;
-                if (recoveringServices !== undefined)
-                    return await recoveringServices;
-                recoveringServices = (async () => {
-                    if (this.requiresStatefulAuthorityRuntime()) {
-                        const routes = this.createStatefulAuthorityRoutes(store, spotNodeRuntime);
-                        await routes.start(this.executionState?.abortController.signal);
-                        this.statefulAuthorityRoutes = routes;
-                    }
-                })();
-                try {
-                    await recoveringServices;
-                }
-                finally {
-                    recoveringServices = undefined;
-                }
-            };
-            void transportFence
-                .then(() => spotNodeRuntime.publishMeshNodeState(contracts_1.ZLinkFrameworkRuntimeState.Preparing, signal))
-                .then(() => recoverServices())
-                .then(() => {
-                const currentOwnerToken = runtime.currentOwnerToken;
-                if (runtime.ownerLeaseUsable === false
-                    || currentOwnerToken === undefined
-                    || currentOwnerToken.ownerId !== ownerToken.ownerId
-                    || currentOwnerToken.leaseGeneration !== ownerToken.leaseGeneration) {
-                    throw new Error('Owner lease changed during recovery.');
-                }
-                return spotNodeRuntime.publishMeshNodeState(this.runtimeState, signal);
-            })
-                .then(() => spotNodeRuntime.startLocationAutoConnect(signal))
-                .then(() => channelRuntime.reclaimLocationOwnerRows(signal))
-                .then(() => this.locationOwner.currentLifecycle?.reclaimOwnerRows() ?? Promise.resolve())
-                .then(() => {
-                lastPublishedOwnerToken = ownerToken;
-                recoveryRequired = false;
-                publishing = false;
-            }, error => {
+            void this.resumeOwnerLeaseOwnedWork(runtime, spotNodeRuntime, channelRuntime)
+                .catch(error => {
+                runtime.closeOwnerLeaseAdmission(error);
                 this.runtimeOrPreStartErrorSink.reportRuntimeTaskException('owner lease recovery', error);
-                publishing = false;
             });
         };
         this.ownerLeaseRecoveryRuntime = runtime;
         this.ownerLeaseRecoveryHandler = handler;
-        const failureHandler = () => {
-            recoveryRequired = true;
-            if (runtime.ownerLeaseUsable)
-                return;
-            // A Store failure invalidates new ownership and discovery work, but it
-            // must not tear down transports that were already established. The
-            // auto-connect reconciler applies storeFailureGraceMs to the last
-            // complete descriptor set; fencing here would bypass that contract and
-            // make existing requests fail during the grace window.
-            const routes = this.statefulAuthorityRoutes;
-            if (routes === undefined)
-                return;
-            this.statefulAuthorityRoutes = undefined;
-            stoppingServices = routes.stop();
-            void stoppingServices.catch(error => this.runtimeOrPreStartErrorSink.reportRuntimeTaskException('stateful authority route fencing', error));
-        };
-        this.ownerLeaseFailureHandler = failureHandler;
         runtime.addOwnerLeaseRenewedHandler(handler);
-        if (typeof runtime.addOwnerLeaseRenewalFailedHandler === 'function') {
-            runtime.addOwnerLeaseRenewalFailedHandler(failureHandler);
-        }
+    }
+    async resumeOwnerLeaseOwnedWork(runtime, spotNodeRuntime, channelRuntime) {
+        if (!this.ownerAdmissionOpen())
+            return;
+        const signal = this.executionState?.abortController.signal;
+        await spotNodeRuntime.publishMeshNodeState(contracts_1.ZLinkFrameworkRuntimeState.Preparing, signal);
+        const store = this.locationOwner.currentStores?.locationStore;
+        if (this.statefulAuthorityRoutes === undefined
+            && store !== undefined
+            && this.executionState !== undefined
+            && this.requiresStatefulAuthorityRuntime()) {
+            const routes = this.createStatefulAuthorityRoutes(store, spotNodeRuntime);
+            await routes.start(signal);
+            this.statefulAuthorityRoutes = routes;
+        }
+        if (runtime.ownerLeaseUsable === false)
+            return;
+        await spotNodeRuntime.publishMeshNodeState(this.runtimeState, signal);
+        await spotNodeRuntime.startLocationAutoConnect(signal);
+        await channelRuntime.startLocationAutoConnect(signal);
+        await channelRuntime.reclaimLocationOwnerRows(signal);
+        await this.locationOwner.currentLifecycle?.reclaimOwnerRows();
     }
     createStatefulAuthorityRoutes(locationStore, spotNodeRuntime) {
         return new stateful_authority_route_runtime_1.ZLinkStatefulAuthorityRouteRuntime({
@@ -1268,14 +1219,14 @@ class ZLinkFrameworkRuntimeHost {
         if (this.ownerLeaseRecoveryRuntime !== undefined && this.ownerLeaseRecoveryHandler !== undefined) {
             this.ownerLeaseRecoveryRuntime.removeOwnerLeaseRenewedHandler(this.ownerLeaseRecoveryHandler);
         }
-        if (this.ownerLeaseRecoveryRuntime !== undefined && this.ownerLeaseFailureHandler !== undefined) {
-            if (typeof this.ownerLeaseRecoveryRuntime.removeOwnerLeaseRenewalFailedHandler === 'function') {
-                this.ownerLeaseRecoveryRuntime.removeOwnerLeaseRenewalFailedHandler(this.ownerLeaseFailureHandler);
-            }
-        }
         this.ownerLeaseRecoveryRuntime = undefined;
         this.ownerLeaseRecoveryHandler = undefined;
-        this.ownerLeaseFailureHandler = undefined;
+    }
+    ownerAdmissionOpen() {
+        const runtime = this.locationOwner.currentRuntime;
+        if (runtime !== undefined)
+            return runtime.ownerLeaseUsable;
+        return this.locationOwner.currentStores?.locationStore === undefined;
     }
     async onApplicationBootstrap() {
         await this.start();
@@ -1478,15 +1429,7 @@ class ZLinkFrameworkRuntimeHost {
                 detachedTaskRunner: this.detachedTaskRunner(),
                 metrics: this.metrics,
                 admission: this.admission,
-                statefulExecutionAllowed: () => {
-                    const locationRuntime = this.locationOwner.currentRuntime;
-                    if (locationRuntime !== undefined)
-                        return locationRuntime.ownerLeaseUsable;
-                    // A host configured with a Location Store must not execute stateful
-                    // timers while its lease runtime is unavailable. Hosts without a
-                    // Location Store have no owner lease to fence.
-                    return this.locationOwner.currentStores?.locationStore === undefined;
-                }
+                statefulExecutionAllowed: () => this.ownerAdmissionOpen()
             }).create(this.actorTransferRuntime),
             activationConcurrencyLimitProvider: (meshName) => this.options.registration.spotNodes.get(meshName)?.activationConcurrencyLimit ?? 128,
             onInstanceActivationConcurrencyChanged: (meshName) => {
@@ -1549,17 +1492,21 @@ class ZLinkFrameworkRuntimeHost {
                 if (selected === undefined)
                     return undefined;
                 const localStatus = meshNode?.status();
+                if (localStatus === undefined) {
+                    throw (0, framework_errors_internal_1.createInternalFrameworkException)(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ObjectClientNotConfigured, `RouteMesh '${meshName}' has no local client node.`);
+                }
                 return {
                     meshName,
                     nodeRid: selected.rid,
                     nodeGeneration: selected.lifecycleGeneration,
+                    sourceNodeRid: String(localStatus.routingId),
+                    sourceNodeGeneration: localStatus.lifecycleGeneration,
                     entrySpotId: selected.entrySpotId,
                     owner: {
                         ownerId: selected.ownerId,
                         leaseGeneration: selected.leaseGeneration
                     },
-                    isLocal: localStatus !== undefined
-                        && String(localStatus.routingId) === String(selected.rid)
+                    isLocal: String(localStatus.routingId) === String(selected.rid)
                         && localStatus.lifecycleGeneration === selected.lifecycleGeneration
                 };
             }
@@ -1723,12 +1670,20 @@ class ZLinkFrameworkRuntimeHost {
                         }
                         return await actorPlacement.handleRemoteCreate(record, async (requestPayload, authority, createSignal) => {
                             const request = (0, creation_payload_codec_1.decodeFrameworkCreationPayload)(requestPayload, this.options.registration.messageSerializers);
+                            const entrySpotId = this.spotNodeRuntime
+                                ?.meshNodeDescriptor(meshName)?.entrySpotId;
+                            if (entrySpotId === undefined) {
+                                throw new ConfigurationException_1.ZLinkConfigurationException(`Remote Actor creation requires the '${meshName}' MeshNode Entry Spot descriptor.`);
+                            }
                             const entrySpot = node.entrySpot().status();
                             const nativeActorRef = node.restoreActorAuthority?.(record.actorId, record.stableType, authority.objectGeneration, authority.authorityOwnerGeneration, String(entrySpot.routingId), entrySpot.lifecycleGeneration, 1n);
                             if (nativeActorRef === undefined) {
                                 throw new ConfigurationException_1.ZLinkConfigurationException('Remote Actor creation requires native authority restoration support.');
                             }
                             const local = await actors.createReservedActorResult(record.actorId, record.stableType, request, createSignal, nativeActorRef);
+                            if (local.status === 'failed') {
+                                return { result: 'failed', error: local.error };
+                            }
                             if (local.status === 'rejected') {
                                 let reply;
                                 if (local.reply !== undefined) {
@@ -1765,7 +1720,7 @@ class ZLinkFrameworkRuntimeHost {
                                     ...local.actorRef,
                                     meshName
                                 },
-                                entrySpotId: String(entrySpot.routingId),
+                                entrySpotId,
                                 entrySpotGeneration: entrySpot.lifecycleGeneration,
                                 ...(reply === undefined ? {} : { reply }),
                                 onPublished: () => actors.adoptCreatedAuthority(record.actorId, authority.authorityOwnerGeneration, authority.ownerLeaseGeneration)
@@ -2064,26 +2019,28 @@ class ZLinkFrameworkRuntimeHost {
             }
             case service_runtime_contracts_1.ReceiveKind.NodeSend:
             case service_runtime_contracts_1.ReceiveKind.NodeRequest: {
-                if (await this.serviceRelocation.tryHandleControl(meshName, record, signal)) {
-                    return;
-                }
-                if (record.kind === service_runtime_contracts_1.ReceiveKind.NodeRequest && record.parts.length === 1) {
-                    const terminal = (0, actors_2.decodeRemoteActorSourceLeaveTerminal)(record.parts[0].data());
-                    if (terminal !== undefined) {
-                        if (await this.spotManager?.completeFormalSourceLeaveTerminal(terminal.actorId, terminal.transferId, terminal.succeeded) !== true) {
-                            throw new ConfigurationException_1.ZLinkConfigurationException(`Actor '${terminal.actorId}' has no matching formal transfer terminal gate.`);
-                        }
-                        if (record.reply(Buffer.alloc(0)) !== runtime_values_1.SubmitResult.Ok) {
-                            throw new ConfigurationException_1.ZLinkConfigurationException(`Actor '${terminal.actorId}' formal transfer terminal acknowledgement failed.`);
+                return this.admission.run(meshName, 'RouteMesh node dispatch', async () => {
+                    if (await this.serviceRelocation.tryHandleControl(meshName, record, signal)) {
+                        return;
+                    }
+                    if (record.kind === service_runtime_contracts_1.ReceiveKind.NodeRequest && record.parts.length === 1) {
+                        const terminal = (0, actors_2.decodeRemoteActorSourceLeaveTerminal)(record.parts[0].data());
+                        if (terminal !== undefined) {
+                            if (await this.spotManager?.completeFormalSourceLeaveTerminal(terminal.actorId, terminal.transferId, terminal.succeeded) !== true) {
+                                throw new ConfigurationException_1.ZLinkConfigurationException(`Actor '${terminal.actorId}' has no matching formal transfer terminal gate.`);
+                            }
+                            if (record.reply(Buffer.alloc(0)) !== runtime_values_1.SubmitResult.Ok) {
+                                throw new ConfigurationException_1.ZLinkConfigurationException(`Actor '${terminal.actorId}' formal transfer terminal acknowledgement failed.`);
+                            }
+                            return;
                         }
-                        return Promise.resolve();
                     }
-                }
-                const channelRuntime = this.channelRuntime;
-                if (channelRuntime === undefined) {
-                    throw new ConfigurationException_1.ZLinkConfigurationException('MeshNode node-direct dispatch requires the channel runtime.');
-                }
-                return this.admission.run(meshName, 'RouteMesh node dispatch', () => channelRuntime.dispatchMeshRoute(meshName, record, signal));
+                    const channelRuntime = this.channelRuntime;
+                    if (channelRuntime === undefined) {
+                        throw new ConfigurationException_1.ZLinkConfigurationException('MeshNode node-direct dispatch requires the channel runtime.');
+                    }
+                    return await channelRuntime.dispatchMeshRoute(meshName, record, signal);
+                });
             }
             case service_runtime_contracts_1.ReceiveKind.SpotSend:
             case service_runtime_contracts_1.ReceiveKind.SpotRequest:

package/dist/runtime/host/location-runtime-owner.js

@@ -143,9 +143,11 @@ class ZLinkLocationRuntimeOwner {
             return runtime;
         }
         spotNodeRuntime.configureLocationAutoConnect(runtime, stores, this.options.registration.locations.options, this.currentEvents);
-        await spotNodeRuntime.startLocationAutoConnect();
         channelRuntime.configureLocationAutoConnect(runtime, stores, this.options.registration.locations.options, this.currentEvents);
-        await channelRuntime.startLocationAutoConnect();
+        if (runtime.ownerLeaseUsable) {
+            await spotNodeRuntime.startLocationAutoConnect();
+            await channelRuntime.startLocationAutoConnect();
+        }
         return runtime;
     }
     clearForStop() {

package/dist/runtime/host/remote-actor-join-receiver.js

@@ -126,9 +126,18 @@ class ZLinkRemoteActorJoinReceiver {
 }
 exports.ZLinkRemoteActorJoinReceiver = ZLinkRemoteActorJoinReceiver;
 function hasAuthorityFence(join) {
-    return join.actorNodeGeneration !== undefined
-        && join.expectedAuthorityOwnerGeneration !== undefined
-        && join.expectedOwnerLeaseGeneration !== undefined;
+    const fields = [
+        join.actorNodeGeneration,
+        join.expectedAuthorityOwnerGeneration,
+        join.expectedOwnerLeaseGeneration
+    ];
+    if (fields.every(value => value === undefined)) {
+        return false;
+    }
+    if (fields.some(value => value === undefined)) {
+        throw (0, framework_errors_internal_1.createInternalFrameworkException)(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.RequestProtocolError, `Actor '${join.actorId}' Join carries an incomplete Authority fence.`);
+    }
+    return true;
 }
 function decodeJoinAuthorityFence(join) {
     let objectGeneration;

package/dist/runtime/locations/creation-operation-id.d.ts

@@ -0,0 +1,4 @@
+export declare function randomOperationId(entropy?: Uint8Array): {
+    readonly high: bigint;
+    readonly low: bigint;
+};

package/dist/runtime/locations/creation-operation-id.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.randomOperationId = randomOperationId;
+const node_crypto_1 = require("node:crypto");
+function randomOperationId(entropy = (0, node_crypto_1.randomBytes)(16)) {
+    if (entropy.byteLength !== 16)
+        throw new RangeError('Operation ID entropy must be 16 bytes.');
+    const bytes = Buffer.from(entropy);
+    const high = bytes.readBigUInt64BE(0);
+    const low = bytes.readBigUInt64BE(8);
+    return high === 0n && low === 0n ? { high, low: 1n } : { high, low };
+}