@zlink-systems/framework 0.19.0 → 0.20.0

package/src/runtime/host/index.ts

@@ -279,7 +279,6 @@ export class ZLinkFrameworkRuntimeHost implements
   private spotManager?: DefaultZLinkSpotManager;
   private ownerLeaseRecoveryRuntime?: ZLinkLocationRuntime;
   private ownerLeaseRecoveryHandler?: () => void;
-  private ownerLeaseFailureHandler?: () => void;
   private registerUserSpotHandlers?: (runtime: ZLinkSpotNodeRuntimeManager) => void;
   private readonly destroyedActorRefs = new Map<string, ActorRef>();
   private readonly runtimeEventPublisher: ZLinkRuntimeEventPublisher;
@@ -287,7 +286,7 @@ export class ZLinkFrameworkRuntimeHost implements
   private readonly applicationJobQueue: ApplicationJobQueue;
   private readonly capacityStatus: HostCapacityStatusProjection;
   private readonly metricRegistrations: import('../diagnostics').ZLinkRuntimeMetricRegistration[] = [];
-  private readonly admission = new ZLinkRuntimeAdmissionGate();
+  private readonly admission = new ZLinkRuntimeAdmissionGate(() => this.ownerAdmissionOpen());
   private cachedLocationSpotRouteResolver?: ZLinkSpotRouteResolver;
   private actorClientLocationResolver?: ZLinkStoreLocationResolvers;
   // Shared, runtime-mutable message-flow mode cell — installed once so
@@ -939,6 +938,14 @@ export class ZLinkFrameworkRuntimeHost implements
           : ZLinkFrameworkRelocationReason.RuntimeNotReady
       });
     }
+    if (!this.admission.acceptsNewWork) {
+      return Promise.resolve({
+        mode: options.mode,
+        effectiveTargetApplicationVersion,
+        outcome: ZLinkFrameworkRelocationOutcome.Blocked,
+        reason: ZLinkFrameworkRelocationReason.StoreUnavailable
+      });
+    }
     if (hasUnsupportedManualTopology(this.options.registration)) {
       return Promise.resolve({
         mode: options.mode,
@@ -1526,103 +1533,45 @@ export class ZLinkFrameworkRuntimeHost implements
     spotNodeRuntime: ZLinkSpotNodeRuntimeManager,
     channelRuntime: ZLinkChannelRuntimeManager
   ): void {
-    let publishing = false;
-    let lastPublishedOwnerToken = runtime.currentOwnerToken;
-    let recoveryRequired = false;
-    let recoveringServices: Promise<void> | undefined;
-    let stoppingServices: Promise<void> | undefined;
-    let transportFence = Promise.resolve();
     const handler = () => {
-      const ownerToken = runtime.currentOwnerToken;
-      if (
-        publishing
-        || ownerToken === undefined
-        || (ownerToken === lastPublishedOwnerToken && !recoveryRequired)
-      ) return;
-      if (runtime.ownerLeaseUsable === false) return;
-      publishing = true;
-      const signal = this.executionState?.abortController.signal;
-      const recoverServices = async (): Promise<void> => {
-        if (stoppingServices !== undefined) {
-          await stoppingServices;
-          stoppingServices = undefined;
-        }
-        if (this.statefulAuthorityRoutes !== undefined) return;
-        const store = this.locationOwner.currentStores?.locationStore;
-        if (store === undefined || this.executionState === undefined) return;
-        if (recoveringServices !== undefined) return await recoveringServices;
-        recoveringServices = (async () => {
-          if (this.requiresStatefulAuthorityRuntime()) {
-            const routes = this.createStatefulAuthorityRoutes(store, spotNodeRuntime);
-            await routes.start(this.executionState?.abortController.signal);
-            this.statefulAuthorityRoutes = routes;
-          }
-        })();
-        try {
-          await recoveringServices;
-        } finally {
-          recoveringServices = undefined;
-        }
-      };
-      void transportFence
-        .then(() => spotNodeRuntime.publishMeshNodeState(
-          ZLinkFrameworkRuntimeState.Preparing,
-          signal
-        ))
-        .then(() => recoverServices())
-        .then(() => {
-          const currentOwnerToken = runtime.currentOwnerToken;
-          if (runtime.ownerLeaseUsable === false
-            || currentOwnerToken === undefined
-            || currentOwnerToken.ownerId !== ownerToken.ownerId
-            || currentOwnerToken.leaseGeneration !== ownerToken.leaseGeneration) {
-            throw new Error('Owner lease changed during recovery.');
-          }
-          return spotNodeRuntime.publishMeshNodeState(this.runtimeState, signal);
-        })
-        .then(() => spotNodeRuntime.startLocationAutoConnect(signal))
-        .then(() => channelRuntime.reclaimLocationOwnerRows(signal))
-        .then(() => this.locationOwner.currentLifecycle?.reclaimOwnerRows() ?? Promise.resolve())
-        .then(
-        () => {
-          lastPublishedOwnerToken = ownerToken;
-          recoveryRequired = false;
-          publishing = false;
-        },
-        error => {
+      if (!this.ownerAdmissionOpen()) return;
+      void this.resumeOwnerLeaseOwnedWork(runtime, spotNodeRuntime, channelRuntime)
+        .catch(error => {
+          runtime.closeOwnerLeaseAdmission(error);
           this.runtimeOrPreStartErrorSink.reportRuntimeTaskException(
             'owner lease recovery',
             error
           );
-          publishing = false;
-        }
-      );
+        });
     };
     this.ownerLeaseRecoveryRuntime = runtime;
     this.ownerLeaseRecoveryHandler = handler;
-    const failureHandler = () => {
-      recoveryRequired = true;
-      if (runtime.ownerLeaseUsable) return;
-      // A Store failure invalidates new ownership and discovery work, but it
-      // must not tear down transports that were already established. The
-      // auto-connect reconciler applies storeFailureGraceMs to the last
-      // complete descriptor set; fencing here would bypass that contract and
-      // make existing requests fail during the grace window.
-      const routes = this.statefulAuthorityRoutes;
-      if (routes === undefined) return;
-      this.statefulAuthorityRoutes = undefined;
-      stoppingServices = routes.stop();
-      void stoppingServices.catch(error =>
-        this.runtimeOrPreStartErrorSink.reportRuntimeTaskException(
-          'stateful authority route fencing',
-          error
-        ));
-    };
-    this.ownerLeaseFailureHandler = failureHandler;
     runtime.addOwnerLeaseRenewedHandler(handler);
-    if (typeof runtime.addOwnerLeaseRenewalFailedHandler === 'function') {
-      runtime.addOwnerLeaseRenewalFailedHandler(failureHandler);
+  }
+
+  private async resumeOwnerLeaseOwnedWork(
+    runtime: ZLinkLocationRuntime,
+    spotNodeRuntime: ZLinkSpotNodeRuntimeManager,
+    channelRuntime: ZLinkChannelRuntimeManager
+  ): Promise<void> {
+    if (!this.ownerAdmissionOpen()) return;
+    const signal = this.executionState?.abortController.signal;
+    await spotNodeRuntime.publishMeshNodeState(ZLinkFrameworkRuntimeState.Preparing, signal);
+    const store = this.locationOwner.currentStores?.locationStore;
+    if (this.statefulAuthorityRoutes === undefined
+      && store !== undefined
+      && this.executionState !== undefined
+      && this.requiresStatefulAuthorityRuntime()) {
+      const routes = this.createStatefulAuthorityRoutes(store, spotNodeRuntime);
+      await routes.start(signal);
+      this.statefulAuthorityRoutes = routes;
     }
+    if (runtime.ownerLeaseUsable === false) return;
+    await spotNodeRuntime.publishMeshNodeState(this.runtimeState, signal);
+    await spotNodeRuntime.startLocationAutoConnect(signal);
+    await channelRuntime.startLocationAutoConnect(signal);
+    await channelRuntime.reclaimLocationOwnerRows(signal);
+    await this.locationOwner.currentLifecycle?.reclaimOwnerRows();
   }
 
   private createStatefulAuthorityRoutes(
@@ -1715,14 +1664,14 @@ export class ZLinkFrameworkRuntimeHost implements
     if (this.ownerLeaseRecoveryRuntime !== undefined && this.ownerLeaseRecoveryHandler !== undefined) {
       this.ownerLeaseRecoveryRuntime.removeOwnerLeaseRenewedHandler(this.ownerLeaseRecoveryHandler);
     }
-    if (this.ownerLeaseRecoveryRuntime !== undefined && this.ownerLeaseFailureHandler !== undefined) {
-      if (typeof this.ownerLeaseRecoveryRuntime.removeOwnerLeaseRenewalFailedHandler === 'function') {
-        this.ownerLeaseRecoveryRuntime.removeOwnerLeaseRenewalFailedHandler(this.ownerLeaseFailureHandler);
-      }
-    }
     this.ownerLeaseRecoveryRuntime = undefined;
     this.ownerLeaseRecoveryHandler = undefined;
-    this.ownerLeaseFailureHandler = undefined;
+  }
+
+  private ownerAdmissionOpen(): boolean {
+    const runtime = this.locationOwner.currentRuntime;
+    if (runtime !== undefined) return runtime.ownerLeaseUsable;
+    return this.locationOwner.currentStores?.locationStore === undefined;
   }
 
   async onApplicationBootstrap(): Promise<void> {
@@ -2068,14 +2017,7 @@ export class ZLinkFrameworkRuntimeHost implements
       detachedTaskRunner: this.detachedTaskRunner(),
       metrics: this.metrics,
       admission: this.admission,
-      statefulExecutionAllowed: () => {
-        const locationRuntime = this.locationOwner.currentRuntime;
-        if (locationRuntime !== undefined) return locationRuntime.ownerLeaseUsable;
-        // A host configured with a Location Store must not execute stateful
-        // timers while its lease runtime is unavailable. Hosts without a
-        // Location Store have no owner lease to fence.
-        return this.locationOwner.currentStores?.locationStore === undefined;
-      }
+      statefulExecutionAllowed: () => this.ownerAdmissionOpen()
     }).create(this.actorTransferRuntime),
       activationConcurrencyLimitProvider: (meshName: string) =>
         this.options.registration.spotNodes.get(meshName)?.activationConcurrencyLimit ?? 128,
@@ -2165,17 +2107,24 @@ export class ZLinkFrameworkRuntimeHost implements
         );
         if (selected === undefined) return undefined;
         const localStatus = meshNode?.status();
+        if (localStatus === undefined) {
+          throw createInternalFrameworkException(
+            ZLinkFrameworkInternalErrorKind.ObjectClientNotConfigured,
+            `RouteMesh '${meshName}' has no local client node.`
+          );
+        }
         return {
           meshName,
           nodeRid: selected.rid,
           nodeGeneration: selected.lifecycleGeneration,
+          sourceNodeRid: String(localStatus.routingId),
+          sourceNodeGeneration: localStatus.lifecycleGeneration,
           entrySpotId: selected.entrySpotId!,
           owner: {
             ownerId: selected.ownerId,
             leaseGeneration: selected.leaseGeneration
           },
-          isLocal: localStatus !== undefined
-            && String(localStatus.routingId) === String(selected.rid)
+          isLocal: String(localStatus.routingId) === String(selected.rid)
             && localStatus.lifecycleGeneration === selected.lifecycleGeneration
         };
       }
@@ -2394,6 +2343,13 @@ export class ZLinkFrameworkRuntimeHost implements
                 requestPayload,
                 this.options.registration.messageSerializers
               );
+              const entrySpotId = this.spotNodeRuntime
+                ?.meshNodeDescriptor(meshName)?.entrySpotId;
+              if (entrySpotId === undefined) {
+                throw new ZLinkConfigurationException(
+                  `Remote Actor creation requires the '${meshName}' MeshNode Entry Spot descriptor.`
+                );
+              }
               const entrySpot = node.entrySpot().status();
               const nativeActorRef = node.restoreActorAuthority?.(
                 record.actorId,
@@ -2416,6 +2372,9 @@ export class ZLinkFrameworkRuntimeHost implements
                 createSignal,
                 nativeActorRef
               );
+              if (local.status === 'failed') {
+                return { result: 'failed' as const, error: local.error };
+              }
               if (local.status === 'rejected') {
                 let reply: Buffer | undefined;
                 if (local.reply !== undefined) {
@@ -2461,7 +2420,7 @@ export class ZLinkFrameworkRuntimeHost implements
                   ...local.actorRef,
                   meshName
                 },
-                entrySpotId: String(entrySpot.routingId),
+                entrySpotId,
                 entrySpotGeneration: entrySpot.lifecycleGeneration,
                 ...(reply === undefined ? {} : { reply }),
                 onPublished: () => actors.adoptCreatedAuthority(
@@ -2849,37 +2808,38 @@ export class ZLinkFrameworkRuntimeHost implements
       }
       case ReceiveKind.NodeSend:
       case ReceiveKind.NodeRequest: {
-        if (await this.serviceRelocation.tryHandleControl(meshName, record, signal)) {
-          return;
-        }
-        if (record.kind === ReceiveKind.NodeRequest && record.parts.length === 1) {
-          const terminal = decodeRemoteActorSourceLeaveTerminal(record.parts[0]!.data());
-          if (terminal !== undefined) {
-            if (
-              await this.spotManager?.completeFormalSourceLeaveTerminal(
-                terminal.actorId,
-                terminal.transferId,
-                terminal.succeeded
-              ) !== true
-            ) {
-              throw new ZLinkConfigurationException(
-                `Actor '${terminal.actorId}' has no matching formal transfer terminal gate.`
-              );
-            }
-            if (record.reply(Buffer.alloc(0)) !== SubmitResult.Ok) {
-              throw new ZLinkConfigurationException(
-                `Actor '${terminal.actorId}' formal transfer terminal acknowledgement failed.`
-              );
+        return this.admission.run(meshName, 'RouteMesh node dispatch', async () => {
+          if (await this.serviceRelocation.tryHandleControl(meshName, record, signal)) {
+            return;
+          }
+          if (record.kind === ReceiveKind.NodeRequest && record.parts.length === 1) {
+            const terminal = decodeRemoteActorSourceLeaveTerminal(record.parts[0]!.data());
+            if (terminal !== undefined) {
+              if (
+                await this.spotManager?.completeFormalSourceLeaveTerminal(
+                  terminal.actorId,
+                  terminal.transferId,
+                  terminal.succeeded
+                ) !== true
+              ) {
+                throw new ZLinkConfigurationException(
+                  `Actor '${terminal.actorId}' has no matching formal transfer terminal gate.`
+                );
+              }
+              if (record.reply(Buffer.alloc(0)) !== SubmitResult.Ok) {
+                throw new ZLinkConfigurationException(
+                  `Actor '${terminal.actorId}' formal transfer terminal acknowledgement failed.`
+                );
+              }
+              return;
             }
-            return Promise.resolve();
           }
-        }
-        const channelRuntime = this.channelRuntime;
-        if (channelRuntime === undefined) {
-          throw new ZLinkConfigurationException('MeshNode node-direct dispatch requires the channel runtime.');
-        }
-        return this.admission.run(meshName, 'RouteMesh node dispatch', () =>
-          channelRuntime.dispatchMeshRoute(meshName, record, signal));
+          const channelRuntime = this.channelRuntime;
+          if (channelRuntime === undefined) {
+            throw new ZLinkConfigurationException('MeshNode node-direct dispatch requires the channel runtime.');
+          }
+          return await channelRuntime.dispatchMeshRoute(meshName, record, signal);
+        });
       }
       case ReceiveKind.SpotSend:
       case ReceiveKind.SpotRequest:

package/src/runtime/host/location-runtime-owner.ts

@@ -212,14 +212,16 @@ export class ZLinkLocationRuntimeOwner {
       this.options.registration.locations.options,
       this.currentEvents
     );
-    await spotNodeRuntime.startLocationAutoConnect();
     channelRuntime.configureLocationAutoConnect(
       runtime,
       stores,
       this.options.registration.locations.options,
       this.currentEvents
     );
-    await channelRuntime.startLocationAutoConnect();
+    if (runtime.ownerLeaseUsable) {
+      await spotNodeRuntime.startLocationAutoConnect();
+      await channelRuntime.startLocationAutoConnect();
+    }
     return runtime;
   }
 

package/src/runtime/host/remote-actor-join-receiver.ts

@@ -215,9 +215,21 @@ function hasAuthorityFence(
   readonly expectedAuthorityOwnerGeneration: string;
   readonly expectedOwnerLeaseGeneration: string;
 } {
-  return join.actorNodeGeneration !== undefined
-    && join.expectedAuthorityOwnerGeneration !== undefined
-    && join.expectedOwnerLeaseGeneration !== undefined;
+  const fields = [
+    join.actorNodeGeneration,
+    join.expectedAuthorityOwnerGeneration,
+    join.expectedOwnerLeaseGeneration
+  ];
+  if (fields.every(value => value === undefined)) {
+    return false;
+  }
+  if (fields.some(value => value === undefined)) {
+    throw createInternalFrameworkException(
+      ZLinkFrameworkInternalErrorKind.RequestProtocolError,
+      `Actor '${join.actorId}' Join carries an incomplete Authority fence.`
+    );
+  }
+  return true;
 }
 
 function decodeJoinAuthorityFence(join: JoinAuthorityFence): {

package/src/runtime/locations/creation-operation-id.ts

@@ -0,0 +1,11 @@
+import { randomBytes } from 'node:crypto';
+
+export function randomOperationId(
+  entropy: Uint8Array = randomBytes(16)
+): { readonly high: bigint; readonly low: bigint } {
+  if (entropy.byteLength !== 16) throw new RangeError('Operation ID entropy must be 16 bytes.');
+  const bytes = Buffer.from(entropy);
+  const high = bytes.readBigUInt64BE(0);
+  const low = bytes.readBigUInt64BE(8);
+  return high === 0n && low === 0n ? { high, low: 1n } : { high, low };
+}

package/src/runtime/locations/in-memory-authority-store.ts

@@ -1,4 +1,4 @@
-import { createHash, randomUUID, timingSafeEqual } from 'node:crypto';
+import { randomUUID } from 'node:crypto';
 import type {
   ZLinkAggregateAbortResult,
   ZLinkAggregateCommitResult,
@@ -31,6 +31,7 @@ import type {
   ZLinkPlacementAllocation
 } from './internal-location-contracts';
 import { encodeAuthorityKey } from './authority-key-codec';
+import { creationTerminalPreimage } from './opaque-record-key';
 
 const MAX_GENERATION = 0x7fff_ffff_ffff_ffffn;
 const MAX_PAYLOAD_BYTES = 1024 * 1024;
@@ -394,9 +395,6 @@ export class ZLinkInMemoryAuthorityStore {
     }
     const terminalRecord = validateTerminalForMutation(
       request.completion.terminal,
-      request.completion.kind,
-      request.reservationId,
-      request.key.kind,
       this.now()
     )!;
     const existingTerminal = this.operationTerminals.get(
@@ -486,7 +484,7 @@ export class ZLinkInMemoryAuthorityStore {
       this.operationTerminals.delete(key);
       return { kind: 'missing', storeNow: this.now() };
     }
-    return { kind: 'found', record: copyTerminalRecord(record) };
+    return { kind: 'found', terminalEnvelope: Buffer.from(record.terminalEnvelope) };
   }
 
   private creationTerminalAvailable(record: ZLinkCreationTerminalRecord): boolean {
@@ -843,7 +841,7 @@ function validateCreationOperation(operation: ZLinkCreationOperationIdentity): v
   }
   if (
     operation.sourceNodeGeneration < 1n
-    || operation.sourceNodeGeneration > MAX_GENERATION
+    || operation.sourceNodeGeneration > MAX_U64
     || operation.operationId.high < 0n
     || operation.operationId.high > MAX_U64
     || operation.operationId.low < 0n
@@ -856,22 +854,12 @@ function validateCreationOperation(operation: ZLinkCreationOperationIdentity): v
 
 function validateTerminalForMutation(
   publication: ZLinkCreationTerminalPublication | undefined,
-  state: ZLinkCreationTerminalRecord['state'],
-  reservationId: string,
-  objectKind: ZLinkPlacementAllocation['objectKind'],
   storeNow: Date
 ): ZLinkCreationTerminalRecord | undefined {
   if (publication === undefined) return undefined;
   validateCreationOperation(publication.operation);
-  if (
-    publication.terminalEnvelope.byteLength > MAX_PAYLOAD_BYTES
-    || publication.terminalEnvelopeSha256.byteLength !== 32
-  ) {
-    throw new RangeError('Creation terminal envelope must not exceed 1 MiB and requires a SHA-256 digest.');
-  }
-  const actualSha = createHash('sha256').update(publication.terminalEnvelope).digest();
-  if (!timingSafeEqual(actualSha, Buffer.from(publication.terminalEnvelopeSha256))) {
-    throw new TypeError('Creation terminal envelope SHA-256 does not match its bytes.');
+  if (publication.terminalEnvelope.byteLength > MAX_PAYLOAD_BYTES) {
+    throw new RangeError('Creation terminal envelope must not exceed 1 MiB.');
   }
   const deadlineMs = publication.operationDeadline.getTime();
   const expiresAtMs = deadlineMs + CREATION_TERMINAL_RETENTION_MS;
@@ -883,24 +871,15 @@ function validateTerminalForMutation(
     throw new RangeError('Creation terminal expiry must be the live operation deadline plus five minutes.');
   }
   return {
-    state,
     operation: copyCreationOperation(publication.operation),
-    reservationId: requireText(reservationId, 'creation reservation ID'),
-    objectKind,
     terminalEnvelope: Buffer.from(publication.terminalEnvelope),
-    terminalEnvelopeSha256: Buffer.from(publication.terminalEnvelopeSha256),
     expiresAt: new Date(expiresAtMs),
     storeNow: new Date(storeNow.getTime())
   };
 }
 
 function creationOperationKey(operation: ZLinkCreationOperationIdentity): string {
-  return [
-    String(operation.sourceNodeRid),
-    operation.sourceNodeGeneration.toString(),
-    operation.operationId.high.toString(16).padStart(16, '0'),
-    operation.operationId.low.toString(16).padStart(16, '0')
-  ].join('\0');
+  return creationTerminalPreimage(operation);
 }
 
 function copyCreationOperation(
@@ -921,7 +900,6 @@ function copyTerminalRecord(record: ZLinkCreationTerminalRecord): ZLinkCreationT
     ...record,
     operation: copyCreationOperation(record.operation),
     terminalEnvelope: Buffer.from(record.terminalEnvelope),
-    terminalEnvelopeSha256: Buffer.from(record.terminalEnvelopeSha256),
     expiresAt: new Date(record.expiresAt),
     storeNow: new Date(record.storeNow)
   };