@zlink-systems/framework 0.29.0 → 0.30.0

package/src/runtime/foundation/service-liveness-registry.ts

@@ -1,7 +1,7 @@
 import { UINT64_MAX } from '@zlink-systems/stream-wire';
 export interface ServiceProbe {
   readonly nodeRoutingId: string;
-  readonly connectionId: string;
+  readonly connectionId: string | bigint;
   readonly probeId: bigint;
 }
 
@@ -10,20 +10,52 @@ export interface ServiceLivenessTick {
   readonly timedOutNodes: readonly string[];
 }
 
-interface PeerState {
-  readonly connectionId: string;
+export const DEFAULT_SERVICE_PROBE_INTERVAL_MS = 5_000;
+export const DEFAULT_SERVICE_PEER_TIMEOUT_MS = 15_000;
+
+export class ServiceLivenessConnection {
   deadlineMs: number;
   nextProbeMs: number;
   outstandingProbe?: bigint;
-  ready: boolean;
-}
 
-export const DEFAULT_SERVICE_PROBE_INTERVAL_MS = 5_000;
-export const DEFAULT_SERVICE_PEER_TIMEOUT_MS = 15_000;
+  constructor(
+    readonly connectionId: string | bigint,
+    nowMs: number,
+    private readonly probeIntervalMs = DEFAULT_SERVICE_PROBE_INTERVAL_MS,
+    private readonly peerTimeoutMs = DEFAULT_SERVICE_PEER_TIMEOUT_MS
+  ) {
+    this.deadlineMs = nowMs + peerTimeoutMs;
+    this.nextProbeMs = nowMs;
+  }
+
+  readonly recordReceived = (nowMs = performance.now()): void => {
+    this.deadlineMs = Math.max(this.deadlineMs, nowMs + this.peerTimeoutMs);
+  };
+
+  isExpired(nowMs: number): boolean {
+    return nowMs >= this.deadlineMs;
+  }
+
+  tryGetProbe(nowMs: number, allocateProbeId: () => bigint): bigint | undefined {
+    if (nowMs < this.nextProbeMs) return undefined;
+    this.outstandingProbe ??= allocateProbeId();
+    do {
+      this.nextProbeMs += this.probeIntervalMs;
+    } while (this.nextProbeMs <= nowMs);
+    return this.outstandingProbe;
+  }
+
+  acknowledge(probeId: bigint, nowMs: number): boolean {
+    this.recordReceived(nowMs);
+    if (this.outstandingProbe !== probeId) return false;
+    this.outstandingProbe = undefined;
+    return true;
+  }
+}
 
 /** Uses monotonic milliseconds and never treats application traffic as a probe ACK. */
 export class ServiceLivenessRegistry {
-  private readonly peers = new Map<string, PeerState>();
+  private readonly peers = new Map<string, ServiceLivenessConnection>();
   private nextProbeId = 1n;
 
   constructor(
@@ -40,33 +72,31 @@ export class ServiceLivenessRegistry {
     }
   }
 
-  admit(nodeRoutingId: string, connectionId: string, nowMs: number): void {
+  admit(
+    nodeRoutingId: string,
+    connectionId: string | bigint,
+    nowMs: number
+  ): ServiceLivenessConnection {
     requireIdentity(nodeRoutingId, 'nodeRoutingId');
-    requireIdentity(connectionId, 'connectionId');
+    if (typeof connectionId === 'string') requireIdentity(connectionId, 'connectionId');
+    else if (connectionId === 0n) throw new TypeError('connectionId must be nonzero.');
     const current = this.peers.get(nodeRoutingId);
-    if (current?.connectionId === connectionId) return;
-    this.peers.set(nodeRoutingId, {
+    if (current?.connectionId === connectionId) return current;
+    const admitted = new ServiceLivenessConnection(
       connectionId,
-      deadlineMs: nowMs + this.peerTimeoutMs,
-      nextProbeMs: nowMs + this.probeIntervalMs,
-      ready: false
-    });
+      nowMs,
+      this.probeIntervalMs,
+      this.peerTimeoutMs
+    );
+    this.peers.set(nodeRoutingId, admitted);
+    return admitted;
   }
 
-  requestProbe(nodeRoutingId: string, connectionId: string, nowMs: number): boolean {
-    const current = this.peers.get(nodeRoutingId);
-    if (
-      current === undefined ||
-      current.connectionId !== connectionId ||
-      current.ready ||
-      current.outstandingProbe !== undefined
-    )
-      return false;
-    current.nextProbeMs = Math.min(current.nextProbeMs, nowMs);
-    return true;
+  recordReceived(connection: ServiceLivenessConnection, nowMs: number): void {
+    connection.recordReceived(nowMs);
   }
 
-  disconnect(nodeRoutingId: string, connectionId: string): boolean {
+  disconnect(nodeRoutingId: string, connectionId: string | bigint): boolean {
     const current = this.peers.get(nodeRoutingId);
     if (current === undefined || current.connectionId !== connectionId) return false;
     this.peers.delete(nodeRoutingId);
@@ -75,27 +105,18 @@ export class ServiceLivenessRegistry {
 
   acknowledge(
     nodeRoutingId: string,
-    connectionId: string,
+    connectionId: string | bigint,
     probeId: bigint,
     nowMs: number
   ): boolean {
     const current = this.peers.get(nodeRoutingId);
-    if (
-      current === undefined ||
-      current.connectionId !== connectionId ||
-      current.outstandingProbe !== probeId
-    ) {
-      return false;
-    }
-    current.outstandingProbe = undefined;
-    current.deadlineMs = nowMs + this.peerTimeoutMs;
-    current.ready = true;
-    return true;
+    if (current === undefined || current.connectionId !== connectionId) return false;
+    return current.acknowledge(probeId, nowMs);
   }
 
   acknowledgeProbe(
     nodeRoutingId: string,
-    connectionId: string,
+    connectionId: string | bigint,
     probeId: bigint
   ): ServiceProbe | undefined {
     const current = this.peers.get(nodeRoutingId);
@@ -109,21 +130,14 @@ export class ServiceLivenessRegistry {
     const probes: ServiceProbe[] = [];
     const timedOutNodes: string[] = [];
     for (const [nodeRoutingId, peer] of this.peers) {
-      if (peer.deadlineMs <= nowMs) {
+      if (peer.isExpired(nowMs)) {
         timedOutNodes.push(nodeRoutingId);
         this.peers.delete(nodeRoutingId);
         continue;
       }
-      if (peer.nextProbeMs > nowMs) continue;
-      peer.outstandingProbe ??= this.allocateProbeId();
-      probes.push({
-        nodeRoutingId,
-        connectionId: peer.connectionId,
-        probeId: peer.outstandingProbe
-      });
-      do {
-        peer.nextProbeMs += this.probeIntervalMs;
-      } while (peer.nextProbeMs <= nowMs);
+      const probeId = peer.tryGetProbe(nowMs, this.allocateProbeId);
+      if (probeId !== undefined)
+        probes.push({ nodeRoutingId, connectionId: peer.connectionId, probeId });
     }
     timedOutNodes.sort();
     return { probes, timedOutNodes };
@@ -133,16 +147,11 @@ export class ServiceLivenessRegistry {
     return this.peers.size;
   }
 
-  isReady(nodeRoutingId: string, connectionId: string): boolean {
-    const current = this.peers.get(nodeRoutingId);
-    return current?.connectionId === connectionId && current.ready;
-  }
-
-  private allocateProbeId(): bigint {
+  private readonly allocateProbeId = (): bigint => {
     const result = this.nextProbeId++;
     if (this.nextProbeId > UINT64_MAX) this.nextProbeId = 1n;
     return result;
-  }
+  };
 }
 
 function requireIdentity(value: string, field: string): void {

package/src/runtime/foundation/service-mailbox.ts

@@ -1,4 +1,5 @@
 import type { ApplicationJobRecordLease } from '../application-jobs/application-ingress-record-owner';
+import type { ReceiveRecord } from './service-runtime-contracts';
 
 export type ServiceMailboxDomain = 'application' | 'infrastructure';
 
@@ -23,6 +24,8 @@ export interface ServiceMailboxRecord {
   readonly stateful?: unknown;
   /** Host queue permit plus its ordinary Framework-owned ingress record. */
   readonly applicationJob?: ApplicationJobRecordLease;
+  /** Unregistered raw-pull terminal, retained by the existing mailbox owner. */
+  readonly completion?: ReceiveRecord;
 }
 
 export interface ServiceMailboxClaim {
@@ -85,7 +88,10 @@ export class ServiceMailbox {
   }
 
   tryEnqueue(record: ServiceMailboxRecord): boolean {
-    if (record.owner.length === 0 || record.parts.length === 0) {
+    if (
+      record.owner.length === 0 ||
+      (record.parts.length === 0 && record.completion === undefined)
+    ) {
       throw new TypeError('Mailbox records require an owner and retained payload.');
     }
     const target = this.domain(record.domain);

package/src/runtime/foundation/service-stateful-runtime.ts

@@ -90,6 +90,7 @@ import {
   type ServiceBoundSessionActorAuthority,
   type ServiceDirectSpotRouteFence,
   type ServiceInstanceActivationTarget,
+  type ServiceInstanceColdActivationTarget,
   type ServiceInstanceRouteFence,
   type ServiceMessageFollowRoute,
   type ServiceRetiredBoundSessionRouteFence,
@@ -313,7 +314,12 @@ export interface ServiceInstanceActivationAuthority {
  * operation completes; the binding callback itself never blocks.
  */
 export interface ServiceAsyncInstanceActivationAuthority {
-  read(target: ServiceInstanceActivationTarget): Promise<ServiceInstanceAuthorityRead>;
+  read(
+    target: ServiceInstanceActivationTarget,
+    instanceIntent?: true,
+    activation?: ServiceInstanceActivationRecoveryEnvelope,
+    acceptsLocal?: (route: ServiceInstanceRouteFence) => boolean
+  ): Promise<ServiceInstanceAuthorityRead>;
   reserve(
     activation: Omit<ServiceInstanceActivationRecoveryEnvelope, 'targetMeshName'>,
     signal?: AbortSignal
@@ -432,6 +438,7 @@ export class ServiceStatefulRuntime {
     Promise<{
       readonly spot: Pick<ServiceSpotState, 'ref'>;
       readonly route: ServiceInstanceRouteFence;
+      readonly stableType: string;
     }>
   >();
   private closed = false;
@@ -882,7 +889,7 @@ export class ServiceStatefulRuntime {
     const record = {
       kind: 'instanceSpot' as const,
       activation: 'missing' as const,
-      target,
+      target: { ...target, targetMeshName: envelope.targetMeshName },
       sourceNodeGeneration: envelope.sourceNodeGeneration,
       sourceNodeRid: envelope.sourceNodeRid,
       ...(envelope.sourceSpotId === undefined ? {} : { sourceSpotId: envelope.sourceSpotId }),
@@ -901,7 +908,7 @@ export class ServiceStatefulRuntime {
           sourceRoutingId: envelope.sourceNodeRid,
           parts: [
             encodeInstanceSpotActivationHeader(
-              target,
+              record.target,
               envelope.sourceNodeGeneration,
               envelope.sourceNodeRid,
               envelope.sourceSpotId,
@@ -1441,7 +1448,7 @@ export class ServiceStatefulRuntime {
   }
 
   async sendToMissingInstanceSpot(
-    target: ServiceInstanceActivationTarget,
+    target: ServiceInstanceColdActivationTarget,
     payload: ServiceApplicationPayload,
     deadlineUnixMs: bigint,
     sourceSpotId?: string,
@@ -1457,7 +1464,7 @@ export class ServiceStatefulRuntime {
   }
 
   async sendToMissingInstanceSpotFrame(
-    target: ServiceInstanceActivationTarget,
+    target: ServiceInstanceColdActivationTarget,
     payloadFrame: Buffer,
     deadlineUnixMs: bigint,
     sourceSpotId?: string,
@@ -1522,7 +1529,7 @@ export class ServiceStatefulRuntime {
   }
 
   requestToMissingInstanceSpot(
-    target: ServiceInstanceActivationTarget,
+    target: ServiceInstanceColdActivationTarget,
     payload: ServiceApplicationPayload,
     timeoutMs: number,
     sourceSpotId?: string,
@@ -1540,7 +1547,7 @@ export class ServiceStatefulRuntime {
   }
 
   requestToMissingInstanceSpotFrame(
-    target: ServiceInstanceActivationTarget,
+    target: ServiceInstanceColdActivationTarget,
     payloadFrame: Buffer,
     deadlineUnixMs: bigint,
     sourceSpotId?: string,
@@ -1559,7 +1566,7 @@ export class ServiceStatefulRuntime {
             this.nodeRid,
             sourceSpotId,
             'request',
-            { high: this.nodeGeneration, low: pending.id },
+            { high: ZLINK_NATIVE_CORRELATION_OPERATION_NAMESPACE, low: pending.id },
             deadlineUnixMs,
             pending.id,
             metadataFrame !== undefined
@@ -2520,27 +2527,24 @@ export class ServiceStatefulRuntime {
   ): Promise<void> {
     try {
       this.validateInstanceIngress(ingress, record);
-      const activation = await this.activateMissingInstanceAsync(
-        record,
-        payloadFrame,
-        metadataFrame
-      );
-      if (this.closed) return;
-      const terminal = this.activationTerminalCompletion(record.target, activation.route);
-      const admitted = this.enqueueActivatedInstanceSpot(
-        ingress,
-        record,
-        payloadFrame,
-        activation.spot,
-        localReply,
-        terminal.onTerminalCompletion,
-        metadataFrame,
-        terminal.onHandlerTurnStarted,
-        this.instanceApplicationTarget(record, activation.route.objectGeneration)
-      );
-      if (admitted !== 'application') {
-        throw new Error('Activated Instance message was not admitted to the local queue.');
-      }
+      await this.activateMissingInstanceAsync(record, payloadFrame, metadataFrame, (activation) => {
+        if (this.closed) return;
+        const terminal = this.activationTerminalCompletion(record.target, activation.route);
+        const admitted = this.enqueueActivatedInstanceSpot(
+          ingress,
+          record,
+          payloadFrame,
+          activation.spot,
+          localReply,
+          terminal.onTerminalCompletion,
+          metadataFrame,
+          terminal.onHandlerTurnStarted,
+          this.instanceApplicationTarget(record, activation.route.objectGeneration)
+        );
+        if (admitted !== 'application') {
+          throw new Error('Activated Instance message was not admitted to the local queue.');
+        }
+      });
     } catch (error) {
       this.finishMissingInstanceActivationFailure(ingress, record, error, localReply);
     }
@@ -2943,21 +2947,72 @@ export class ServiceStatefulRuntime {
     return activation.spot;
   }
 
+  private localReadyInstance(
+    target: ServiceInstanceActivationTarget,
+    route: ServiceInstanceRouteFence
+  ): ServiceSpotState | undefined {
+    const local = this.registry.spot(target.targetSpotId);
+    if (
+      local !== undefined &&
+      (local.kind !== 'instance' || local.stableType !== target.stableType)
+    ) {
+      throw new ServiceStaleGenerationError('spot', target.targetSpotId);
+    }
+    return local !== undefined &&
+      local.state === 'ready' &&
+      routeMatchesLocal(route, local, this.nodeRid, this.nodeGeneration)
+      ? local
+      : undefined;
+  }
+
   private activateMissingInstanceAsync(
     record: Extract<
       ServiceStatefulWireRecord,
       { readonly kind: 'instanceSpot'; readonly activation: 'missing' }
     >,
     payloadFrame: Buffer,
-    metadataFrame?: Buffer
-  ): Promise<{
-    readonly spot: Pick<ServiceSpotState, 'ref'>;
-    readonly route: ServiceInstanceRouteFence;
-  }> {
-    const key = instanceActivationKey(record);
+    metadataFrame: Buffer | undefined,
+    admit: (activation: {
+      readonly spot: Pick<ServiceSpotState, 'ref'>;
+      readonly route: ServiceInstanceRouteFence;
+    }) => void
+  ): Promise<unknown> {
+    const key = record.target.targetSpotId;
     const pending = this.pendingInstanceActivations.get(key);
-    if (pending !== undefined) return pending;
-    const activation = this.runMissingInstanceActivation(record, payloadFrame, metadataFrame);
+    const ready =
+      pending ??
+      this.runMissingInstanceActivation(record, payloadFrame, metadataFrame).then((activation) => ({
+        ...activation,
+        stableType: record.target.stableType
+      }));
+    const operation = ready.then(async (value) => {
+      if (pending !== undefined) {
+        await this.asyncInstanceAuthority?.read(
+          record.target,
+          undefined,
+          {
+            ...record,
+            targetMeshName: record.target.targetMeshName,
+            ...(metadataFrame === undefined ? {} : { metadataFrame }),
+            applicationPayloadFrame: payloadFrame
+          },
+          (route) => this.localReadyInstance(record.target, route) !== undefined
+        );
+      }
+      if (value.stableType !== record.target.stableType) {
+        throw new ZLinkFrameworkException(
+          ZLinkFrameworkErrorKind.TypeMismatch,
+          'Instance Spot type does not match.'
+        );
+      }
+      admit(value);
+      return value;
+    });
+    // The caller owns operation failure. Keep the activation available for later participants.
+    const activation = operation.then(
+      (value) => value,
+      () => ready
+    );
     this.pendingInstanceActivations.set(key, activation);
     const clear = () => {
       if (this.pendingInstanceActivations.get(key) === activation) {
@@ -2965,7 +3020,7 @@ export class ServiceStatefulRuntime {
       }
     };
     void activation.then(clear, clear);
-    return activation;
+    return operation;
   }
 
   private async runMissingInstanceActivation(
@@ -2992,8 +3047,19 @@ export class ServiceStatefulRuntime {
       throw new Error('Async Instance activation authority is not registered.');
     }
 
+    const activationRequest = {
+      ...record,
+      targetMeshName: record.target.targetMeshName,
+      ...(metadataFrame === undefined ? {} : { metadataFrame }),
+      applicationPayloadFrame: payloadFrame
+    };
+    const current = await authority.read(
+      target,
+      true,
+      activationRequest,
+      (route) => this.localReadyInstance(target, route) !== undefined
+    );
     const local = this.registry.spot(target.targetSpotId);
-    const current = await authority.read(target);
     if (
       current.kind === 'creating' &&
       current.authority?.allocation.state === 'active' &&
@@ -3043,17 +3109,9 @@ export class ServiceStatefulRuntime {
       if (current.route.targetNodeGeneration !== this.nodeGeneration) {
         throw new ServiceStaleGenerationError('spot', target.targetSpotId);
       }
-      if (
-        local !== undefined &&
-        (local.kind !== 'instance' || local.stableType !== target.stableType)
-      ) {
-        throw new ServiceStaleGenerationError('spot', target.targetSpotId);
-      }
-      if (
-        local !== undefined &&
-        routeMatchesLocal(current.route, local, this.nodeRid, this.nodeGeneration)
-      ) {
-        return { spot: local, route: current.route };
+      const readyLocal = this.localReadyInstance(target, current.route);
+      if (readyLocal !== undefined) {
+        return { spot: readyLocal, route: current.route };
       }
       return await this.reconcileReadyInstanceProjection(target, current.route, authority);
     }
@@ -3095,21 +3153,7 @@ export class ServiceStatefulRuntime {
     const deadline = operationDeadline(record.deadlineUnixMs);
     let reserved: ServiceInstanceAuthorityReserve;
     try {
-      reserved = await authority.reserve(
-        {
-          target,
-          sourceNodeRid: record.sourceNodeRid,
-          sourceNodeGeneration: record.sourceNodeGeneration,
-          ...(record.sourceSpotId === undefined ? {} : { sourceSpotId: record.sourceSpotId }),
-          operationKind: record.operationKind,
-          operation: record.operation,
-          ...(record.replyRouteId === undefined ? {} : { replyRouteId: record.replyRouteId }),
-          deadlineUnixMs: record.deadlineUnixMs,
-          ...(metadataFrame === undefined ? {} : { metadataFrame }),
-          applicationPayloadFrame: payloadFrame
-        },
-        deadline.signal
-      );
+      reserved = await authority.reserve(activationRequest, deadline.signal);
     } finally {
       deadline.close();
     }
@@ -5817,18 +5861,6 @@ function instanceOperationKey(
   );
 }
 
-function instanceActivationKey(
-  record: Extract<
-    ServiceStatefulWireRecord,
-    { readonly kind: 'instanceSpot'; readonly activation: 'missing' }
-  >
-): string {
-  return (
-    `${record.target.targetNodeRid}\0${record.target.targetNodeGeneration}\0` +
-    `${record.target.targetSpotId}\0${record.target.stableType}`
-  );
-}
-
 function subscriptionKey(channelName: string, topicFilter: string): string {
   if (channelName.length === 0 || topicFilter.length === 0) {
     throw new TypeError('Channel name and topic filter must be non-empty.');