@zlink-systems/framework 0.29.1 → 0.30.1

package/src/runtime/foundation/service-stateful-runtime.ts

@@ -10,7 +10,7 @@ import {
   ZLinkDispatchMessageKind,
   ZLinkRuntimeMessageFlowOutcome as ZLinkMessageFlowOutcome
 } from '../../contracts/Dispatch/ZLinkDispatchOptions';
-import { awaitWithAbort } from '../abort';
+import { requestDurableService } from './durable-service-request';
 import { shouldCompactBackingArray } from '../admission';
 import { RequestResult, SubmitResult, isZLinkBackendResultError } from '../backend/runtime-values';
 import type { ZLinkDispatchErrorReporter } from '../channels/dispatch-error-reporter';
@@ -90,6 +90,7 @@ import {
   type ServiceBoundSessionActorAuthority,
   type ServiceDirectSpotRouteFence,
   type ServiceInstanceActivationTarget,
+  type ServiceInstanceColdActivationTarget,
   type ServiceInstanceRouteFence,
   type ServiceMessageFollowRoute,
   type ServiceRetiredBoundSessionRouteFence,
@@ -111,7 +112,6 @@ import {
   encodeApplicationPayload,
   type ServiceApplicationPayload
 } from './service-wire-m6a-codec';
-const STATEFUL_OPERATION_RETRY_TICK_MS = 20;
 
 const MAX_NODE_TIMER_DELAY_MS = 2_147_483_647;
 
@@ -313,7 +313,12 @@ export interface ServiceInstanceActivationAuthority {
  * operation completes; the binding callback itself never blocks.
  */
 export interface ServiceAsyncInstanceActivationAuthority {
-  read(target: ServiceInstanceActivationTarget): Promise<ServiceInstanceAuthorityRead>;
+  read(
+    target: ServiceInstanceActivationTarget,
+    instanceIntent?: true,
+    activation?: ServiceInstanceActivationRecoveryEnvelope,
+    acceptsLocal?: (route: ServiceInstanceRouteFence) => boolean
+  ): Promise<ServiceInstanceAuthorityRead>;
   reserve(
     activation: Omit<ServiceInstanceActivationRecoveryEnvelope, 'targetMeshName'>,
     signal?: AbortSignal
@@ -448,7 +453,7 @@ export class ServiceStatefulRuntime {
     );
     this.registry = new ServiceStatefulRegistry(nodeRid, nodeGeneration);
     this.registry.createEntrySpot(nodeRid);
-    raw.setServiceIngress((record) => this.ingress(record));
+    raw.setServiceIngress((record) => this.ingress(record), acceptsStatefulIngress);
   }
 
   setMailboxDropHandler(
@@ -883,7 +888,7 @@ export class ServiceStatefulRuntime {
     const record = {
       kind: 'instanceSpot' as const,
       activation: 'missing' as const,
-      target,
+      target: { ...target, targetMeshName: envelope.targetMeshName },
       sourceNodeGeneration: envelope.sourceNodeGeneration,
       sourceNodeRid: envelope.sourceNodeRid,
       ...(envelope.sourceSpotId === undefined ? {} : { sourceSpotId: envelope.sourceSpotId }),
@@ -902,7 +907,7 @@ export class ServiceStatefulRuntime {
           sourceRoutingId: envelope.sourceNodeRid,
           parts: [
             encodeInstanceSpotActivationHeader(
-              target,
+              record.target,
               envelope.sourceNodeGeneration,
               envelope.sourceNodeRid,
               envelope.sourceSpotId,
@@ -1320,7 +1325,8 @@ export class ServiceStatefulRuntime {
     sourceSpotId: string,
     requested: ServiceDirectSpotRouteFence,
     payload: ServiceApplicationPayload,
-    timeoutMs: number
+    timeoutMs: number,
+    onAdmitted?: () => void
   ): ServiceStatefulPendingOperation {
     const pending = this.operations.register(timeoutMs);
     const target = this.acceptSpotAuthority(requested);
@@ -1340,7 +1346,9 @@ export class ServiceStatefulRuntime {
       target.targetNodeRid,
       [header, encodeApplicationPayload(payload)],
       timeoutMs,
-      'spotRequest'
+      'spotRequest',
+      undefined,
+      onAdmitted
     );
     return pending;
   }
@@ -1378,7 +1386,8 @@ export class ServiceStatefulRuntime {
     payload: ServiceApplicationPayload,
     timeoutMs: number,
     sourceActor?: ServiceActorRef,
-    boundSession?: { readonly sessionRid: string; readonly bindingGeneration: bigint }
+    boundSession?: { readonly sessionRid: string; readonly bindingGeneration: bigint },
+    onAdmitted?: () => void
   ): ServiceStatefulPendingOperation {
     const pending = this.operations.register(timeoutMs);
     const route = this.tryActorFence(target);
@@ -1407,7 +1416,9 @@ export class ServiceStatefulRuntime {
       target.nodeRid,
       [header, encodeApplicationPayload(payload)],
       timeoutMs,
-      'actorRequest'
+      'actorRequest',
+      undefined,
+      onAdmitted
     );
     return pending;
   }
@@ -1442,7 +1453,7 @@ export class ServiceStatefulRuntime {
   }
 
   async sendToMissingInstanceSpot(
-    target: ServiceInstanceActivationTarget,
+    target: ServiceInstanceColdActivationTarget,
     payload: ServiceApplicationPayload,
     deadlineUnixMs: bigint,
     sourceSpotId?: string,
@@ -1458,7 +1469,7 @@ export class ServiceStatefulRuntime {
   }
 
   async sendToMissingInstanceSpotFrame(
-    target: ServiceInstanceActivationTarget,
+    target: ServiceInstanceColdActivationTarget,
     payloadFrame: Buffer,
     deadlineUnixMs: bigint,
     sourceSpotId?: string,
@@ -1493,7 +1504,8 @@ export class ServiceStatefulRuntime {
     timeoutMs: number,
     sourceSpotId?: string,
     metadataFrame?: Uint8Array,
-    instanceIntent = false
+    instanceIntent = false,
+    onAdmitted?: () => void
   ): ServiceStatefulPendingOperation {
     const pending = this.operations.register(timeoutMs);
     this.submitRequest(
@@ -1517,13 +1529,15 @@ export class ServiceStatefulRuntime {
         metadataFrame
       ),
       timeoutMs,
-      'instanceSpotRequest'
+      'instanceSpotRequest',
+      undefined,
+      onAdmitted
     );
     return pending;
   }
 
   requestToMissingInstanceSpot(
-    target: ServiceInstanceActivationTarget,
+    target: ServiceInstanceColdActivationTarget,
     payload: ServiceApplicationPayload,
     timeoutMs: number,
     sourceSpotId?: string,
@@ -1541,11 +1555,12 @@ export class ServiceStatefulRuntime {
   }
 
   requestToMissingInstanceSpotFrame(
-    target: ServiceInstanceActivationTarget,
+    target: ServiceInstanceColdActivationTarget,
     payloadFrame: Buffer,
     deadlineUnixMs: bigint,
     sourceSpotId?: string,
-    metadataFrame?: Uint8Array
+    metadataFrame?: Uint8Array,
+    onAdmitted?: () => void
   ): ServiceStatefulPendingOperation {
     const timeoutMs = remainingDeadlineMs(deadlineUnixMs);
     const pending = this.operations.register(timeoutMs);
@@ -1570,7 +1585,9 @@ export class ServiceStatefulRuntime {
         metadataFrame
       ),
       timeoutMs,
-      'instanceSpotRequest'
+      'instanceSpotRequest',
+      undefined,
+      onAdmitted
     );
     return pending;
   }
@@ -2023,29 +2040,6 @@ export class ServiceStatefulRuntime {
   private async ingress(
     record: RawServiceIngressRecord
   ): Promise<RawServicePumpResult | undefined> {
-    // These frozen relocation controls overlap the legacy stateful command
-    // number range (30-34). They are bare infrastructure records and must
-    // fall through to RawServiceMeshRuntime's relocation ingress instead of
-    // being decoded as stateful actor/spot work.
-    if (
-      record.command === M6bServiceWireCommand.relocationReady ||
-      record.command === M6bServiceWireCommand.relocationData ||
-      record.command === M6bServiceWireCommand.replyRelay ||
-      record.command === M6bServiceWireCommand.relocationCutover
-    ) {
-      return undefined;
-    }
-    if (
-      record.command < M6bServiceWireCommand.spotSend ||
-      (record.command > M6bServiceWireCommand.instanceSpot &&
-        record.command !== M6bServiceWireCommand.userSpotCreate &&
-        record.command !== M6bServiceWireCommand.userSpotClose &&
-        record.command !== M6bServiceWireCommand.actorCreate &&
-        record.command !== M6bServiceWireCommand.messageFollow &&
-        record.command !== M6bServiceWireCommand.boundSessionReplaced)
-    ) {
-      return undefined;
-    }
     let decoded: ServiceStatefulWireRecord;
     let canonicalActorJoin: ActorJoin28 | undefined;
     try {
@@ -2365,6 +2359,7 @@ export class ServiceStatefulRuntime {
         localReply,
         metadataFrame
       ).finally(() => ownedIngress.applicationJobOwner?.close());
+      ingress.onAdmitted?.();
       return 'infrastructure';
     }
     if (
@@ -2378,6 +2373,7 @@ export class ServiceStatefulRuntime {
         payloadFrame,
         metadataFrame
       ).finally(() => ownedIngress.applicationJobOwner?.close());
+      ingress.onAdmitted?.();
       return 'infrastructure';
     }
     const spot = this.requireInstanceActivation(ingress, record);
@@ -2941,6 +2937,24 @@ export class ServiceStatefulRuntime {
     return activation.spot;
   }
 
+  private localReadyInstance(
+    target: ServiceInstanceActivationTarget,
+    route: ServiceInstanceRouteFence
+  ): ServiceSpotState | undefined {
+    const local = this.registry.spot(target.targetSpotId);
+    if (
+      local !== undefined &&
+      (local.kind !== 'instance' || local.stableType !== target.stableType)
+    ) {
+      throw new ServiceStaleGenerationError('spot', target.targetSpotId);
+    }
+    return local !== undefined &&
+      local.state === 'ready' &&
+      routeMatchesLocal(route, local, this.nodeRid, this.nodeGeneration)
+      ? local
+      : undefined;
+  }
+
   private activateMissingInstanceAsync(
     record: Extract<
       ServiceStatefulWireRecord,
@@ -2961,7 +2975,20 @@ export class ServiceStatefulRuntime {
         ...activation,
         stableType: record.target.stableType
       }));
-    const operation = ready.then((value) => {
+    const operation = ready.then(async (value) => {
+      if (pending !== undefined) {
+        await this.asyncInstanceAuthority?.read(
+          record.target,
+          undefined,
+          {
+            ...record,
+            targetMeshName: record.target.targetMeshName,
+            ...(metadataFrame === undefined ? {} : { metadataFrame }),
+            applicationPayloadFrame: payloadFrame
+          },
+          (route) => this.localReadyInstance(record.target, route) !== undefined
+        );
+      }
       if (value.stableType !== record.target.stableType) {
         throw new ZLinkFrameworkException(
           ZLinkFrameworkErrorKind.TypeMismatch,
@@ -3010,8 +3037,19 @@ export class ServiceStatefulRuntime {
       throw new Error('Async Instance activation authority is not registered.');
     }
 
+    const activationRequest = {
+      ...record,
+      targetMeshName: record.target.targetMeshName,
+      ...(metadataFrame === undefined ? {} : { metadataFrame }),
+      applicationPayloadFrame: payloadFrame
+    };
+    const current = await authority.read(
+      target,
+      true,
+      activationRequest,
+      (route) => this.localReadyInstance(target, route) !== undefined
+    );
     const local = this.registry.spot(target.targetSpotId);
-    const current = await authority.read(target);
     if (
       current.kind === 'creating' &&
       current.authority?.allocation.state === 'active' &&
@@ -3061,17 +3099,9 @@ export class ServiceStatefulRuntime {
       if (current.route.targetNodeGeneration !== this.nodeGeneration) {
         throw new ServiceStaleGenerationError('spot', target.targetSpotId);
       }
-      if (
-        local !== undefined &&
-        (local.kind !== 'instance' || local.stableType !== target.stableType)
-      ) {
-        throw new ServiceStaleGenerationError('spot', target.targetSpotId);
-      }
-      if (
-        local !== undefined &&
-        routeMatchesLocal(current.route, local, this.nodeRid, this.nodeGeneration)
-      ) {
-        return { spot: local, route: current.route };
+      const readyLocal = this.localReadyInstance(target, current.route);
+      if (readyLocal !== undefined) {
+        return { spot: readyLocal, route: current.route };
       }
       return await this.reconcileReadyInstanceProjection(target, current.route, authority);
     }
@@ -3113,21 +3143,7 @@ export class ServiceStatefulRuntime {
     const deadline = operationDeadline(record.deadlineUnixMs);
     let reserved: ServiceInstanceAuthorityReserve;
     try {
-      reserved = await authority.reserve(
-        {
-          target,
-          sourceNodeRid: record.sourceNodeRid,
-          sourceNodeGeneration: record.sourceNodeGeneration,
-          ...(record.sourceSpotId === undefined ? {} : { sourceSpotId: record.sourceSpotId }),
-          operationKind: record.operationKind,
-          operation: record.operation,
-          ...(record.replyRouteId === undefined ? {} : { replyRouteId: record.replyRouteId }),
-          deadlineUnixMs: record.deadlineUnixMs,
-          ...(metadataFrame === undefined ? {} : { metadataFrame }),
-          applicationPayloadFrame: payloadFrame
-        },
-        deadline.signal
-      );
+      reserved = await authority.reserve(activationRequest, deadline.signal);
     } finally {
       deadline.close();
     }
@@ -3795,7 +3811,10 @@ export class ServiceStatefulRuntime {
       stateful,
       applicationJob
     });
-    if (accepted) return domain;
+    if (accepted) {
+      ingress.onAdmitted?.();
+      return domain;
+    }
     applicationJob.close();
     if (stateful.reply !== undefined) {
       const result = failure(
@@ -4186,64 +4205,15 @@ export class ServiceStatefulRuntime {
     operationKind: 'userSpotCreate' | 'userSpotClose' | 'actorCreate' | 'streamBind' | 'actorJoin',
     deadlineMs: number
   ): Promise<readonly Buffer[]> {
-    const stop = new AbortController();
-    const unobserve = this.raw.observePeerConnectionIntentRemoved((nodeRoutingId) => {
-      if (nodeRoutingId === targetNodeRid) {
-        stop.abort(
-          createInternalFrameworkException(
-            ZLinkFrameworkInternalErrorKind.RouteNotConnected,
-            `${operationKind} target node lifecycle ended.`,
-            true
-          )
-        );
-      }
-    });
-    let wasAdmitted = false;
-    try {
-      // Only durable lifecycle operations replay these frozen frames. Received
-      // envelopes leave this loop before decoding, including failure terminals.
-      for (;;) {
-        this.requireOpen();
-        stop.signal.throwIfAborted();
-        const remainingMs = Math.ceil(deadlineMs - performance.now());
-        if (remainingMs <= 0) throw durableOperationExhausted(operationKind, wasAdmitted);
-        try {
-          return await awaitWithAbort(
-            targetNodeRid === this.nodeRid
-              ? this.requestLocalInfrastructure(parts[0]!, correlation, remainingMs)
-              : this.raw.requestService(targetNodeRid, parts, remainingMs),
-            stop.signal
-          );
-        } catch (error) {
-          stop.signal.throwIfAborted();
-          if (!durableRequestCanReplay(error)) throw error;
-          if (durableRequestWasAdmitted(error)) wasAdmitted = true;
-          const retryDelayMs = Math.min(
-            STATEFUL_OPERATION_RETRY_TICK_MS,
-            deadlineMs - performance.now()
-          );
-          if (retryDelayMs <= 0) {
-            throw durableOperationExhausted(operationKind, wasAdmitted, error);
-          }
-          let timer: ReturnType<typeof setTimeout> | undefined;
-          try {
-            await awaitWithAbort(
-              new Promise<void>((resolve) => {
-                timer = setTimeout(resolve, retryDelayMs);
-              }),
-              stop.signal
-            );
-          } finally {
-            if (timer !== undefined) clearTimeout(timer);
-          }
-        }
-      }
-    } catch (error) {
-      stop.signal.throwIfAborted();
-      throw error;
-    } finally {
-      unobserve();
+    this.requireOpen();
+    if (targetNodeRid === this.nodeRid) {
+      return this.requestLocalInfrastructure(
+        parts[0]!,
+        correlation,
+        Math.ceil(deadlineMs - performance.now())
+      );
     }
+    return requestDurableService(this.raw, targetNodeRid, parts, deadlineMs, operationKind);
   }
 
   private requestLocalInfrastructure(
@@ -4367,7 +4337,8 @@ export class ServiceStatefulRuntime {
       | 'streamBind'
       | 'streamUnbind'
       | 'instanceSpotRequest',
-    actor?: ServiceActorRef
+    actor?: ServiceActorRef,
+    onAdmitted?: () => void
   ): void {
     if (targetNodeRid === this.nodeRid) {
       const deadlineUnixMs = BigInt(Date.now() + Math.max(0, timeoutMs));
@@ -4385,6 +4356,7 @@ export class ServiceStatefulRuntime {
             flags: parts[0]![SERVICE_WIRE_FLAGS_OFFSET]!,
             sourceRoutingId: this.nodeRid,
             requestSequence: pending.id,
+            onAdmitted,
             parts,
             applicationJobOwner
           };
@@ -4403,7 +4375,7 @@ export class ServiceStatefulRuntime {
       })().catch((error) => this.operations.reply(pending.id, failure(error)));
       return;
     }
-    void this.raw.requestService(targetNodeRid, parts, timeoutMs).then(
+    void this.raw.requestService(targetNodeRid, parts, timeoutMs, onAdmitted).then(
       (reply) => this.completeRemoteReply(pending, targetNodeRid, operationKind, actor, reply),
       (error) => this.operations.fail(pending.id, error)
     );
@@ -5388,37 +5360,6 @@ function operationReplayExpired(replayExpiresAtMs: number): boolean {
   return performance.now() > replayExpiresAtMs;
 }
 
-function durableRequestWasAdmitted(error: unknown): boolean {
-  // Request NotConnected includes Core handover of an admitted request. The
-  // submit/request phase, rather than the result number, owns this distinction.
-  return isZLinkBackendResultError(error) && error.operation === 'request';
-}
-
-function durableOperationExhausted(
-  operationKind: 'userSpotCreate' | 'userSpotClose' | 'actorCreate' | 'streamBind' | 'actorJoin',
-  wasAdmitted: boolean,
-  cause?: unknown
-): ZLinkFrameworkException {
-  return createInternalFrameworkException(
-    wasAdmitted
-      ? ZLinkFrameworkInternalErrorKind.DeadlineExceeded
-      : ZLinkFrameworkInternalErrorKind.RouteNotConnected,
-    `${operationKind} exhausted its end-to-end deadline.`,
-    true,
-    cause
-  );
-}
-
-function durableRequestCanReplay(error: unknown): boolean {
-  if (!isZLinkBackendResultError(error)) return false;
-  return error.operation === 'request'
-    ? error.result === RequestResult.NotConnected || error.result === RequestResult.TimedOut
-    : error.result === SubmitResult.NotConnected ||
-        error.result === SubmitResult.NotFound ||
-        error.result === SubmitResult.NotAdmitted ||
-        error.result === SubmitResult.Backpressured;
-}
-
 function remainingDeadlineMs(deadlineUnixMs: bigint): number {
   const remainingMs = deadlineUnixMs - BigInt(Date.now());
   if (remainingMs <= 0n) {
@@ -5911,3 +5852,30 @@ export function statefulMailboxData(
 ): ServiceStatefulMailboxData | undefined {
   return record.stateful as ServiceStatefulMailboxData | undefined;
 }
+
+function acceptsStatefulIngress(command: number): boolean {
+  // These frozen relocation controls overlap the legacy stateful command
+  // number range (30-34). They are bare infrastructure records and must
+  // fall through to RawServiceMeshRuntime's relocation ingress instead of
+  // being decoded as stateful actor/spot work.
+  if (
+    command === M6bServiceWireCommand.relocationReady ||
+    command === M6bServiceWireCommand.relocationData ||
+    command === M6bServiceWireCommand.replyRelay ||
+    command === M6bServiceWireCommand.relocationCutover
+  ) {
+    return false;
+  }
+  if (
+    command < M6bServiceWireCommand.spotSend ||
+    (command > M6bServiceWireCommand.instanceSpot &&
+      command !== M6bServiceWireCommand.userSpotCreate &&
+      command !== M6bServiceWireCommand.userSpotClose &&
+      command !== M6bServiceWireCommand.actorCreate &&
+      command !== M6bServiceWireCommand.messageFollow &&
+      command !== M6bServiceWireCommand.boundSessionReplaced)
+  ) {
+    return false;
+  }
+  return true;
+}