@zlink-systems/framework 0.26.0 → 0.28.0

package/src/runtime/spots/spot-routed-spot-packet-dispatch.ts

@@ -1,3 +1,4 @@
+import { dispatchReasonFromError } from '../diagnostics/dispatch-error-details';
 import type {
   RoutingId,
   Type,
@@ -7,7 +8,7 @@ import type {
 } from '../../contracts';
 import type { ZLinkProviderResolver } from '../../contracts/Common/ZLinkProviderResolver';
 import type { ZLinkApplicationWorkClaim } from '../admission';
-import { ZLinkFrameworkException, zlinkMessageMetadata } from '../../contracts';
+import { zlinkMessageMetadata } from '../../contracts';
 import {
   ZLinkRuntimeDispatchErrorAction as ZLinkDispatchErrorAction,
   ZLinkRuntimeDispatchErrorReason as ZLinkDispatchErrorReason,
@@ -18,8 +19,7 @@ import type { ZLinkDispatchErrorReporter } from '../channels';
 import { ZLinkConfigurationException } from '../configuration';
 import {
   ZLinkFrameworkInternalErrorKind,
-  createInternalFrameworkException,
-  internalFrameworkErrorKind
+  createInternalFrameworkException
 } from '../framework-errors-internal';
 import { resolveLifecycleHandler } from '../handlers/handler-instance-scope';
 import type { ZLinkSpotHandlerRegistration } from './spot-handler-registry';
@@ -31,6 +31,7 @@ import {
 } from '../application-jobs/application-job-queue-scope';
 
 interface ZLinkRoutedSpotPacketActivation {
+  readonly objectGeneration?: bigint;
   readonly meshName?: string;
   readonly spotId: RoutingId;
   readonly spot: ZLinkSpot;
@@ -55,6 +56,9 @@ interface ZLinkRoutedSpotPacketContext {
   readonly signal?: AbortSignal;
   /** Wait until a recovered activation has actually entered its first handler. */
   readonly awaitFirstHandlerTurn?: boolean;
+  readonly onOneWayError?: (error: unknown) => void;
+  /** Retains the caller's Instance address intent through an incarnation boundary. */
+  readonly activationRecord?: import('../foundation/service-runtime-contracts').ReceiveRecord;
 }
 
 export class ZLinkRoutedSpotPacketDispatch {
@@ -143,14 +147,52 @@ export class ZLinkRoutedSpotPacketDispatch {
       activation.meshName === undefined
         ? undefined
         : this.options.claimApplicationWork?.(activation.meshName);
-    const runHandler = async () => {
+    const originalRecord = context.activationRecord?.activationRecord;
+    const instanceIntent =
+      originalRecord?.activation === 'missing' ||
+      (originalRecord?.activation === 'ready' && originalRecord.instanceIntent);
+    const runHandler = async (failure?: unknown) => {
       try {
+        if (failure !== undefined) throw failure;
+        const current = this.options.resolveActivation(spotId);
+        if (
+          current === undefined ||
+          (!instanceIntent &&
+            (current !== activation ||
+              (originalRecord?.activation === 'ready' &&
+                originalRecord.route.objectGeneration !== current.objectGeneration)))
+        ) {
+          throw createInternalFrameworkException(
+            ZLinkFrameworkInternalErrorKind.RequestTargetNotFound,
+            `Spot '${String(spotId)}' is not active.`
+          );
+        }
+        if (
+          context.activationRecord?.deadlineUnixMs !== undefined &&
+          context.activationRecord.deadlineUnixMs < BigInt(Date.now())
+        ) {
+          throw createInternalFrameworkException(
+            ZLinkFrameworkInternalErrorKind.DeadlineExceeded,
+            `Spot '${String(spotId)}' request deadline has expired.`
+          );
+        }
+        const currentRegistrations =
+          current === activation
+            ? registrations
+            : current.handlers
+                .snapshot()
+                .filter(
+                  (registration) =>
+                    registration.kind === 'packet' &&
+                    (registration.packetName ?? registration.handlerType.name) ===
+                      (packetName ?? '')
+                );
         // Decode only after the Spot has acquired both application admission
         // and its execution authority.
         const payload = decodePayload();
-        for (const registration of registrations) {
+        for (const registration of currentRegistrations) {
           const handler = await resolveLifecycleHandler(
-            activation.spot,
+            current.spot,
             registration.handlerType as Type<
               | ZLinkSpotPacketHandler<ZLinkSpot, unknown>
               | ZLinkSpotRequestHandler<ZLinkSpot, unknown, unknown>
@@ -159,8 +201,9 @@ export class ZLinkRoutedSpotPacketDispatch {
           );
           resolveFirstHandlerTurn?.();
           resolveFirstHandlerTurn = undefined;
+          context.activationRecord?.onHandlerTurnStarted?.();
           releaseApplicationJobPermitBeforeHandler();
-          response = await handler.handle(activation.spot, payload, {
+          response = await handler.handle(current.spot, payload, {
             channelName: context.channelName,
             contentType: context.contentType,
             packetName: packetName!,
@@ -186,17 +229,33 @@ export class ZLinkRoutedSpotPacketDispatch {
                 applicationClaim?.close();
               }
             },
-            (error) => this.reportFailure(spotId, packetName, context, false, error),
+            (error) => {
+              if (context.onOneWayError === undefined) {
+                this.reportFailure(spotId, packetName, context, false, error);
+              } else if (resolveFirstHandlerTurn === undefined) {
+                context.onOneWayError(error);
+              }
+            },
             context.workOptions,
-            { signal: context.signal }
+            { signal: context.signal },
+            instanceIntent
+              ? async (failure?: unknown) => {
+                  try {
+                    await runHandler(failure);
+                  } finally {
+                    detachedApplicationPermit?.releaseAfterInternalProcessing();
+                    applicationClaim?.close();
+                  }
+                }
+              : undefined
           );
-          await firstHandlerTurn;
         } catch (error) {
           detached = false;
           detachedApplicationPermit?.releaseAfterInternalProcessing();
           rejectFirstHandlerTurn?.(error);
           throw error;
         }
+        await firstHandlerTurn;
       } else {
         if (activation.serial.isCurrentTurn) {
           throw createInternalFrameworkException(
@@ -204,10 +263,15 @@ export class ZLinkRoutedSpotPacketDispatch {
             `Spot '${spotId}' cannot await a request to its current serial turn.`
           );
         }
-        await activation.serial.execute(runHandler, context.workOptions);
+        await activation.serial.execute(
+          runHandler,
+          context.workOptions,
+          instanceIntent ? runHandler : undefined
+        );
       }
     } catch (error) {
-      this.reportFailure(spotId, packetName, context, returnResponse, error);
+      if (context.activationRecord?.activationRecord === undefined)
+        this.reportFailure(spotId, packetName, context, returnResponse, error);
       throw error;
     } finally {
       if (!detached) applicationClaim?.close();
@@ -222,18 +286,17 @@ export class ZLinkRoutedSpotPacketDispatch {
     returnResponse: boolean,
     error: unknown
   ): void {
-    const frameworkErrorKind =
-      error instanceof ZLinkFrameworkException ? internalFrameworkErrorKind(error) : undefined;
-    this.options.dispatchErrors?.report({
-      surface: ZLinkDispatchErrorSurface.SpotRoute,
+    const reporter = this.options.dispatchErrors;
+    if (reporter?.captureEnabled() !== true) return;
+    reporter.report({
+      surface:
+        context.activationRecord?.activationRecord?.kind === 'instanceSpot'
+          ? ZLinkDispatchErrorSurface.InstanceSpot
+          : ZLinkDispatchErrorSurface.SpotRoute,
       messageKind: returnResponse
         ? ZLinkDispatchMessageKind.Request
         : ZLinkDispatchMessageKind.Send,
-      reason:
-        frameworkErrorKind === ZLinkFrameworkInternalErrorKind.WorkerQueueFull ||
-        frameworkErrorKind === ZLinkFrameworkInternalErrorKind.DeadlineExceeded
-          ? ZLinkDispatchErrorReason.Backpressure
-          : ZLinkDispatchErrorReason.HandlerException,
+      reason: dispatchReasonFromError(error),
       action: returnResponse ? ZLinkDispatchErrorAction.FailCaller : ZLinkDispatchErrorAction.Drop,
       packetName,
       channelName: context.channelName,
@@ -245,9 +308,19 @@ export class ZLinkRoutedSpotPacketDispatch {
   private reportMissing(
     spotId: RoutingId,
     packetName: string | undefined,
-    context: { readonly channelName: string },
+    context: ZLinkRoutedSpotPacketContext,
     returnResponse: boolean
   ): void {
+    if (context.activationRecord?.activationRecord !== undefined) {
+      if (!returnResponse && context.onOneWayError !== undefined)
+        context.onOneWayError(
+          createInternalFrameworkException(
+            ZLinkFrameworkInternalErrorKind.HandlerNotFound,
+            `SPOT route handler not found: ${packetName}`
+          )
+        );
+      return;
+    }
     this.options.dispatchErrors?.report({
       surface: ZLinkDispatchErrorSurface.SpotRoute,
       messageKind: returnResponse

package/src/runtime/spots/spot-runtime-ports.ts

@@ -46,7 +46,7 @@ export interface ZLinkSpotActorTransferRuntime {
     rawReply: Uint8Array,
     replyContentType?: string
   ): ZLinkDeferredJoinCompletion;
-  deliverDeferredJoinAccepted(
+  deliverDeferredJoinCompletion(
     completion: ZLinkDeferredJoinCompletion,
     actor: ZLinkActor,
     actorRef: ActorRef,
@@ -72,15 +72,6 @@ export interface ZLinkSpotActorTransferRuntime {
     deadlineAtMs: number,
     signal?: AbortSignal
   ): Promise<import('../locations/internal-location-contracts').ZLinkAuthoritySnapshot>;
-  rememberRoutedActorTransferTarget(
-    actorId: string,
-    target: ZLinkRemoteBoundSessionTarget | undefined
-  ): void;
-  claimNativeActorLocation(
-    actor: ZLinkActor,
-    spotId: RoutingId,
-    spotMeshName: string
-  ): Promise<ZLinkNativeActorJoinSnapshot>;
   claimRoutedActorLocation(
     actor: ZLinkActor,
     spotId: RoutingId,
@@ -90,28 +81,19 @@ export interface ZLinkSpotActorTransferRuntime {
       readonly membershipEpoch: bigint;
     }
   ): Promise<void>;
-  publishRoutedActorOwnership(actor: ZLinkActor): Promise<void>;
+  publishRoutedActorOwnership(
+    actor: ZLinkActor,
+    sealedSession: ZLinkRemoteBoundSessionTarget | undefined
+  ): Promise<void>;
   openRoutedActorSession(actor: ZLinkActor): Promise<void>;
   bindRoutedActorRef(actor: ZLinkActor, actorRef: ActorRef): void;
   commitRoutedActor(actor: ZLinkActor, spotId: RoutingId, spot: ZLinkSpot): void;
   clearRoutedActor(actor: ZLinkActor): void;
-  rollbackNativeActorJoin(actor: ZLinkActor, snapshot: ZLinkNativeActorJoinSnapshot): Promise<void>;
   rollbackRoutedActor(actor: ZLinkActor, signal?: AbortSignal): Promise<void>;
   notifyCoreSourceLeave(actor: ZLinkActor, callback: () => Promise<void>): Promise<void>;
   actorEntryNodeRid(actor: ZLinkActor): RoutingId | undefined;
 }
 
-export interface ZLinkNativeActorJoinSnapshot {
-  readonly spotId?: RoutingId;
-  readonly spot?: ZLinkSpot;
-  readonly locationSpotId?: RoutingId;
-  readonly spotMeshName?: string;
-  readonly actorRef?: ActorRef;
-  readonly spotGeneration?: bigint;
-  readonly membershipEpoch?: bigint;
-  readonly ownerNodeGeneration?: bigint;
-}
-
 export interface ZLinkSpotBoundSessionRuntime {
   receiveRoutedBoundSession(
     actorId: string,
@@ -140,10 +122,6 @@ export interface ZLinkSpotBoundSessionRuntime {
     actorPacketTarget?: unknown,
     signal?: AbortSignal
   ): Promise<void>;
-  rememberRemoteBoundSessionTarget(
-    actorId: string,
-    target: ZLinkRemoteBoundSessionTarget | undefined
-  ): void;
   resolveRemoteBoundSessionTarget(
     sourceNodeRid: RoutingId,
     sourceSessionRid: RoutingId

package/src/runtime/spots/spot-serial-executor.ts

@@ -108,7 +108,7 @@ export class ZLinkSpotSerialExecutor {
    */
   close(): Promise<void> {
     const children = this.closeChildren();
-    return Promise.all([children, this.spotSerial.close()]).then(() => undefined);
+    return Promise.all([children, this.spotSerial.whenIdle()]).then(() => undefined);
   }
 
   /**
@@ -124,7 +124,7 @@ export class ZLinkSpotSerialExecutor {
     this.timerSerials.clear();
     this.actorExecutors.clear();
     this.closePromise = Promise.all([
-      ...childSerials.map((serial) => serial.close()),
+      ...childSerials.map((serial) => serial.whenIdle()),
       ...actorExecutors.map((executor) => executor.close())
     ]).then(() => undefined);
     return this.closePromise;

package/src/runtime/spots/spot-serial-turn-executor.ts

@@ -7,6 +7,7 @@ import {
   ZLinkSpotSerialTurn
 } from '../execution';
 import type { SpotId } from '../../contracts';
+import type { ZLinkFrameworkException } from '../../contracts/Errors/ZLinkFrameworkException';
 import {
   ZLinkFrameworkInternalErrorKind,
   createInternalFrameworkException
@@ -17,6 +18,11 @@ import {
   type ZLinkSerialWorkOptions,
   type ZLinkSerialWorkRecord
 } from '../execution/serial-execution-queue';
+import type { ZLinkSerialLifecycleContext } from '../execution/serial-execution-queue';
+
+export type ZLinkSpotApplicationTurnContext = ZLinkExecutionBarrierClaim & {
+  readonly replay?: (failure?: unknown) => Promise<unknown> | unknown;
+};
 import {
   bindApplicationJobPermit,
   hasApplicationJobPermit
@@ -29,6 +35,7 @@ export class ZLinkSpotSerialTurnExecutor {
   private executionBarrier: ZLinkExecutionBarrier | undefined;
   private resumedOwnerTurn: ZLinkSpotSerialTurn | undefined;
   private lastActivityAtMs = performance.now();
+  private lifecycleAdmissionClosed = false;
   activeTurnId = 0;
 
   constructor(
@@ -62,15 +69,41 @@ export class ZLinkSpotSerialTurnExecutor {
     return captureZLinkSpotSerialTurn(this);
   }
 
-  /** Stops new turns while allowing turns accepted before close to complete. */
-  close(): Promise<void> {
-    return this.scheduler.close();
+  /** Waits for turns admitted through the execution barrier to complete. */
+  whenIdle(): Promise<void> {
+    return this.scheduler.whenIdle();
   }
 
   closeAdmission(): void {
     this.scheduler.closeAdmission();
   }
 
+  /**
+   * Spot messaging §7 step 3: after Close ends this queue's admission, an Instance
+   * intent message is refused before admission as a released owner fence, and any
+   * other message finds no incarnation. Undefined while admission is open.
+   */
+  applicationAdmissionRefusal(instanceIntent: boolean): ZLinkFrameworkException | undefined {
+    if (!this.scheduler.admissionClosed) return undefined;
+    return instanceIntent
+      ? createInternalFrameworkException(
+          ZLinkFrameworkInternalErrorKind.SpotMoving,
+          'The Spot incarnation released its authority before admitting this Instance intent message.'
+        )
+      : createInternalFrameworkException(
+          ZLinkFrameworkInternalErrorKind.RequestTargetNotFound,
+          'The Spot incarnation is closed.'
+        );
+  }
+
+  /**
+   * Spot messaging §7: once Close commits Closing, membership work can no longer
+   * enter this incarnation while Instance intent messages still queue behind Close.
+   */
+  setLifecycleAdmissionClosed(closed: boolean): void {
+    this.lifecycleAdmissionClosed = closed;
+  }
+
   /** Distinguishes a gate-owning turn from a suspended AsyncLocalStorage tail. */
   isActiveTurn(turn: ZLinkSpotSerialTurn, turnId: number): boolean {
     return (
@@ -90,14 +123,18 @@ export class ZLinkSpotSerialTurnExecutor {
   }
 
   /** Runs `operation` in serial order, one turn at a time. */
-  execute<T>(operation: () => Promise<T> | T, workOptions?: ZLinkSerialWorkOptions): Promise<T> {
+  execute<T>(
+    operation: () => Promise<T> | T,
+    workOptions?: ZLinkSerialWorkOptions,
+    replay?: (failure?: unknown) => Promise<T>
+  ): Promise<T> {
     if (this.isCurrentTurn) {
       throw createInternalFrameworkException(
         ZLinkFrameworkInternalErrorKind.InvalidOperation,
         'A Spot serial turn cannot implicitly execute another turn for the same owner.'
       );
     }
-    return this.enqueueApplicationTurn(operation, workOptions);
+    return this.enqueueApplicationTurn(operation, workOptions, replay);
   }
 
   /**
@@ -119,12 +156,19 @@ export class ZLinkSpotSerialTurnExecutor {
     workOptions: ZLinkSerialWorkOptions = {},
     admission: {
       readonly signal?: AbortSignal;
-    } = {}
+    } = {},
+    replay?: (failure?: unknown) => Promise<unknown>
   ): Promise<void> {
     this.lastActivityAtMs = performance.now();
     let barrierClaim: ZLinkExecutionBarrierClaim | undefined;
     try {
-      barrierClaim = await this.executionBarrier?.enter();
+      const entry = this.executionBarrier?.enter();
+      barrierClaim = entry instanceof Promise ? await entry : entry;
+      const refusal = this.applicationAdmissionRefusal(replay !== undefined);
+      if (refusal !== undefined) throw refusal;
+      if (barrierClaim !== undefined && replay !== undefined) {
+        Object.assign(barrierClaim, { replay: bindApplicationJobPermit(replay) });
+      }
       if (admission.signal?.aborted === true) {
         throw new DOMException('The operation was aborted.', 'AbortError');
       }
@@ -161,19 +205,63 @@ export class ZLinkSpotSerialTurnExecutor {
     return this.enqueueFrameworkTurn(operation, workOptions);
   }
 
-  /** Holds this Spot's lifecycle FIFO until the full operation result settles. */
+  /** Admits membership work and holds the lifecycle FIFO until its result settles. */
   executeLifecycleOperation<T>(operation: () => Promise<T> | T): Promise<T> {
-    return this.scheduler.submitLifecycleOperation(operation);
+    let entry: ZLinkExecutionBarrierClaim | Promise<ZLinkExecutionBarrierClaim> | undefined;
+    try {
+      if (this.lifecycleAdmissionClosed) {
+        throw createInternalFrameworkException(
+          ZLinkFrameworkInternalErrorKind.RequestRejected,
+          'Spot lifecycle admission is closed by a committed Close.'
+        );
+      }
+      entry = this.executionBarrier?.enter();
+    } catch (error) {
+      return Promise.reject(error);
+    }
+    const submit = (claim?: ZLinkExecutionBarrierClaim): Promise<T> => {
+      try {
+        return this.scheduler.submitLifecycleOperation(operation);
+      } catch (error) {
+        return Promise.reject(error);
+      } finally {
+        claim?.release();
+      }
+    };
+    return entry instanceof Promise ? entry.then(submit) : submit(entry);
+  }
+
+  /** Runs owner control work that establishes or completes its own admission seal. */
+  executeControlLifecycleOperation<T>(
+    operation: () => Promise<T> | T,
+    context?: ZLinkSerialLifecycleContext
+  ): Promise<T> {
+    try {
+      return this.scheduler.submitLifecycleOperation(operation, context);
+    } catch (error) {
+      return Promise.reject(error);
+    }
+  }
+
+  visitPendingApplication(visitor: (record: ZLinkSerialWorkRecord<unknown>) => void): void {
+    this.scheduler.visitPendingApplication(visitor);
   }
 
   private async enqueueApplicationTurn<T>(
     operation: () => Promise<T> | T,
-    workOptions: ZLinkSerialWorkOptions = {}
+    workOptions: ZLinkSerialWorkOptions = {},
+    replay?: (failure?: unknown) => Promise<T>
   ): Promise<T> {
     this.lastActivityAtMs = performance.now();
     let barrierClaim: ZLinkExecutionBarrierClaim | undefined;
     try {
-      barrierClaim = await this.executionBarrier?.enter();
+      const entry = this.executionBarrier?.enter();
+      barrierClaim = entry instanceof Promise ? await entry : entry;
+      const refusal = this.applicationAdmissionRefusal(replay !== undefined);
+      if (refusal !== undefined) throw refusal;
+      if (barrierClaim !== undefined && replay !== undefined) {
+        Object.assign(barrierClaim, { replay: bindApplicationJobPermit(replay) });
+      }
       return await this.submitQueuedTurn(operation, workOptions, barrierClaim);
     } catch (error) {
       barrierClaim?.release();
@@ -215,6 +303,19 @@ export class ZLinkSpotSerialTurnExecutor {
   }
 
   private runQueuedRecord(record: ZLinkSerialWorkRecord<unknown>): Promise<void> {
+    const context = record.context as ZLinkSpotApplicationTurnContext | undefined;
+    if (context?.replay !== undefined && record.operation === context.replay) {
+      context.release();
+      return Promise.resolve(this.executionBarrier?.enter()).then((claim) =>
+        this.runTurn(
+          record.operation,
+          (value) => record.resolve(value),
+          (error) => record.reject(error),
+          claim,
+          record.release
+        )
+      );
+    }
     return this.runTurn(
       record.operation,
       (value) => record.resolve(value),

package/src/runtime/spots/spot-timer.ts

@@ -24,13 +24,14 @@ import { createInboundFlow, runWithFlow } from '../diagnostics/flow-context';
 import type { ZLinkExecutionBarrier } from '../execution';
 import { ZLinkStateLane } from '../execution/state-lane';
 import { AsyncResource } from 'node:async_hooks';
-import type { OperationClock } from '../foundation/operation-registry';
 
 const detachedTimerStateLaneResource = new AsyncResource('zlink:spot-timer');
 
-export interface ZLinkTimerClock extends OperationClock {
+export interface ZLinkTimerClock {
   now(): number;
   utcNow(): number;
+  setTimeout(callback: () => void, delayMs: number): unknown;
+  clearTimeout(handle: unknown): void;
 }
 
 const systemTimerClock: ZLinkTimerClock = {