@zlink-systems/framework 0.29.1 → 0.30.1

package/src/runtime/host/index.ts

@@ -1,6 +1,9 @@
+import type { ApplicationJobOrigin } from '../application-jobs/contracts';
+import { currentApplicationJobOrigin } from '../application-jobs/application-job-queue-scope';
 import { listAllMeshNodeDescriptors } from '../locations/location-store-pages';
 import { ZlinkStreamContentType } from '@zlink-systems/stream-wire';
 import { randomBytes, randomUUID } from 'node:crypto';
+import { isDeepStrictEqual } from 'node:util';
 import {
   type ActorRef,
   type RoutingId,
@@ -242,7 +245,7 @@ export class ZLinkFrameworkRuntimeHost
   private readonly lifecycleSink?: string[];
   private executionState?: ZLinkFrameworkExecutionState;
   private runtimeState = ZLinkFrameworkRuntimeState.Preparing;
-  private runtimeSequence = 0n;
+  private runtimeStatus?: ZLinkFrameworkRuntimeStatus;
   private readonly runtimeObservationSource = Symbol('zlink.framework-runtime');
   private runtimeDeadline?: Date;
   private shutdownDeadline?: AbortController;
@@ -332,24 +335,20 @@ export class ZLinkFrameworkRuntimeHost
     this.runtimeEventPublisher =
       options.runtimeEventPublisher ?? new DefaultZLinkRuntimeEventPublisher();
     this.metrics = new ZLinkRuntimeMetrics(options.registration.metrics?.meterProvider);
+    const handlerStartGate = options.providerResolver?.get?.(
+      ZLINK_INTERNAL_APPLICATION_JOB_QUEUE_HANDLER_START_GATE as unknown as Type<ZLinkInternalApplicationJobQueueHandlerStartGate>
+    );
     this.applicationJobQueue = new ApplicationJobQueue(
       resolveApplicationJobQueueConfiguration(options.registration.applicationJobQueue, () =>
         nodeEffectiveProcessorCount(options.registration.worker?.maxThreads)
       ),
       undefined,
-      () => {
-        try {
-          return (
-            this.options.providerResolver
-              ?.get?.(
-                ZLINK_INTERNAL_APPLICATION_JOB_QUEUE_HANDLER_START_GATE as unknown as Type<ZLinkInternalApplicationJobQueueHandlerStartGate>
-              )
-              ?.shouldHoldPermitBeforeHandler() === true
-          );
-        } catch {
-          return false;
-        }
-      }
+      handlerStartGate?.shouldHoldPermitBeforeHandler.bind(handlerStartGate),
+      (error) =>
+        (this.errorSink ?? this.preStartErrorSink).reportRuntimeTaskException(
+          'logger-provider',
+          error
+        )
     );
     this.capacityStatus = new HostCapacityStatusProjection(
       options.registration.coreHwm,
@@ -721,13 +720,13 @@ export class ZLinkFrameworkRuntimeHost
       sessionRelocationWire: () => this.serviceRelocation,
       clearRemoteActorPacketTarget: (actorId) =>
         this.boundSessionRelay.clearRemoteActorPacketTarget(actorId),
-      prepareApplicationJob: async (preparationSignal) => {
+      prepareApplicationJob: async (preparationSignal, origin) => {
         const runtimeSignal = this.executionState?.abortController.signal;
         const signal =
           runtimeSignal === undefined
             ? preparationSignal
             : AbortSignal.any([runtimeSignal, preparationSignal]);
-        const permit = await this.applicationJobQueue.acquire(signal);
+        const permit = await this.applicationJobQueue.acquire(signal, origin);
         let state: 'ready' | 'running' | 'closed' = 'ready';
         let closedReason: unknown;
         let abort: (() => void) | undefined;
@@ -779,6 +778,7 @@ export class ZLinkFrameworkRuntimeHost
     this.serviceRelocation = new ZLinkHostServiceRelocationRuntime({
       registration: options.registration,
       targetAdmissionSealed: () => this.shutdownOperationStartedAt !== undefined,
+      sourceQuiescenceChanged: () => this.publishCurrentRuntimeStatus(),
       providerResolver: options.providerResolver,
       locationStore: () => this.locationOwner.currentStores?.locationStore,
       currentOwner: () => this.locationOwner.currentRuntime?.currentOwnerToken,
@@ -869,6 +869,10 @@ export class ZLinkFrameworkRuntimeHost
   }
 
   get status(): ZLinkFrameworkRuntimeStatus {
+    return this.publishCurrentRuntimeStatus();
+  }
+
+  private buildRuntimeStatus(): ZLinkFrameworkRuntimeStatus {
     return {
       state: this.runtimeState,
       isReady: runtimeStateIsReady(this.runtimeState),
@@ -878,8 +882,8 @@ export class ZLinkFrameworkRuntimeHost
       deadline: this.runtimeDeadline,
       relocationResult: this.runtimeRelocationResult,
       terminationResult: this.runtimeTerminationResult,
-      sequence: this.runtimeSequence,
-      observedAt: new Date()
+      sequence: this.runtimeStatus?.sequence ?? 0n,
+      observedAt: this.runtimeStatus?.observedAt ?? new Date()
     };
   }
 
@@ -897,10 +901,11 @@ export class ZLinkFrameworkRuntimeHost
   }
 
   observe(signal?: AbortSignal): AsyncIterable<ZLinkObservedStatus<ZLinkFrameworkRuntimeStatus>> {
+    const status = this.status;
     const queue = new RuntimeEventQueue<ZLinkFrameworkRuntimeStatus>(undefined, signal);
     this.runtimeObservers.add(queue);
     queue.onClose(() => this.runtimeObservers.delete(queue));
-    this.publishRuntimeStatus(queue, this.status);
+    this.publishRuntimeStatus(queue, status);
     return queue;
   }
 
@@ -1123,6 +1128,9 @@ export class ZLinkFrameworkRuntimeHost
         );
         const drain = await shutdown;
         await unscopedStreamDrain;
+        if (drain.kind === 'forceStopped' && drain.error !== undefined) {
+          this.runtimeOrPreStartErrorSink.reportRuntimeTaskException('host shutdown', drain.error);
+        }
         await this.stop();
         if (drain.kind === 'forceStopped') {
           return this.completeTermination({
@@ -1270,12 +1278,25 @@ export class ZLinkFrameworkRuntimeHost
   private setRuntimeState(state: ZLinkFrameworkRuntimeState): void {
     if (this.runtimeState === state) return;
     this.runtimeState = state;
-    this.runtimeSequence += 1n;
     this.notifyTopologyHostStateChanged();
-    const status = this.status;
+    this.publishCurrentRuntimeStatus();
+  }
+
+  private publishCurrentRuntimeStatus(): ZLinkFrameworkRuntimeStatus {
+    const current = this.buildRuntimeStatus();
+    if (this.runtimeStatus !== undefined && isDeepStrictEqual(this.runtimeStatus, current)) {
+      return this.runtimeStatus;
+    }
+    const status = {
+      ...current,
+      sequence: (this.runtimeStatus?.sequence ?? 0n) + 1n,
+      observedAt: new Date()
+    };
+    this.runtimeStatus = status;
     for (const observer of this.runtimeObservers) {
       this.publishRuntimeStatus(observer, status);
     }
+    return status;
   }
 
   private publishRuntimeStatus(
@@ -1722,6 +1743,10 @@ export class ZLinkFrameworkRuntimeHost
       activationNode.registerAsyncInstanceActivationAuthority?.(
         new ZLinkInstanceActivationAuthority({
           store: locationStore,
+          relocationPolicy: (stableType) =>
+            this.options.registration.spotNodes.get(meshName)?.instanceSpotFactoryRegistrations?.[
+              stableType
+            ]?.relocation.kind,
           relocationStore: this.options.registration.locations.relocationStoreInstance,
           meshName,
           owner: () => this.locationOwner.currentRuntime?.currentOwnerToken,
@@ -2235,6 +2260,9 @@ export class ZLinkFrameworkRuntimeHost
     });
     const coordinator = (this.userSpotCoordinator = new ZLinkUserSpotCreationCoordinator({
       store: locationStore,
+      relocationPolicy: (meshName, stableType) =>
+        this.options.registration.spotNodes.get(meshName)?.spotFactoryRegistrations?.[stableType]
+          ?.relocation.kind,
       publishReadyRoute: (meshName, route) => {
         this.cachedLocationSpotRouteResolver?.invalidate?.(route.spot.spotId);
         this.spotNodeRuntime?.meshNode(meshName)?.rememberSpotRoute?.(route);
@@ -2757,6 +2785,7 @@ export class ZLinkFrameworkRuntimeHost
       },
       meshRecordDispatcher: (meshName: string, owner: ReadyRecord, record: ReceiveRecord) =>
         this.dispatchMeshRecord(
+          currentApplicationJobOrigin,
           meshName,
           owner,
           record,
@@ -2766,12 +2795,68 @@ export class ZLinkFrameworkRuntimeHost
     };
   }
 
-  private async dispatchMeshRecord(
+  private dispatchMeshNodeRecord(
+    meshName: string,
+    record: ReceiveRecord,
+    signal?: AbortSignal
+  ): void | Promise<void> {
+    const control = this.serviceRelocation.tryHandleControl(meshName, record, signal);
+    if (control instanceof Promise)
+      return control.then((handled) =>
+        handled ? undefined : this.dispatchMeshNodePayload(meshName, record, signal)
+      );
+    if (!control) return this.dispatchMeshNodePayload(meshName, record, signal);
+  }
+
+  private dispatchMeshNodePayload(
     meshName: string,
-    owner: ReadyRecord,
     record: ReceiveRecord,
     signal?: AbortSignal
+  ): void | Promise<void> {
+    if (record.kind === ReceiveKind.NodeRequest && record.parts.length === 1) {
+      const terminal = decodeRemoteActorSourceLeaveTerminal(record.parts[0]!.data());
+      if (terminal !== undefined) {
+        return this.completeFormalSourceLeave(record, terminal);
+      }
+    }
+    const channelRuntime = this.channelRuntime;
+    if (channelRuntime === undefined) {
+      throw new ZLinkConfigurationException(
+        'MeshNode node-direct dispatch requires the channel runtime.'
+      );
+    }
+    return channelRuntime.dispatchMeshRoute(meshName, record, signal);
+  }
+
+  private async completeFormalSourceLeave(
+    record: ReceiveRecord,
+    terminal: NonNullable<ReturnType<typeof decodeRemoteActorSourceLeaveTerminal>>
   ): Promise<void> {
+    const completed = await this.spotManager?.completeFormalSourceLeaveTerminal(
+      terminal.actorId,
+      terminal.transferId,
+      terminal.succeeded
+    );
+    if (completed !== true) {
+      throw new ZLinkConfigurationException(
+        `Actor '${terminal.actorId}' has no matching formal transfer terminal gate.`
+      );
+    }
+    if (record.reply(Buffer.alloc(0)) !== SubmitResult.Ok) {
+      throw new ZLinkConfigurationException(
+        `Actor '${terminal.actorId}' formal transfer terminal acknowledgement failed.`
+      );
+    }
+    return;
+  }
+
+  private dispatchMeshRecord(
+    applicationJobOrigin: () => ApplicationJobOrigin,
+    meshName: string,
+    owner: ReadyRecord,
+    record: ReceiveRecord,
+    signal?: AbortSignal
+  ): void | Promise<void> {
     if (record.operationKind === OperationKind.ActorJoin) {
       if (this.spotManager === undefined) {
         throw new ZLinkConfigurationException(
@@ -2780,107 +2865,8 @@ export class ZLinkFrameworkRuntimeHost
       }
       return this.spotManager.dispatchMeshActorJoin(meshName, owner, record);
     }
-    if (record.kind === ReceiveKind.ActorBinding) {
-      if (record.kindData?.kind !== 'actorBinding') {
-        throw new ZLinkConfigurationException(
-          'MeshNode Actor binding record has no binding generation.'
-        );
-      }
-      const binding = record.kindData;
-      const replyBindingFailure = (kind: ZLinkFrameworkInternalErrorKind): void => {
-        const terminal = internalFrameworkWireReply(kind);
-        if (
-          record.replyFailure?.(terminal.terminalResult, terminal.failureCode) !== SubmitResult.Ok
-        ) {
-          throw new ZLinkConfigurationException(
-            `MeshNode Actor binding target '${binding.actor.actorId}' failure acknowledgement failed.`
-          );
-        }
-      };
-      const replyBindingSuccess = (): void => {
-        if (record.reply(Buffer.alloc(0)) !== SubmitResult.Ok) {
-          throw new ZLinkConfigurationException(
-            `MeshNode Actor binding target '${binding.actor.actorId}' acknowledgement failed.`
-          );
-        }
-      };
-      const state = this.actorManager?.getState(binding.actor.actorId);
-      const nativeActorRef = state?.nativeActorRef;
-      if (
-        state === undefined ||
-        nativeActorRef === undefined ||
-        nativeActorRef.actorId !== binding.actor.actorId ||
-        !routingIdsEqual(nativeActorRef.nodeRid, binding.actor.nodeRid)
-      ) {
-        replyBindingFailure(ZLinkFrameworkInternalErrorKind.ActorLocationStale);
-        return;
-      }
-      if (nativeActorRef.generation !== binding.actor.generation) {
-        replyBindingFailure(ZLinkFrameworkInternalErrorKind.ActorGenerationStale);
-        return;
-      }
-      const node =
-        state.meshName === meshName ? this.spotNodeRuntime?.meshNode(meshName) : undefined;
-      const localStatus = node?.status();
-      // ownsLocation tracks local cleanup responsibility, not Actor authority.
-      // A relocation target can exact-read committed authority without owning that cleanup.
-      if (
-        localStatus === undefined ||
-        !routingIdsEqual(localStatus.routingId, binding.actor.nodeRid) ||
-        localStatus.lifecycleGeneration !== binding.actorNodeGeneration ||
-        state.locationGeneration !== binding.authorityOwnerGeneration ||
-        state.isMoving
-      ) {
-        replyBindingFailure(ZLinkFrameworkInternalErrorKind.ActorLocationStale);
-        return;
-      }
-      const sessionOwnerIsLocal =
-        routingIdsEqual(localStatus.routingId, binding.sessionNodeRid) &&
-        localStatus.lifecycleGeneration === binding.sessionOwnerNodeGeneration;
-      if (binding.transition === 'tombstone') {
-        const actorRef: ActorRef = {
-          actorId: binding.actor.actorId,
-          objectGeneration: binding.actor.generation,
-          meshName,
-          nodeRid: binding.actor.nodeRid
-        };
-        if (sessionOwnerIsLocal) {
-          releaseApplicationJobPermitBeforeHandler();
-          await this.streamBindingRuntime.retireRemoteBinding(
-            actorRef,
-            binding.sessionRid,
-            binding.bindingGeneration,
-            signal
-          );
-        }
-        state.retireBoundSessionBinding(binding);
-        replyBindingSuccess();
-        return;
-      }
-      const target = this.boundSessionRelay.boundSessions.resolveRemoteBoundSessionTarget(
-        binding.sessionNodeRid,
-        binding.sessionRid
-      );
-      if (target === undefined) {
-        replyBindingFailure(ZLinkFrameworkInternalErrorKind.ActorLocationStale);
-        return;
-      }
-      const installed = state.installBoundSessionBinding({
-        ...target,
-        sessionNodeRid: binding.sessionNodeRid,
-        sessionRid: binding.sessionRid,
-        sessionOwnerNodeGeneration: binding.sessionOwnerNodeGeneration,
-        sessionOwnerId: binding.sessionOwnerId,
-        sessionOwnerLeaseGeneration: binding.sessionOwnerLeaseGeneration,
-        bindingGeneration: binding.bindingGeneration
-      });
-      if (!installed) {
-        replyBindingFailure(ZLinkFrameworkInternalErrorKind.ActorLocationStale);
-        return;
-      }
-      replyBindingSuccess();
-      return Promise.resolve();
-    }
+    if (record.kind === ReceiveKind.ActorBinding)
+      return this.dispatchMeshActorBinding(meshName, record, signal);
     switch (record.kind) {
       case ReceiveKind.TransferControl:
         if (record.kindData?.kind !== 'transferControl') {
@@ -2903,41 +2889,11 @@ export class ZLinkFrameworkRuntimeHost
       }
       case ReceiveKind.NodeSend:
       case ReceiveKind.NodeRequest: {
-        return this.admission.run(meshName, 'RouteMesh node dispatch', async () => {
-          if (await this.serviceRelocation.tryHandleControl(meshName, record, signal)) {
-            return;
-          }
-          if (record.kind === ReceiveKind.NodeRequest && record.parts.length === 1) {
-            const terminal = decodeRemoteActorSourceLeaveTerminal(record.parts[0]!.data());
-            if (terminal !== undefined) {
-              if (
-                (await this.spotManager?.completeFormalSourceLeaveTerminal(
-                  terminal.actorId,
-                  terminal.transferId,
-                  terminal.succeeded
-                )) !== true
-              ) {
-                throw new ZLinkConfigurationException(
-                  `Actor '${terminal.actorId}' has no matching formal transfer terminal gate.`
-                );
-              }
-              if (record.reply(Buffer.alloc(0)) !== SubmitResult.Ok) {
-                throw new ZLinkConfigurationException(
-                  `Actor '${terminal.actorId}' formal transfer terminal acknowledgement failed.`
-                );
-              }
-              return;
-            }
-          }
-          const channelRuntime = this.channelRuntime;
-          if (channelRuntime === undefined) {
-            throw new ZLinkConfigurationException(
-              'MeshNode node-direct dispatch requires the channel runtime.'
-            );
-          }
-          return await channelRuntime.dispatchMeshRoute(meshName, record, signal);
-        });
+        return this.admission.run(meshName, 'RouteMesh node dispatch', () =>
+          this.dispatchMeshNodeRecord(meshName, record, signal)
+        );
       }
+
       case ReceiveKind.SpotSend:
       case ReceiveKind.SpotRequest:
       case ReceiveKind.SpotMulticast:
@@ -2968,7 +2924,7 @@ export class ZLinkFrameworkRuntimeHost
           );
         }
         return this.admission.run(meshName, 'RouteMesh Actor dispatch', () =>
-          this.spotManager!.dispatchMeshActor(meshName, owner, record)
+          this.spotManager!.dispatchMeshActor(applicationJobOrigin(), meshName, owner, record)
         );
       case ReceiveKind.SendReady:
         return Promise.resolve();
@@ -2979,6 +2935,111 @@ export class ZLinkFrameworkRuntimeHost
     }
   }
 
+  private async dispatchMeshActorBinding(
+    meshName: string,
+    record: ReceiveRecord,
+    signal?: AbortSignal
+  ): Promise<void> {
+    if (record.kindData?.kind !== 'actorBinding') {
+      throw new ZLinkConfigurationException(
+        'MeshNode Actor binding record has no binding generation.'
+      );
+    }
+    const binding = record.kindData;
+    const replyBindingFailure = (kind: ZLinkFrameworkInternalErrorKind): void => {
+      const terminal = internalFrameworkWireReply(kind);
+      if (
+        record.replyFailure?.(terminal.terminalResult, terminal.failureCode) !== SubmitResult.Ok
+      ) {
+        throw new ZLinkConfigurationException(
+          `MeshNode Actor binding target '${binding.actor.actorId}' failure acknowledgement failed.`
+        );
+      }
+    };
+    const replyBindingSuccess = (): void => {
+      if (record.reply(Buffer.alloc(0)) !== SubmitResult.Ok) {
+        throw new ZLinkConfigurationException(
+          `MeshNode Actor binding target '${binding.actor.actorId}' acknowledgement failed.`
+        );
+      }
+    };
+    const state = this.actorManager?.getState(binding.actor.actorId);
+    const nativeActorRef = state?.nativeActorRef;
+    if (
+      state === undefined ||
+      nativeActorRef === undefined ||
+      nativeActorRef.actorId !== binding.actor.actorId ||
+      !routingIdsEqual(nativeActorRef.nodeRid, binding.actor.nodeRid)
+    ) {
+      replyBindingFailure(ZLinkFrameworkInternalErrorKind.ActorLocationStale);
+      return;
+    }
+    if (nativeActorRef.generation !== binding.actor.generation) {
+      replyBindingFailure(ZLinkFrameworkInternalErrorKind.ActorGenerationStale);
+      return;
+    }
+    const node = state.meshName === meshName ? this.spotNodeRuntime?.meshNode(meshName) : undefined;
+    const localStatus = node?.status();
+    // ownsLocation tracks local cleanup responsibility, not Actor authority.
+    // A relocation target can exact-read committed authority without owning that cleanup.
+    if (
+      localStatus === undefined ||
+      !routingIdsEqual(localStatus.routingId, binding.actor.nodeRid) ||
+      localStatus.lifecycleGeneration !== binding.actorNodeGeneration ||
+      state.locationGeneration !== binding.authorityOwnerGeneration ||
+      state.isMoving
+    ) {
+      replyBindingFailure(ZLinkFrameworkInternalErrorKind.ActorLocationStale);
+      return;
+    }
+    const sessionOwnerIsLocal =
+      routingIdsEqual(localStatus.routingId, binding.sessionNodeRid) &&
+      localStatus.lifecycleGeneration === binding.sessionOwnerNodeGeneration;
+    if (binding.transition === 'tombstone') {
+      const actorRef: ActorRef = {
+        actorId: binding.actor.actorId,
+        objectGeneration: binding.actor.generation,
+        meshName,
+        nodeRid: binding.actor.nodeRid
+      };
+      if (sessionOwnerIsLocal) {
+        releaseApplicationJobPermitBeforeHandler();
+        await this.streamBindingRuntime.retireRemoteBinding(
+          actorRef,
+          binding.sessionRid,
+          binding.bindingGeneration,
+          signal
+        );
+      }
+      state.retireBoundSessionBinding(binding);
+      replyBindingSuccess();
+      return;
+    }
+    const target = this.boundSessionRelay.boundSessions.resolveRemoteBoundSessionTarget(
+      binding.sessionNodeRid,
+      binding.sessionRid
+    );
+    if (target === undefined) {
+      replyBindingFailure(ZLinkFrameworkInternalErrorKind.ActorLocationStale);
+      return;
+    }
+    const installed = state.installBoundSessionBinding({
+      ...target,
+      sessionNodeRid: binding.sessionNodeRid,
+      sessionRid: binding.sessionRid,
+      sessionOwnerNodeGeneration: binding.sessionOwnerNodeGeneration,
+      sessionOwnerId: binding.sessionOwnerId,
+      sessionOwnerLeaseGeneration: binding.sessionOwnerLeaseGeneration,
+      bindingGeneration: binding.bindingGeneration
+    });
+    if (!installed) {
+      replyBindingFailure(ZLinkFrameworkInternalErrorKind.ActorLocationStale);
+      return;
+    }
+    replyBindingSuccess();
+    return Promise.resolve();
+  }
+
   private submitLocalMeshRoute(
     meshName: string,
     sourceNodeRid: string,
@@ -3012,7 +3073,10 @@ export class ZLinkFrameworkRuntimeHost
     try {
       state.taskRunner.runDetached('mesh-node-local-route-dispatch', async () => {
         try {
-          const permit = await this.applicationJobQueue.acquire(state.abortController.signal);
+          const permit = await this.applicationJobQueue.acquire(
+            state.abortController.signal,
+            'local'
+          );
           permit.markApplicationQueued();
           await runWithApplicationJobPermit(permit, () =>
             channelRuntime.dispatchLocalMeshRoute(meshName, sourceNodeRid, ownedParts)