@zlink-systems/framework 0.29.1 → 0.30.1

package/src/runtime/host/application-job-queue.ts

@@ -1,3 +1,5 @@
+import { SeverityNumber } from '@opentelemetry/api-logs';
+import { telemetryLogger } from '../diagnostics/message-flow';
 import { MILLISECONDS_PER_SECOND } from '../diagnostics/runtime-metrics';
 import { ZLinkApplicationJobQueueProfile } from '../../contracts/Dispatch';
 import {
@@ -11,7 +13,9 @@ import {
 import { readFileSync } from 'node:fs';
 import { availableParallelism } from 'node:os';
 import type {
+  ApplicationJobOrigin,
   ApplicationJobPermitPort,
+  ApplicationJobReceiveRotationPort,
   ApplicationJobQueuePort
 } from '../application-jobs/contracts';
 import {
@@ -119,10 +123,42 @@ export function resolveApplicationJobQueueConfiguration(
   });
 }
 
+const enum QueuePermitCount {
+  Reserved,
+  Queued,
+  Peak,
+  Size
+}
+
 export class ApplicationJobQueuePermit implements ApplicationJobPermitPort {
-  private state: 'reserved' | 'queued' | 'released' = 'reserved';
+  detachedForHandlerTurn = false;
+  private state: 'idle' | 'reserved' | 'queued' | 'released';
+
+  constructor(
+    private readonly owner: ApplicationJobQueue,
+    readonly origin: ApplicationJobOrigin,
+    private readonly reusable = false
+  ) {
+    this.state = reusable ? 'idle' : 'reserved';
+  }
 
-  constructor(private readonly owner: ApplicationJobQueue) {}
+  reserveStorage(): boolean {
+    if (this.state !== 'idle') return false;
+    this.state = 'reserved';
+    this.detachedForHandlerTurn = false;
+    return true;
+  }
+
+  releaseStorage(): void {
+    if (!this.reusable || this.state === 'idle') return;
+    if (this.state !== 'released')
+      throw new Error('Reservation storage still owns queue capacity.');
+    this.state = 'idle';
+  }
+
+  detachForHandlerTurn(): void {
+    this.detachedForHandlerTurn = true;
+  }
 
   markApplicationQueued(): void {
     if (this.state !== 'reserved') {
@@ -134,21 +170,78 @@ export class ApplicationJobQueuePermit implements ApplicationJobPermitPort {
 
   releaseBeforeHandler(): void {
     if (this.owner.shouldHoldPermitBeforeHandler()) return;
-    this.release();
+    this.owner.releasePermit(this);
   }
 
   releaseAfterInternalProcessing(): void {
-    this.release();
+    this.owner.releasePermit(this);
   }
 
-  private release(): void {
-    if (this.state === 'released') return;
-    this.owner.release(this.state);
+  takeReleaseState(): 'reserved' | 'queued' | undefined {
+    if (this.state === 'released' || this.state === 'idle') return undefined;
+    const state = this.state;
     this.state = 'released';
+    return state;
+  }
+}
+
+/** Connection-owned storage; queue accounting remains with the host queue. */
+class ApplicationJobReceiveRotation implements ApplicationJobReceiveRotationPort {
+  private readonly storage: ApplicationJobQueuePermit[] = [];
+  private readonly grants: (ApplicationJobQueuePermit | undefined)[];
+  private granted = 0;
+  private consumed = 0;
+
+  constructor(
+    private readonly owner: ApplicationJobQueue,
+    private readonly capacity: number,
+    private readonly origin: ApplicationJobOrigin
+  ) {
+    this.grants = new Array<ApplicationJobQueuePermit | undefined>(capacity);
+    this.prepareStorage();
+  }
+
+  private prepareStorage(): void {
+    for (let index = 0; index < this.capacity; index++)
+      this.storage.push(new ApplicationJobQueuePermit(this.owner, this.origin, true));
+  }
+
+  reserve(): void {
+    this.releaseUnused();
+    const count = this.owner.receiveRotationCapacity(this.capacity);
+    if (count === 0) return;
+    for (let index = 0; index < this.storage.length; index++) {
+      const permit = this.storage[index]!;
+      if (permit.reserveStorage()) this.grants[this.granted++] = permit;
+      if (this.granted === count) break;
+    }
+    if (this.granted < count) {
+      const start = this.storage.length;
+      this.prepareStorage();
+      for (let index = start; this.granted < count; index++) {
+        const permit = this.storage[index]!;
+        permit.reserveStorage();
+        this.grants[this.granted++] = permit;
+      }
+    }
+    this.owner.reserveReceiveRotation(count);
+  }
+
+  take(): ApplicationJobQueuePermit | undefined {
+    if (this.consumed === this.granted) return undefined;
+    return this.grants[this.consumed++];
+  }
+
+  releaseUnused(): void {
+    this.owner.releaseUnusedReceiveRotation(this.grants, this.consumed, this.granted);
+    // Keep the prepared grant slots and reset only the logical count.
+    this.granted = 0;
+    this.consumed = 0;
   }
 }
 
 interface CapacityWaiter {
+  readonly origin: ApplicationJobOrigin;
   readonly resolve: (permit: ApplicationJobQueuePermit) => void;
   readonly reject: (reason: unknown) => void;
   readonly signal?: AbortSignal;
@@ -164,9 +257,13 @@ interface CapacityWaiter {
  * directly to the oldest live waiter without allocating the configured limit.
  */
 export class ApplicationJobQueue implements ApplicationJobQueuePort {
-  private reservedSupplyPermits = 0n;
-  private queuedApplicationJobs = 0n;
-  private peakPermitsInUse = 0n;
+  private localWaiters = 0;
+  private localBacklogLogged = false;
+  // The contract bounds capacity by INT32_MAX; public snapshots remain bigint.
+  private readonly permitCounts = new Uint32Array(QueuePermitCount.Size);
+  private readonly limitCount: number;
+  private readonly pauseCount: number;
+  private readonly resumeCount: number;
   private capacityWaitCount = 0n;
   private capacityWaitDurationSeconds = 0;
   private metricsEpoch = 0n;
@@ -191,8 +288,12 @@ export class ApplicationJobQueue implements ApplicationJobQueuePort {
   constructor(
     private readonly configuration: ApplicationJobQueueConfiguration,
     private readonly nowMs: () => number = () => performance.now(),
-    private readonly handlerStartGate?: () => boolean
+    private readonly handlerStartGate?: () => boolean,
+    private readonly loggerFailureReporter?: (error: unknown) => void
   ) {
+    this.limitCount = Number(configuration.effectiveMaxQueuedApplicationJobs);
+    this.pauseCount = Number(configuration.pausePermitCount);
+    this.resumeCount = Number(configuration.resumePermitCount);
     this.receiveFlowController = new ApplicationJobReceiveFlowController(this, (state) => state);
   }
 
@@ -200,6 +301,24 @@ export class ApplicationJobQueue implements ApplicationJobQueuePort {
     return this.handlerStartGate?.() === true;
   }
 
+  createReceiveRotation(
+    capacity: number,
+    origin: ApplicationJobOrigin
+  ): ApplicationJobReceiveRotationPort {
+    if (!Number.isSafeInteger(capacity) || capacity < 1)
+      throw new RangeError('Receive capacity must be positive.');
+    return new ApplicationJobReceiveRotation(this, capacity, origin);
+  }
+
+  receiveRotationCapacity(capacity: number): number {
+    if (this.waiters.length !== 0) return 0;
+    return Math.min(capacity, this.limitCount - this.permitsInUse());
+  }
+
+  reserveReceiveRotation(count: number): void {
+    this.reserveSupply(count);
+  }
+
   registerReceiveFlowTarget(
     identity: object,
     applyState: (state: ApplicationJobQueuePressureState) => void,
@@ -242,16 +361,33 @@ export class ApplicationJobQueue implements ApplicationJobQueuePort {
     this.receiveFlowController.unregister(target);
   }
 
-  acquire(signal?: AbortSignal): Promise<ApplicationJobQueuePermit> {
+  acquire(
+    signal: AbortSignal | undefined,
+    origin: ApplicationJobOrigin
+  ): Promise<ApplicationJobQueuePermit> {
+    return new Promise((resolve, reject) => {
+      const permit = this.acquireOrResume(resolve, reject, signal, origin);
+      if (permit !== undefined) resolve(permit);
+    });
+  }
+
+  acquireOrResume(
+    resume: (permit: ApplicationJobQueuePermit) => void,
+    reject: (reason: unknown) => void,
+    signal: AbortSignal | undefined,
+    origin: ApplicationJobOrigin
+  ): ApplicationJobQueuePermit | undefined {
     if (signal?.aborted === true) {
-      return Promise.reject(abortReason(signal));
+      reject(abortReason(signal));
+      return undefined;
     }
-    if (this.waiters.length === 0 && this.permitsInUse() < this.limit()) {
-      return Promise.resolve(this.reserve());
+    if (this.waiters.length === 0 && this.permitsInUse() < this.limitCount) {
+      return this.reserve(origin);
     }
 
-    return new Promise<ApplicationJobQueuePermit>((resolve, reject) => {
+    new Promise<ApplicationJobQueuePermit>((resolve, reject) => {
       const waiter: CapacityWaiter = {
+        origin,
         resolve,
         reject,
         signal,
@@ -262,9 +398,14 @@ export class ApplicationJobQueue implements ApplicationJobQueuePort {
       };
       this.waiters.push(waiter);
       this.capacityWaitCount += 1n;
+      const warningCount = this.updateLocalWaiters(origin, 1);
       signal?.addEventListener('abort', waiter.abort, { once: true });
       this.drainWaiters();
-    });
+      if (warningCount !== undefined) this.logLocalBacklog(warningCount);
+    })
+      .then(resume)
+      .catch(reject);
+    return undefined;
   }
 
   snapshot(): ApplicationJobQueueSnapshot {
@@ -272,10 +413,10 @@ export class ApplicationJobQueue implements ApplicationJobQueuePort {
     const now = this.nowMs();
     return Object.freeze({
       ...this.configuration,
-      reservedSupplyPermits: this.reservedSupplyPermits,
-      queuedApplicationJobs: this.queuedApplicationJobs,
-      permitsInUse,
-      peakPermitsInUse: this.peakPermitsInUse,
+      reservedSupplyPermits: BigInt(this.permitCounts[QueuePermitCount.Reserved]!),
+      queuedApplicationJobs: BigInt(this.permitCounts[QueuePermitCount.Queued]!),
+      permitsInUse: BigInt(permitsInUse),
+      peakPermitsInUse: BigInt(this.permitCounts[QueuePermitCount.Peak]!),
       capacityWaiters: BigInt(this.waiters.length),
       capacityWaitCount: this.capacityWaitCount,
       capacityWaitDurationSeconds: this.capacityWaitDurationSeconds,
@@ -291,7 +432,7 @@ export class ApplicationJobQueue implements ApplicationJobQueuePort {
 
   resetMetrics(): void {
     this.metricsEpoch += 1n;
-    this.peakPermitsInUse = this.permitsInUse();
+    this.permitCounts[QueuePermitCount.Peak] = this.permitsInUse();
     this.capacityWaitCount = 0n;
     this.capacityWaitDurationSeconds = 0;
     this.runningTransitionCount = 0n;
@@ -325,25 +466,42 @@ export class ApplicationJobQueue implements ApplicationJobQueuePort {
   }
 
   markQueued(): void {
-    if (this.reservedSupplyPermits === 0n) {
+    if (this.permitCounts[QueuePermitCount.Reserved] === 0) {
       throw new Error('Application job queue has no reserved supply permit.');
     }
-    this.reservedSupplyPermits -= 1n;
-    this.queuedApplicationJobs += 1n;
+    this.permitCounts[QueuePermitCount.Reserved] =
+      this.permitCounts[QueuePermitCount.Reserved]! - 1;
+    this.permitCounts[QueuePermitCount.Queued] = this.permitCounts[QueuePermitCount.Queued]! + 1;
   }
 
-  release(state: 'reserved' | 'queued'): void {
-    if (state === 'reserved') {
-      if (this.reservedSupplyPermits === 0n) {
-        throw new Error('Application job queue reserved permit accounting underflow.');
-      }
-      this.reservedSupplyPermits -= 1n;
-    } else {
-      if (this.queuedApplicationJobs === 0n) {
-        throw new Error('Application job queue queued permit accounting underflow.');
-      }
-      this.queuedApplicationJobs -= 1n;
+  releasePermit(permit: ApplicationJobQueuePermit): void {
+    const state = permit.takeReleaseState();
+    if (state !== undefined) this.release(state);
+  }
+
+  releaseUnusedReceiveRotation(
+    grants: readonly (ApplicationJobQueuePermit | undefined)[],
+    consumed: number,
+    granted: number
+  ): void {
+    const count = granted - consumed;
+    if (count === 0) return;
+    // Unconsumed grants still belong to this receive round; no record has acquired them.
+    for (let index = consumed; index < granted; index++) {
+      const permit = grants[index]!;
+      permit.takeReleaseState();
+      permit.releaseStorage();
+    }
+    this.release('reserved', count);
+  }
+
+  release(state: 'reserved' | 'queued', count = 1): void {
+    const index = state === 'reserved' ? QueuePermitCount.Reserved : QueuePermitCount.Queued;
+    const available = this.permitCounts[index]!;
+    if (available < count) {
+      throw new Error(`Application job queue ${state} permit accounting underflow.`);
     }
+    this.permitCounts[index] = available - count;
     this.drainWaiters();
     this.evaluatePressure();
   }
@@ -352,26 +510,38 @@ export class ApplicationJobQueue implements ApplicationJobQueuePort {
     return this.configuration.effectiveMaxQueuedApplicationJobs;
   }
 
-  private permitsInUse(): bigint {
-    return this.reservedSupplyPermits + this.queuedApplicationJobs;
+  private permitsInUse(): number {
+    return (
+      this.permitCounts[QueuePermitCount.Reserved]! + this.permitCounts[QueuePermitCount.Queued]!
+    );
   }
 
-  private reserve(evaluatePressure = true): ApplicationJobQueuePermit {
-    this.reservedSupplyPermits += 1n;
-    const current = this.permitsInUse();
-    if (current > this.peakPermitsInUse) this.peakPermitsInUse = current;
+  private reserveSupply(count: number, evaluatePressure = true): void {
+    this.permitCounts[QueuePermitCount.Reserved] =
+      this.permitCounts[QueuePermitCount.Reserved]! + count;
+    this.permitCounts[QueuePermitCount.Peak] = Math.max(
+      this.permitCounts[QueuePermitCount.Peak]!,
+      this.permitsInUse()
+    );
     if (evaluatePressure) this.evaluatePressure();
-    return new ApplicationJobQueuePermit(this);
+  }
+
+  private reserve(
+    origin: ApplicationJobOrigin,
+    evaluatePressure = true
+  ): ApplicationJobQueuePermit {
+    this.reserveSupply(1, evaluatePressure);
+    return new ApplicationJobQueuePermit(this, origin);
   }
 
   private drainWaiters(): void {
-    while (this.waiters.length > 0 && this.permitsInUse() < this.limit()) {
+    while (this.waiters.length > 0 && this.permitsInUse() < this.limitCount) {
       const waiter = this.waiters.shift()!;
       if (!waiter.active) continue;
       waiter.active = false;
       waiter.signal?.removeEventListener('abort', waiter.abort);
       this.recordCompletedWait(waiter);
-      waiter.resolve(this.reserve(false));
+      waiter.resolve(this.reserve(waiter.origin, false));
     }
   }
 
@@ -387,19 +557,50 @@ export class ApplicationJobQueue implements ApplicationJobQueuePort {
   }
 
   private recordCompletedWait(waiter: CapacityWaiter): void {
+    this.updateLocalWaiters(waiter.origin, -1);
     if (waiter.metricsEpoch !== this.metricsEpoch) return;
     const durationMs = Math.max(0, this.nowMs() - waiter.startedAtMs);
     this.capacityWaitDurationSeconds += durationMs / MILLISECONDS_PER_SECOND;
   }
 
+  private updateLocalWaiters(origin: ApplicationJobOrigin, delta: number): number | undefined {
+    if (origin !== 'local') return undefined;
+    this.localWaiters += delta;
+    if (this.localWaiters === 0) this.localBacklogLogged = false;
+    if (!this.localBacklogLogged && this.localWaiters > Number(this.limit())) {
+      this.localBacklogLogged = true;
+      return this.localWaiters;
+    }
+    return undefined;
+  }
+
+  private logLocalBacklog(localWaiters: number): void {
+    try {
+      telemetryLogger.emit({
+        eventName: 'zlink.runtime.host.local_job_backlog_exceeded',
+        timestamp: new Date(),
+        severityNumber: SeverityNumber.WARN,
+        severityText: 'WARN',
+        body: 'Same-host jobs waiting for a host permit exceeded the application job queue maximum.',
+        attributes: {
+          source_kind: 'host',
+          source_name: 'zlink.runtime.host',
+          local_waiters: localWaiters,
+          effective_maximum: Number(this.limit())
+        }
+      });
+    } catch (error) {
+      this.loggerFailureReporter?.(error);
+    }
+  }
+
   private evaluatePressure(): void {
-    const permitsInUse = this.permitsInUse();
     const next =
       this.pressureStateValue === 'running'
-        ? permitsInUse >= this.configuration.pausePermitCount
+        ? this.permitsInUse() >= this.pauseCount
           ? 'paused'
           : 'running'
-        : permitsInUse <= this.configuration.resumePermitCount
+        : this.permitsInUse() <= this.resumeCount
           ? 'running'
           : 'paused';
     if (next === this.pressureStateValue) return;

package/src/runtime/host/entry-actor-runtime.ts

@@ -33,6 +33,10 @@ export class ZLinkEntryActorRuntimeService implements ZLinkEntryActorRuntime {
     return this.options.actorManager()?.getState(actorId)?.actor;
   }
 
+  async releaseMembers(meshName: string): Promise<void> {
+    await this.options.actorManager()?.releaseSpotActorMembers(meshName);
+  }
+
   async commitActorTransaction(actor: ZLinkActor, onJoined: () => Promise<void>): Promise<void> {
     const state = this.options.actorManager()?.getState(actor.context.actorId);
     const entryNode = this.options.spotNodeRuntime()?.primaryMeshNode;
@@ -95,6 +99,7 @@ export class ZLinkEntryActorRuntimeService implements ZLinkEntryActorRuntime {
     return {
       handled: true,
       response: await spotManager.dispatchRoutedActorPacket(
+        'remote',
         spotId,
         actorId,
         parts,