@zlink-systems/framework 0.30.3 → 0.32.0

package/src/runtime/foundation/request-runtime.ts

@@ -0,0 +1,227 @@
+import { requestTerminalError } from '../messaging/request-deadline';
+import { ZLinkFrameworkException, ZLinkFrameworkErrorKind } from '../../contracts';
+import {
+  OperationRegistry,
+  OperationCancelledError,
+  type OperationHandle,
+  type OperationObserver,
+  type PendingOperation
+} from './operation-registry';
+import { createAbortError } from '../abort';
+import {
+  identityRequestReply,
+  rejectZLinkRequestReply,
+  type ZLinkRequestResult
+} from '../messaging/request-result';
+
+/** Owns the request entries and deadline visits of one runtime instance. */
+export class RequestRuntime {
+  readonly registry = new OperationRegistry<unknown, unknown>(
+    undefined,
+    () => this.schedule(),
+    (reason) => requestTerminalError('Request', reason)
+  );
+  private timer?: NodeJS.Timeout;
+  private dueMs = Number.POSITIVE_INFINITY;
+
+  begin<T>(
+    timeoutMs: number | undefined,
+    signal?: AbortSignal,
+    existing?: PendingOperation<unknown>,
+    absoluteDeadlineMs?: number
+  ): PendingOperation<T> {
+    if (existing !== undefined) return existing as PendingOperation<T>;
+    const operation = this.registry.register(
+      timeoutMs ?? 0,
+      timeoutMs === undefined ? 'sender' : 'registry',
+      undefined,
+      absoluteDeadlineMs
+    );
+    operation.setSignal(signal);
+    return operation as PendingOperation<T>;
+  }
+
+  run<T>(
+    pending: PendingOperation<unknown>,
+    start: (receiver: OperationObserver<unknown, unknown>) => void,
+    decode: (raw: unknown) => T,
+    mapFailure: (error: unknown) => unknown,
+    destination?: OperationObserver<unknown, unknown>
+  ): Promise<T> {
+    if (!pending.isPending()) return pending.promise as Promise<T>;
+    pending.deferCompletion();
+    const receiver: OperationObserver<unknown, unknown> = {
+      operationCompleted: (_id, raw) => {
+        try {
+          const value = decode(raw);
+          if (destination !== undefined)
+            destination.operationCompleted(pending.id, value, undefined);
+          else pending.complete(value);
+        } catch (error) {
+          if (destination !== undefined) destination.operationFailed(pending.id, error, undefined);
+          else pending.fail(error);
+        }
+      },
+      operationFailed: (_id, error) => {
+        let failure = error;
+        try {
+          failure = mapFailure(error);
+        } catch (mapped) {
+          failure = mapped;
+        }
+        if (destination !== undefined) destination.operationFailed(pending.id, failure, undefined);
+        else pending.fail(failure);
+      }
+    };
+    try {
+      start(receiver);
+    } catch (error) {
+      receiver.operationFailed(pending.id, error, undefined);
+    }
+    return pending.promise as Promise<T>;
+  }
+
+  request<T>(
+    timeoutMs: number | undefined,
+    signal: AbortSignal | undefined,
+    work: (pending: PendingOperation<unknown>) => Promise<T>,
+    existing?: PendingOperation<unknown>,
+    discardLateResult?: (value: T) => void
+  ): Promise<T> {
+    if (existing !== undefined) {
+      if (!existing.isPending())
+        return existing.promise.then<T>(() => {
+          throw new OperationCancelledError(existing.id, 'Request is already terminal.');
+        });
+      return work(existing);
+    }
+    const pending = this.begin(timeoutMs, signal, existing);
+    return this.run(
+      pending,
+      (receiver) => {
+        void work(pending).then(
+          (value) => receiver.operationCompleted(pending.id, value, undefined),
+          (error) => receiver.operationFailed(pending.id, error, undefined)
+        );
+      },
+      (value) => {
+        if (!pending.isPending()) discardLateResult?.(value as T);
+        return value as T;
+      },
+      identityRequestReply
+    );
+  }
+
+  prepare<T>(
+    timeoutMs: number | undefined,
+    signal: AbortSignal | undefined,
+    start: (
+      pending: PendingOperation<unknown>,
+      receiver: OperationObserver<unknown, unknown>
+    ) => void,
+    decode: (raw: unknown) => T,
+    mapFailure: (error: unknown) => unknown,
+    existing?: PendingOperation<unknown>,
+    destination?: OperationObserver<unknown, unknown>
+  ): ZLinkRequestResult<T> {
+    if (signal?.aborted === true && destination === undefined)
+      return rejectZLinkRequestReply(signal.reason ?? createAbortError());
+    const operation = this.begin(timeoutMs, signal, existing);
+    if (signal?.aborted === true && destination !== undefined)
+      destination.operationFailed(operation.id, signal.reason ?? createAbortError(), undefined);
+    const pending =
+      signal?.aborted === true
+        ? operation.promise
+        : this.run(
+            operation,
+            (receiver) => start(operation, receiver),
+            decode,
+            mapFailure,
+            destination
+          );
+    return {
+      pending,
+      decode: identityRequestReply<T>,
+      mapFailure: identityRequestReply,
+      operation: operation as PendingOperation<T>
+    };
+  }
+
+  observe<T, R = T>(
+    timeoutMs: number,
+    submit: (entry: OperationHandle<T>, receiver: OperationObserver<T, unknown>) => void,
+    onReply: (value: T) => R,
+    onFailed: (error: unknown) => R
+  ): Promise<R> {
+    return new Promise<R>((resolve, reject) => {
+      const observer: OperationObserver<T, unknown> = {
+        operationCompleted: (_id, value) => {
+          try {
+            resolve(onReply(value));
+          } catch (error) {
+            reject(error);
+          }
+        },
+        operationFailed: (_id, reason) => {
+          try {
+            resolve(onFailed(requestTerminalError('Request', reason)));
+          } catch (error) {
+            reject(error);
+          }
+        }
+      };
+      const id = this.registry.registerObserved(timeoutMs, observer);
+      const entry = this.registry.handle(id) as OperationHandle<T>;
+      try {
+        submit(entry, {
+          operationCompleted: (_id, value) => {
+            if (!entry.complete(value)) observer.operationCompleted(id, value, undefined);
+          },
+          operationFailed: (_id, error) => entry.fail(error)
+        });
+      } catch (error) {
+        entry.fail(error);
+      }
+    });
+  }
+
+  operations<T, Context = unknown>(): OperationRegistry<T, Context> {
+    return this.registry as unknown as OperationRegistry<T, Context>;
+  }
+
+  close(
+    reason: unknown = new ZLinkFrameworkException(
+      ZLinkFrameworkErrorKind.ShuttingDown,
+      'Request runtime closed.'
+    )
+  ): void {
+    this.registry.clear(reason);
+    this.schedule();
+  }
+
+  private schedule(): void {
+    const next = this.registry.nextDeadlineMs();
+    if (next < this.dueMs) {
+      clearTimeout(this.timer);
+      this.timer = undefined;
+      this.dueMs = next;
+      if (Number.isFinite(next))
+        this.timer = setTimeout(
+          this.visit,
+          Math.min(0x7fff_ffff, Math.ceil(next - performance.now()))
+        );
+    }
+    if (this.timer !== undefined) {
+      if (this.registry.size === 0) this.timer.unref();
+      else this.timer.ref();
+    }
+  }
+
+  private readonly visit = (): void => {
+    this.timer = undefined;
+    this.dueMs = Number.POSITIVE_INFINITY;
+    const now = performance.now();
+    this.registry.expire(now, now + 1);
+    this.schedule();
+  };
+}

package/src/runtime/foundation/service-discovery-registry.ts

@@ -1,3 +1,4 @@
+import { inspectRuntime } from '../internal-observation';
 import {
   isValidPublicWeight,
   ZLINK_MAX_PUBLIC_WEIGHT
@@ -36,6 +37,12 @@ export interface SelectedClientServer {
 
 /** Dedicated discovery state; fanout publishers never reuse RouteMesh peer rows. */
 export class ServiceDiscoveryRegistry {
+  [inspectRuntime]() {
+    return Object.freeze({
+      clientServerObservers: this.clientServerChanged.size
+    });
+  }
+
   private readonly clientServerChanged = new Set<(error?: Error) => void>();
 
   onClientServerChanged(listener: (error?: Error) => void): () => void {

package/src/runtime/foundation/service-liveness-registry.ts

@@ -60,7 +60,7 @@ const NO_LIVENESS_WORK: ServiceLivenessTick = Object.freeze({
 });
 
 export class ServiceLivenessRegistry {
-  private readonly peers = new Map<string, ServiceLivenessConnection>();
+  readonly peers = new Map<string, ServiceLivenessConnection>();
   private nextProbeId = 1n;
   /**
    * The earliest probe or timeout among admitted peers. Received traffic only

package/src/runtime/foundation/service-mailbox.ts

@@ -9,6 +9,7 @@ export interface ServiceMailboxDispatchOwner {
 }
 
 export interface ServiceMailboxRecord {
+  receivedAt?: Date;
   readonly sourcePreparationOwner?: ServiceMailboxDispatchOwner;
   readonly ingress?: RawServiceIngressRecord;
   readonly owner: string;
@@ -18,7 +19,7 @@ export interface ServiceMailboxRecord {
   readonly requestSource?: import('../locations/request-source-resolver').ZLinkRequestSourceFence;
   readonly sourceRoute?: Uint8Array;
   readonly requestSequence?: bigint;
-  readonly reply?: (parts: readonly Uint8Array[]) => void;
+  readonly reply?: (parts: readonly (Uint8Array | string)[]) => void;
   readonly correlation?: bigint;
   readonly localReply?: (
     terminalResult: number,
@@ -82,7 +83,10 @@ export class ServiceMailbox {
   private nextRelocationSerial = 1n;
   private closed = false;
 
-  constructor(private readonly onReady?: (domain: ServiceMailboxDomain) => void) {
+  constructor(
+    private readonly onReady?: (domain: ServiceMailboxDomain) => void,
+    private readonly receivedAtClock: () => Date = () => new Date()
+  ) {
     // Host ApplicationJobQueue permits own ordinary-record admission. The
     // Framework-owned payload lifetime does not retain Core HWM credit. A
     // second mailbox cap would reject already-admitted work before the shared
@@ -105,6 +109,7 @@ export class ServiceMailbox {
     const target = this.domain(record.domain);
     const bytes = retainedBytes(record);
     if (this.closed) return false;
+    record.receivedAt ??= this.receivedAtClock();
     if (record.domain === 'application') {
       if (this.relocatedOwners.has(record.owner)) return false;
       const relocation = this.relocationSeals.get(record.owner);

package/src/runtime/foundation/service-relocation-object-owner.ts

@@ -31,7 +31,7 @@ export interface ServiceRelocationCaptureUnit {
   seal(signal?: AbortSignal): Promise<ServiceRelocationSealedWork>;
   captureApplicationState(signal?: AbortSignal): Promise<Uint8Array>;
   commitSeal(followAuthorities: ReadonlyMap<string, ZLinkAuthoritySnapshot>): Promise<void> | void;
-  abortSeal(): Promise<void> | void;
+  abortSeal(onCleanupError?: (error: unknown) => void): Promise<void> | void;
   /** Expired-owner terminal: pending requests fail once and dispatch stays closed. */
   discardSeal?(reason: unknown): Promise<void> | void;
 }
@@ -55,13 +55,13 @@ export class ServiceCapturedObjectRelocation {
     this.terminal = 'committed';
   }
 
-  async abortSource(): Promise<void> {
+  async abortSource(onCleanupError?: (error: unknown) => void): Promise<void> {
     if (this.terminal === 'aborted') return;
     if (this.terminal !== 'open') {
       throw new Error(`A ${this.terminal} relocation source capture cannot be aborted.`);
     }
     try {
-      await abortSourceSeals(this.units);
+      await abortSourceSeals(this.units, onCleanupError);
     } finally {
       this.terminal = 'aborted';
     }
@@ -218,17 +218,32 @@ export class ServiceRelocationObjectCaptureOwner {
   }
 }
 
-async function abortSourceSeals(units: readonly ServiceRelocationCaptureUnit[]): Promise<void> {
+async function abortSourceSeals(
+  units: readonly ServiceRelocationCaptureUnit[],
+  onCleanupError?: (error: unknown) => void
+): Promise<void> {
   const failures: unknown[] = [];
+  const cleanupFailures: unknown[] = [];
   for (const unit of [...units].reverse()) {
     try {
-      await unit.abortSeal();
+      await unit.abortSeal((error) => cleanupFailures.push(error));
     } catch (error) {
       failures.push(error);
     }
   }
   if (failures.length !== 0) {
-    throw new AggregateError(failures, 'Relocation source rollback was incomplete.');
+    throw new AggregateError(
+      [...failures, ...cleanupFailures],
+      'Relocation source rollback was incomplete.'
+    );
+  }
+  if (cleanupFailures.length !== 0) {
+    const cleanup = new AggregateError(
+      cleanupFailures,
+      'Relocation source rollback was incomplete.'
+    );
+    if (onCleanupError === undefined) throw cleanup;
+    onCleanupError(cleanup);
   }
 }
 

package/src/runtime/foundation/service-runtime-contracts.ts

@@ -1,3 +1,4 @@
+import type { PendingOperation } from './operation-registry';
 import type { RoutingId } from '../../contracts';
 import type { Message } from '../../contracts/Common/Message';
 import type { ApplicationJobPermitPort } from '../application-jobs/contracts';
@@ -252,6 +253,7 @@ export interface ZLinkMessageFollowOrigin {
 }
 
 export interface ReceiveRecord {
+  readonly receivedAt?: Date;
   /** Resolve record prerequisites in owner order, outside an application handler turn. */
   readonly prepareDispatch?: (
     this: ReceiveRecord,
@@ -275,6 +277,7 @@ export interface ReceiveRecord {
   readonly kindData: ReceiveKindData | null;
   readonly terminalResult: number;
   readonly failureErrno: number;
+  readonly failureMessage?: string;
   readonly parts: Message[];
   /** Returns whether the local operation can still accept a lifecycle reply. */
   readonly isPending?: () => boolean;
@@ -298,7 +301,11 @@ export interface ReceiveRecord {
     replyContentType?: string
   ): SubmitResult;
   /** Emits a typed command-20 terminal for a lifecycle request before admission. */
-  replyFailure?(terminalResult: number, failureCode: number): SubmitResult;
+  replyFailure?(
+    terminalResult: number,
+    failureCode: number,
+    tail?: import('./service-stateful-wire-codec').ServiceStatefulReplyTail
+  ): SubmitResult;
 }
 
 export interface ClaimReceiveResult {
@@ -436,6 +443,7 @@ export interface ServiceSpot {
     options?: {
       flags?: number;
       timeoutMs?: number;
+      readonly requestOperation?: PendingOperation<unknown>;
       applicationMetadata?: Buffer;
       onAdmitted?: () => void;
     }
@@ -459,11 +467,13 @@ export interface ServiceSpot {
     options?: {
       flags?: number;
       timeoutMs?: number;
+      readonly requestOperation?: PendingOperation<unknown>;
       applicationMetadata?: Buffer;
       routeFence?: ServiceDirectSpotRouteFence;
       onAdmitted?: () => void;
       entrySpot?: boolean;
-    }
+    },
+    completion?: import('../backend/mesh-completion-table').ZLinkMeshCompletionHandler
   ): MeshOperationId;
   publish(
     channelName: string,

package/src/runtime/foundation/service-stateful-registry.ts

@@ -62,6 +62,11 @@ export interface ServiceSessionBinding {
   readonly membershipEpoch: bigint;
 }
 
+interface ServiceSessionBindingEntry {
+  readonly binding: ServiceSessionBinding;
+  lastAcceptedSequence: bigint;
+}
+
 export interface ServiceObjectReservation {
   readonly id: bigint;
   readonly objectKind: 'actor' | 'userSpot' | 'instanceSpot';
@@ -112,7 +117,7 @@ export class ServiceStatefulRegistry {
   >();
   private readonly actors = new Map<string, ServiceActorState>();
   private readonly actorTypes = new Map<string, string>();
-  private readonly bindings = new Map<string, ServiceSessionBinding>();
+  private readonly bindings = new Map<string, ServiceSessionBindingEntry>();
   private readonly reservations = new Map<string, ServiceObjectReservation>();
   private readonly spotGenerations = new Map<string, bigint>();
   private readonly actorGenerations = new Map<string, bigint>();
@@ -408,7 +413,7 @@ export class ServiceStatefulRegistry {
     );
     requireText(binding.sessionOwnerNodeRid, 'sessionOwnerNodeRid');
     requirePositive(binding.bindingGeneration, 'bindingGeneration');
-    const current = this.bindings.get(actorKey(binding.actor));
+    const current = this.bindings.get(actorKey(binding.actor))?.binding;
     if (
       current !== undefined &&
       current.sessionOwnerNodeRid === binding.sessionOwnerNodeRid &&
@@ -423,7 +428,7 @@ export class ServiceStatefulRegistry {
       throw new ServiceStaleGenerationError('binding', binding.actor.actorId);
     }
     const accepted = Object.freeze({ ...binding });
-    this.bindings.set(actorKey(binding.actor), accepted);
+    this.bindings.set(actorKey(binding.actor), { binding: accepted, lastAcceptedSequence: 0n });
     return accepted;
   }
 
@@ -432,10 +437,29 @@ export class ServiceStatefulRegistry {
   }
 
   private bindingCore(actor: ServiceActorRef): ServiceSessionBinding | undefined {
-    const binding = this.bindings.get(actorKey(actor));
+    const binding = this.bindings.get(actorKey(actor))?.binding;
     return binding?.actor.generation === actor.generation ? binding : undefined;
   }
 
+  /** The Actor owner admits each relayed Session sequence once per binding. */
+  acceptBoundSessionSequence(
+    actor: ServiceActorRef,
+    expectedBindingGeneration: bigint,
+    sequence: bigint
+  ): boolean {
+    const entry = this.bindings.get(actorKey(actor));
+    if (
+      entry === undefined ||
+      entry.binding.actor.generation !== actor.generation ||
+      entry.binding.bindingGeneration !== expectedBindingGeneration ||
+      sequence <= entry.lastAcceptedSequence
+    ) {
+      return false;
+    }
+    entry.lastAcceptedSequence = sequence;
+    return true;
+  }
+
   unbindSession(
     actor: ServiceActorRef,
     expectedBindingGeneration: bigint,
@@ -443,8 +467,9 @@ export class ServiceStatefulRegistry {
     expectedSessionOwnerNodeRid?: string
   ): boolean {
     const key = actorKey(actor);
-    const binding = this.bindings.get(key);
-    if (binding === undefined) return false;
+    const entry = this.bindings.get(key);
+    if (entry === undefined) return false;
+    const binding = entry.binding;
     if (
       binding.bindingGeneration !== expectedBindingGeneration ||
       (expectedSessionRid !== undefined &&
@@ -577,16 +602,16 @@ export class ServiceStatefulRegistry {
 
   private updateBindingMembership(actor: ServiceActorState): void {
     const key = actorKey(actor.ref);
-    const binding = this.bindings.get(key);
-    if (binding === undefined) return;
-    this.bindings.set(
-      key,
-      Object.freeze({
-        ...binding,
+    const entry = this.bindings.get(key);
+    if (entry === undefined) return;
+    this.bindings.set(key, {
+      binding: Object.freeze({
+        ...entry.binding,
         actor: actor.ref,
         membershipEpoch: actor.membershipEpoch
-      })
-    );
+      }),
+      lastAcceptedSequence: entry.lastAcceptedSequence
+    });
   }
 }
 
@@ -619,20 +644,28 @@ export const RELOCATION_REPLY_RETENTION_MS = 24 * 60 * 60 * 1_000;
 export class ServiceTerminalOperationRegistry<T, Context = unknown> {
   private readonly operations: OperationRegistry<T, Context>;
 
-  constructor(operations = new OperationRegistry<T, Context>()) {
+  constructor(
+    operations = new OperationRegistry<T, Context>(),
+    private readonly release = () => operations.close('Stateful runtime closed.')
+  ) {
     this.operations = operations;
   }
 
   register(
     timeoutMs: number,
     timeoutOwner: 'registry' | 'sender' = 'registry',
-    context?: Context
+    context?: Context,
+    absoluteDeadlineMs?: number
   ): PendingOperation<T> {
-    return this.operations.register(timeoutMs, timeoutOwner, context);
+    return this.operations.register(timeoutMs, timeoutOwner, context, absoluteDeadlineMs);
+  }
+
+  setContext(id: bigint, context: Context): void {
+    this.operations.setContext(id, context);
   }
 
   reply(id: bigint, value: T, terminalContextRetentionMs = 0): boolean {
-    return this.operations.complete(id, value, terminalContextRetentionMs);
+    return this.operations.reply(id, value, terminalContextRetentionMs);
   }
 
   fail(id: bigint, error: unknown): boolean {
@@ -664,7 +697,7 @@ export class ServiceTerminalOperationRegistry<T, Context = unknown> {
   }
 
   close(): void {
-    this.operations.close('Stateful runtime closed.');
+    this.release();
   }
 
   failAll(reason: unknown): void {