@zlink-systems/framework 0.30.3 → 0.32.0

package/src/runtime/channels/spot-route-dispatch-strategy.ts

@@ -1,3 +1,6 @@
+import type { RequestRuntime } from '../foundation/request-runtime';
+import type { OperationObserver, PendingOperation } from '../foundation/operation-registry';
+import { identityRequestReply } from '../messaging/request-result';
 import type { Message } from '../../contracts/Common/Message';
 import type { RoutingId } from '../../contracts';
 import type { ZLinkFrameworkRegistration } from '../configuration';
@@ -42,6 +45,7 @@ export interface ZLinkLocalSpotRouteDispatcher {
     context: {
       readonly channelName: string;
       readonly signal?: AbortSignal;
+      readonly requestOperation?: PendingOperation<unknown>;
       readonly onAdmitted?: () => void;
       readonly metadata?: ReadonlyMap<string, string>;
     }
@@ -49,6 +53,7 @@ export interface ZLinkLocalSpotRouteDispatcher {
 }
 
 export interface ZLinkSpotRouteDispatchStrategyOptions {
+  readonly requests: RequestRuntime;
   readonly registration: ZLinkFrameworkRegistration;
   readonly sockets: ZLinkChannelSocketRegistry;
   readonly codecs: ZLinkChannelEnvelopeCodecRegistry;
@@ -66,17 +71,29 @@ export class ZLinkSpotRouteDispatchStrategy {
   private readonly bridgeTransport: ZLinkSpotRouteBridgeTransport;
 
   constructor(private readonly options: ZLinkSpotRouteDispatchStrategyOptions) {
-    this.sourceSpotRouter = new ZLinkSourceSpotRouter(options.registration.requestTimeoutMs);
+    this.sourceSpotRouter = new ZLinkSourceSpotRouter(
+      options.registration.requestTimeoutMs,
+      options.requests
+    );
     this.targets = new ZLinkSpotRouteTargetResolver(options.registration);
-    this.spotNodeTransport = new ZLinkSpotNodeRouteTransport(options.registration, this.targets);
-    this.routerSocketTransport = new ZLinkRouterSocketSpotRouteTransport(options.sockets);
+    this.spotNodeTransport = new ZLinkSpotNodeRouteTransport(
+      options.registration,
+      this.targets,
+      options.requests
+    );
+    this.routerSocketTransport = new ZLinkRouterSocketSpotRouteTransport(
+      options.sockets,
+      options.requests
+    );
     this.localTransport = new ZLinkLocalSpotRouteTransport(
       options.localSpotRouteDispatcher,
-      options.registration.requestTimeoutMs
+      options.registration.requestTimeoutMs,
+      options.requests
     );
     this.bridgeTransport = new ZLinkSpotRouteBridgeTransport(
       options.spotRouteBridges,
-      options.registration.requestTimeoutMs
+      options.registration.requestTimeoutMs,
+      options.requests
     );
   }
 
@@ -165,29 +182,60 @@ export class ZLinkSpotRouteDispatchStrategy {
     timeoutMs: number | undefined,
     signal?: AbortSignal,
     metadata?: ReadonlyMap<string, string>,
-    onAdmitted?: () => void
+    onAdmitted?: () => void,
+    requestOperation?: PendingOperation<unknown>,
+    destination?: OperationObserver<unknown, unknown>
   ): Promise<TReply> {
-    return runWithOutboundFlow(this.options.flowCreationEnabled?.() ?? true, () =>
-      this.routeRequestToSpotScoped<TReply>(
-        spotRouteTarget,
-        packetName,
-        request,
-        timeoutMs,
-        signal,
-        metadata,
-        onAdmitted
-      )
-    );
+    return runWithOutboundFlow(this.options.flowCreationEnabled?.() ?? true, () => {
+      if (requestOperation !== undefined) {
+        return this.options.requests.run<TReply>(
+          requestOperation,
+          (receiver) => {
+            this.routeRequestToSpotScoped<TReply>(
+              spotRouteTarget,
+              packetName,
+              request,
+              timeoutMs,
+              signal,
+              metadata,
+              onAdmitted,
+              requestOperation,
+              receiver
+            );
+          },
+          (value) => value as TReply,
+          identityRequestReply,
+          destination
+        );
+      }
+      try {
+        return this.routeRequestToSpotScoped<TReply>(
+          spotRouteTarget,
+          packetName,
+          request,
+          timeoutMs,
+          signal,
+          metadata,
+          onAdmitted,
+          requestOperation,
+          destination
+        );
+      } catch (error) {
+        return Promise.reject(error);
+      }
+    });
   }
 
-  private async routeRequestToSpotScoped<TReply>(
+  private routeRequestToSpotScoped<TReply>(
     spotRouteTarget: ZLinkSpotRouteTarget,
     packetName: string | undefined,
     request: unknown,
     timeoutMs: number | undefined,
     signal?: AbortSignal,
     metadata?: ReadonlyMap<string, string>,
-    onAdmitted?: () => void
+    onAdmitted?: () => void,
+    requestOperation?: PendingOperation<unknown>,
+    receiver?: OperationObserver<unknown, unknown>
   ): Promise<TReply> {
     throwIfAborted(signal);
     const localSpotRouteNode = this.targets.localRouteNode(spotRouteTarget);
@@ -199,7 +247,9 @@ export class ZLinkSpotRouteDispatchStrategy {
         request,
         timeoutMs,
         signal,
-        onAdmitted
+        onAdmitted,
+        requestOperation,
+        receiver
       );
     }
     const codecs = codecsForFrameworkPacket(packetName, this.options.codecs);
@@ -222,7 +272,9 @@ export class ZLinkSpotRouteDispatchStrategy {
         codecs,
         timeoutMs,
         signal,
-        onAdmitted
+        onAdmitted,
+        requestOperation,
+        receiver
       );
       if (namedSpotNodeRequest !== undefined) return namedSpotNodeRequest;
     }
@@ -233,7 +285,9 @@ export class ZLinkSpotRouteDispatchStrategy {
         codecs,
         timeoutMs,
         signal,
-        onAdmitted
+        onAdmitted,
+        requestOperation,
+        receiver
       );
     }
     if (this.targets.hasBoundRouteRouter(spotRouteTarget.routerChannelId)) {
@@ -244,7 +298,9 @@ export class ZLinkSpotRouteDispatchStrategy {
         ZLINK_SEND_DONT_WAIT,
         timeoutMs,
         signal,
-        onAdmitted
+        onAdmitted,
+        requestOperation,
+        receiver
       );
     }
     const spotNodeRequest = this.spotNodeTransport.request<TReply>(
@@ -253,7 +309,9 @@ export class ZLinkSpotRouteDispatchStrategy {
       codecs,
       timeoutMs,
       signal,
-      onAdmitted
+      onAdmitted,
+      requestOperation,
+      receiver
     );
     if (spotNodeRequest !== undefined) {
       return spotNodeRequest;
@@ -265,7 +323,9 @@ export class ZLinkSpotRouteDispatchStrategy {
       0,
       timeoutMs,
       signal,
-      onAdmitted
+      onAdmitted,
+      requestOperation,
+      receiver
     );
   }
 
@@ -276,7 +336,9 @@ export class ZLinkSpotRouteDispatchStrategy {
     request: unknown,
     timeoutMs: number | undefined,
     signal?: AbortSignal,
-    onAdmitted?: () => void
+    onAdmitted?: () => void,
+    requestOperation?: PendingOperation<unknown>,
+    destination?: OperationObserver<unknown, unknown>
   ): Promise<TReply> {
     return this.sourceSpotRouter.request<TReply>(
       sourceSpot,
@@ -285,7 +347,9 @@ export class ZLinkSpotRouteDispatchStrategy {
       request,
       timeoutMs,
       signal,
-      onAdmitted
+      onAdmitted,
+      requestOperation,
+      destination
     );
   }
 
@@ -305,7 +369,8 @@ export class ZLinkSpotRouteDispatchStrategy {
     request: Message,
     timeoutMs: number | undefined,
     signal?: AbortSignal,
-    onAdmitted?: () => void
+    onAdmitted?: () => void,
+    requestOperation?: PendingOperation<unknown>
   ): Promise<readonly Message[]> {
     return this.sourceSpotRouter.requestRaw(
       sourceSpot,
@@ -313,7 +378,8 @@ export class ZLinkSpotRouteDispatchStrategy {
       request,
       timeoutMs,
       signal,
-      onAdmitted
+      onAdmitted,
+      requestOperation
     );
   }
 
@@ -322,7 +388,8 @@ export class ZLinkSpotRouteDispatchStrategy {
     request: Message,
     timeoutMs: number | undefined,
     signal?: AbortSignal,
-    onAdmitted?: () => void
+    onAdmitted?: () => void,
+    requestOperation?: PendingOperation<unknown>
   ): Promise<readonly Message[]> {
     throwIfAborted(signal);
     if (this.targets.hasNamedSpotNode(spotRouteTarget.routerChannelId)) {
@@ -331,7 +398,8 @@ export class ZLinkSpotRouteDispatchStrategy {
         request,
         timeoutMs,
         signal,
-        onAdmitted
+        onAdmitted,
+        requestOperation
       );
       if (namedSpotNodeRequest !== undefined) return namedSpotNodeRequest;
     }
@@ -341,7 +409,8 @@ export class ZLinkSpotRouteDispatchStrategy {
         request,
         timeoutMs,
         signal,
-        onAdmitted
+        onAdmitted,
+        requestOperation
       );
     }
     const spotNodeRequest = this.spotNodeTransport.requestRaw(
@@ -349,7 +418,8 @@ export class ZLinkSpotRouteDispatchStrategy {
       request,
       timeoutMs,
       signal,
-      onAdmitted
+      onAdmitted,
+      requestOperation
     );
     if (spotNodeRequest !== undefined) {
       return spotNodeRequest;
@@ -360,7 +430,8 @@ export class ZLinkSpotRouteDispatchStrategy {
       request,
       timeoutMs,
       signal,
-      onAdmitted
+      onAdmitted,
+      requestOperation
     );
   }
 }

package/src/runtime/diagnostics/dispatch-error-details.ts

@@ -14,6 +14,16 @@ export function dispatchReasonFromError(error: unknown): ZLinkDispatchErrorReaso
   if (origin === ZLinkFrameworkInternalErrorKind.PayloadDecodeFailed)
     return ZLinkDispatchErrorReason.PayloadDecodeFailed;
   if (
+    origin === ZLinkFrameworkInternalErrorKind.HandlerNotFound ||
+    origin === ZLinkFrameworkInternalErrorKind.RouteHandlerNotFound ||
+    origin === ZLinkFrameworkInternalErrorKind.ActorDispatchHandlerNotFound
+  )
+    return ZLinkDispatchErrorReason.HandlerMissing;
+  if (
+    origin === ZLinkFrameworkInternalErrorKind.ActorRouteNotFound ||
+    origin === ZLinkFrameworkInternalErrorKind.SpotRouteNotFound ||
+    origin === ZLinkFrameworkInternalErrorKind.RequestTargetNotFound ||
+    origin === ZLinkFrameworkInternalErrorKind.MeshNotFound ||
     origin === ZLinkFrameworkInternalErrorKind.SpotGenerationStale ||
     origin === ZLinkFrameworkInternalErrorKind.ActorGenerationStale ||
     origin === ZLinkFrameworkInternalErrorKind.SpotMoving ||
@@ -22,8 +32,6 @@ export function dispatchReasonFromError(error: unknown): ZLinkDispatchErrorReaso
   )
     return ZLinkDispatchErrorReason.StaleTarget;
   switch (error.kind) {
-    case ZLinkFrameworkErrorKind.NotFound:
-      return ZLinkDispatchErrorReason.HandlerMissing;
     case ZLinkFrameworkErrorKind.ProtocolError:
       return ZLinkDispatchErrorReason.InvalidFrame;
     case ZLinkFrameworkErrorKind.ShuttingDown:

package/src/runtime/diagnostics/message-flow-bootstrap.ts

@@ -0,0 +1,6 @@
+import { logs } from '@opentelemetry/api-logs';
+
+// Loaded before the application's standard preloads in this worker only. The
+// public API returns this proxy until a logger provider is registered, then the
+// registered provider itself. No provider is changed or called to probe it.
+export const initialLoggerProvider = logs.getLoggerProvider();

package/src/runtime/diagnostics/message-flow-delivery.ts

@@ -0,0 +1,171 @@
+import { Worker } from 'node:worker_threads';
+
+// The process owner counts records until the worker takes them, including batches
+// already posted to its port. Only the worker decrements in-transit records.
+export const MESSAGE_FLOW_PENDING_LIMIT = 8192;
+export const TELEMETRY_WORKER_AVAILABLE = 'zlink.telemetry.available';
+export enum TelemetryState {
+  Pending,
+  Enabled,
+  Length
+}
+export enum TelemetryCounter {
+  Overflow,
+  LoggerFailures,
+  TraceFailures,
+  FallbackFailures,
+  WorkerFailures,
+  Length
+}
+export const TELEMETRY_STATE_BYTES =
+  Math.ceil(
+    (TelemetryState.Length * Int32Array.BYTES_PER_ELEMENT) / BigUint64Array.BYTES_PER_ELEMENT
+  ) * BigUint64Array.BYTES_PER_ELEMENT;
+
+export interface ZLinkTelemetryLease {
+  readonly retired: Int32Array;
+}
+
+export interface ZLinkTelemetryDelivery<T = unknown> {
+  readonly lease: SharedArrayBuffer;
+  readonly record: T;
+}
+
+export function openTelemetryLease(): ZLinkTelemetryLease {
+  return { retired: new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT)) };
+}
+
+export function renewTelemetryLease(lease: ZLinkTelemetryLease | undefined): ZLinkTelemetryLease {
+  return lease === undefined || Atomics.load(lease.retired, 0) !== 0 ? openTelemetryLease() : lease;
+}
+
+let processOwner: ZLinkMessageFlowDelivery | undefined;
+
+export function messageFlowDelivery(): ZLinkMessageFlowDelivery {
+  return (processOwner ??= new ZLinkMessageFlowDelivery());
+}
+
+export function telemetryProviderFailureCount(): number {
+  return processOwner === undefined
+    ? 0
+    : Number(
+        Atomics.load(processOwner.counters, TelemetryCounter.LoggerFailures) +
+          Atomics.load(processOwner.counters, TelemetryCounter.TraceFailures)
+      );
+}
+
+export function retireTelemetryLease(lease: ZLinkTelemetryLease): void {
+  Atomics.store(lease.retired, 0, 1);
+  processOwner?.retire(lease);
+}
+
+export class ZLinkMessageFlowDelivery {
+  private readonly sharedState = new SharedArrayBuffer(
+    TELEMETRY_STATE_BYTES + TelemetryCounter.Length * BigUint64Array.BYTES_PER_ELEMENT
+  );
+  readonly state = new Int32Array(this.sharedState, 0, TelemetryState.Length);
+  readonly counters = new BigUint64Array(
+    this.sharedState,
+    TELEMETRY_STATE_BYTES,
+    TelemetryCounter.Length
+  );
+  readonly ready: Promise<void>;
+  private readonly worker?: Worker;
+  private failure?: Error;
+  private pending: ZLinkTelemetryDelivery[] = [];
+  private executing = false;
+  private workerReady = false;
+  private scheduled?: NodeJS.Immediate;
+
+  constructor() {
+    let published!: () => void;
+    // A failed bootstrap leaves this incomplete. Runtime processing never waits
+    // for it, and no rejected promise can escape into the host process.
+    this.ready = new Promise((resolve) => {
+      published = resolve;
+    });
+    // NODE_OPTIONS preloads precede execArgv preloads. Keep both application
+    // lists intact and capture the public default provider before either list.
+    try {
+      const bootstrap = require.resolve('./message-flow-bootstrap').replace(/\\/g, '/');
+      this.worker = new Worker(require.resolve('./message-flow-worker'), {
+        workerData: this.state.buffer,
+        env: {
+          ...process.env,
+          NODE_OPTIONS: `--require "${bootstrap}" ${process.env.NODE_OPTIONS ?? ''}`
+        }
+      });
+      this.worker.once('error', (error) => this.workerFailed(error));
+      this.worker.once('exit', (code) => {
+        if (this.failure === undefined)
+          this.workerFailed(new Error(`Telemetry worker exited with code ${code}.`));
+      });
+      this.worker.on('message', (message: unknown) => {
+        if (message !== TELEMETRY_WORKER_AVAILABLE || this.failure !== undefined) return;
+        this.workerReady = true;
+        this.executing = false;
+        this.flush();
+        published();
+      });
+      this.worker.unref();
+    } catch (error) {
+      this.workerFailed(error instanceof Error ? error : new Error(String(error)));
+    }
+  }
+
+  private workerFailed(error: Error): void {
+    if (this.failure !== undefined) return;
+    this.failure = error;
+    Atomics.add(this.counters, TelemetryCounter.WorkerFailures, 1n);
+    Atomics.store(this.state, TelemetryState.Enabled, 0);
+    this.workerReady = false;
+    this.executing = false;
+    this.pending = [];
+    Atomics.store(this.state, TelemetryState.Pending, 0);
+  }
+
+  enabled(): boolean {
+    return Atomics.load(this.state, TelemetryState.Enabled) !== 0;
+  }
+
+  accepts(lease: ZLinkTelemetryLease): boolean {
+    return this.enabled() && Atomics.load(lease.retired, 0) === 0;
+  }
+
+  submit(lease: ZLinkTelemetryLease, record: unknown): void {
+    if (!this.accepts(lease)) return;
+    if (Atomics.load(this.state, TelemetryState.Pending) === MESSAGE_FLOW_PENDING_LIMIT) {
+      Atomics.add(this.counters, TelemetryCounter.Overflow, 1n);
+      return;
+    }
+    Atomics.add(this.state, TelemetryState.Pending, 1);
+    this.pending.push({ lease: lease.retired.buffer as SharedArrayBuffer, record });
+    if (this.workerReady && !this.executing && this.scheduled === undefined) {
+      this.scheduled = setImmediate(() => {
+        this.scheduled = undefined;
+        this.flush();
+      });
+      this.scheduled.unref();
+    }
+  }
+
+  retire(lease: ZLinkTelemetryLease): void {
+    const retained = this.pending.filter((item) => item.lease !== lease.retired.buffer);
+    Atomics.sub(this.state, TelemetryState.Pending, this.pending.length - retained.length);
+    this.pending = retained;
+  }
+
+  private flush(): void {
+    if (
+      this.worker === undefined ||
+      !this.workerReady ||
+      this.executing ||
+      this.pending.length === 0
+    )
+      return;
+    const batch = this.pending;
+    this.pending = [];
+    this.executing = true;
+    this.worker.postMessage(batch);
+  }
+}