@zlink-systems/framework 0.29.0 → 0.30.0

package/src/runtime/foundation/raw-service-mesh-runtime.ts

@@ -100,6 +100,8 @@ export type RawServiceIngressHandler = (
 ) => RawServicePumpResult | undefined | Promise<RawServicePumpResult | undefined>;
 
 export interface RawServiceMeshRuntimeOptions {
+  readonly onReceiveReady?: () => void;
+  readonly infrastructureTaskRunner?: import('../spots/spot-actor-join-dispatch').ZLinkDetachedTaskRunner;
   readonly onPendingOperationsChanged?: () => void;
   readonly descriptor: ServiceNodeDescriptor;
   readonly resolveAdvertisedEndpoint?: (boundEndpoint: string) => string;
@@ -170,6 +172,19 @@ export class RawServiceMeshRuntime {
   private readonly applicationJobQueue: ApplicationJobQueuePort;
   private readonly peerAdmissionSealed?: () => boolean;
   private readonly applicationJobStop = new AbortController();
+  private receiveAcquisition?: Promise<ApplicationJobPermitPort>;
+  private receivePermit?: ApplicationJobPermitPort;
+  private readonly controlSends = new Map<
+    string,
+    {
+      readonly task: Promise<boolean>;
+      next?: () => boolean | Promise<boolean>;
+      current?: () => boolean;
+    }
+  >();
+  private infrastructureFailure?: { readonly error: unknown };
+  private readonly onReceiveReady?: () => void;
+  private readonly infrastructureTaskRunner?: RawServiceMeshRuntimeOptions['infrastructureTaskRunner'];
   private readonly onPeerNotRequired?: RawServiceMeshRuntimeOptions['onPeerNotRequired'];
   private readonly onPeerDisconnected?: RawServiceMeshRuntimeOptions['onPeerDisconnected'];
   private readonly onProtocolError?: RawServiceMeshRuntimeOptions['onProtocolError'];
@@ -183,6 +198,8 @@ export class RawServiceMeshRuntime {
   private closed = false;
 
   constructor(options: RawServiceMeshRuntimeOptions) {
+    this.onReceiveReady = options.onReceiveReady;
+    this.infrastructureTaskRunner = options.infrastructureTaskRunner;
     this.operations = new OperationRegistry(undefined, options.onPendingOperationsChanged);
     this.descriptor = options.descriptor;
     this.topology = new ServiceTopologyRegistry(options.descriptor);
@@ -371,7 +388,7 @@ export class RawServiceMeshRuntime {
     for (const listener of this.peerConnectionIntentRemoved) listener(nodeRoutingId);
   }
 
-  async announcePeer(nodeRoutingId: string): Promise<boolean> {
+  announcePeer(nodeRoutingId: string): Promise<boolean> {
     const expected = this.expectedPeers.get(nodeRoutingId);
     const generation = this.selectedRoutes.get(nodeRoutingId);
     if (
@@ -380,17 +397,27 @@ export class RawServiceMeshRuntime {
       expected.helloSubmittedGeneration === generation ||
       this.peerAdmissionSealed?.() === true
     )
-      return false;
-    const accepted = await this.send(nodeRoutingId, [
-      encodeRouteMeshAdmission(M6aServiceWireCommand.hello, this.topology.localDescriptor())
-    ]);
-    if (
-      accepted &&
+      return Promise.resolve(false);
+    const current = () =>
       this.expectedPeers.get(nodeRoutingId) === expected &&
-      this.selectedRoutes.get(nodeRoutingId) === generation
-    )
-      expected.helloSubmittedGeneration = generation;
-    return accepted;
+      this.selectedRoutes.get(nodeRoutingId) === generation;
+    return Promise.resolve(
+      this.startControlSend(
+        nodeRoutingId,
+        M6aServiceWireCommand.hello,
+        () =>
+          this.submitControl(
+            nodeRoutingId,
+            [
+              encodeRouteMeshAdmission(M6aServiceWireCommand.hello, this.topology.localDescriptor())
+            ],
+            () => {
+              if (current()) expected.helloSubmittedGeneration = generation;
+            }
+          ),
+        current
+      )
+    );
   }
 
   isPeerRouteReady(nodeRoutingId: string, lifecycleGeneration?: bigint): boolean {
@@ -400,8 +427,7 @@ export class RawServiceMeshRuntime {
     return (
       peer !== undefined &&
       (lifecycleGeneration === undefined ||
-        peer.descriptor.lifecycleGeneration === lifecycleGeneration) &&
-      this.liveness.isReady(nodeRoutingId, peer.connectionId)
+        peer.descriptor.lifecycleGeneration === lifecycleGeneration)
     );
   }
 
@@ -548,7 +574,7 @@ export class RawServiceMeshRuntime {
       signal === undefined
         ? this.applicationJobStop.signal
         : AbortSignal.any([this.applicationJobStop.signal, signal]);
-    const permit = await this.applicationJobQueue.acquire(waitSignal);
+    const permit = await this.applicationJobQueue.acquire(waitSignal, 'local');
     return ApplicationIngressRecordOwner.create(this.applicationJobQueue, permit, { close() {} });
   }
 
@@ -561,7 +587,13 @@ export class RawServiceMeshRuntime {
     parts: readonly Uint8Array[],
     timeoutMs: number
   ): Promise<readonly Buffer[]> {
-    return this.requireStarted().request(targetNodeRoutingId, parts, timeoutMs);
+    const admission = this.topology.peer(targetNodeRoutingId)?.liveness;
+    return this.requireStarted().request(
+      targetNodeRoutingId,
+      parts,
+      timeoutMs,
+      admission?.recordReceived
+    );
   }
 
   replyService(
@@ -607,6 +639,18 @@ export class RawServiceMeshRuntime {
     nowMs = performance.now(),
     observe?: RawServicePumpObserver
   ): Promise<RawServicePumpResult> {
+    const result = await this.receiveOne(nowMs, observe);
+    const acquisition = this.receiveAcquisition;
+    if (acquisition === undefined) return result;
+    // A single receive operation may await its FIFO grant. Infrastructure
+    // rounds use receiveOne directly and end while that grant is pending.
+    try {
+      await acquisition;
+    } catch (error) {
+      if (this.closed) return 'noData';
+      throw error;
+    }
+    if (this.closed) return 'noData';
     return this.receiveOne(nowMs, observe);
   }
 
@@ -616,6 +660,7 @@ export class RawServiceMeshRuntime {
 
   /** Returns whether a receive budget ended before no-data; idle ticks only maintain peers. */
   async pumpBatch(receiveReady = true, routeReady = false): Promise<boolean> {
+    this.observeInfrastructureFailure();
     if (routeReady) await this.observeSelectedRoutes();
     const startedAtMs = performance.now();
     let messages = 0;
@@ -639,8 +684,10 @@ export class RawServiceMeshRuntime {
         break;
     }
     if (!this.closed) {
-      await this.announceExpectedPeers();
-      await this.tickLiveness();
+      for (const nodeRoutingId of this.expectedPeers.keys()) {
+        if (this.topology.peer(nodeRoutingId) === undefined) await this.announcePeer(nodeRoutingId);
+      }
+      this.tickLiveness();
     }
     return receiveReady && messages > 0;
   }
@@ -650,13 +697,40 @@ export class RawServiceMeshRuntime {
     observe?: RawServicePumpObserver
   ): Promise<RawServicePumpResult> {
     const router = this.requireStarted();
-    let permit: ApplicationJobPermitPort;
-    try {
-      permit = await this.applicationJobQueue.acquire(this.applicationJobStop.signal);
-    } catch (error) {
-      if (this.closed || this.applicationJobStop.signal.aborted) return 'noData';
-      throw error;
+    this.observeInfrastructureFailure();
+    if (this.receivePermit === undefined && this.receiveAcquisition === undefined) {
+      let resolveAcquisition!: (permit: ApplicationJobPermitPort) => void;
+      let rejectAcquisition!: (error: unknown) => void;
+      const acquisition = new Promise<ApplicationJobPermitPort>((resolve, reject) => {
+        resolveAcquisition = resolve;
+        rejectAcquisition = reject;
+      });
+      const permit = this.applicationJobQueue.acquireOrResume(
+        (permit) => {
+          this.receiveAcquisition = undefined;
+          if (this.closed) permit.releaseAfterInternalProcessing();
+          else {
+            this.receivePermit = permit;
+            this.onReceiveReady?.();
+          }
+          resolveAcquisition(permit);
+        },
+        rejectAcquisition,
+        this.applicationJobStop.signal,
+        'remote'
+      );
+      if (permit !== undefined) this.receivePermit = permit;
+      else {
+        this.receiveAcquisition = acquisition;
+        acquisition.catch((error) => {
+          this.receiveAcquisition = undefined;
+          if (!this.applicationJobStop.signal.aborted) this.reportInfrastructureFailure(error);
+        });
+      }
     }
+    const permit = this.receivePermit;
+    this.receivePermit = undefined;
+    if (permit === undefined) return 'noData';
     if (this.closed) {
       permit.releaseAfterInternalProcessing();
       return 'noData';
@@ -744,7 +818,7 @@ export class RawServiceMeshRuntime {
       (received.parts.length === 1 && received.parts[0]!.byteLength === 0)
     ) {
       if (this.peerAdmissionSealed?.() === true) return 'dropped';
-      return (await this.send(received.sourceRid, [
+      return (await this.sendControl(received.sourceRid, [
         encodeRouteMeshAdmission(M6aServiceWireCommand.hello, this.topology.localDescriptor())
       ]))
         ? 'infrastructure'
@@ -772,17 +846,12 @@ export class RawServiceMeshRuntime {
           (expected.meshName !== descriptor.meshName ||
             expected.nodeRoutingId !== descriptor.nodeRoutingId)
         ) {
-          await this.send(received.sourceRid, [
+          await this.sendControl(received.sourceRid, [
             encodeReject(enumWireRejectReason('identityMismatch'))
           ]);
           return 'infrastructure';
         }
-        const result = this.admitPeer(
-          descriptor,
-          routeConnectionId(received.routeGeneration),
-          nowMs,
-          expected
-        );
+        const result = this.admitPeer(descriptor, received.routeGeneration, nowMs, expected);
         if (result !== 'admitted' && result !== 'notRequired') {
           const detail =
             result === 'invalidDescriptor'
@@ -796,7 +865,7 @@ export class RawServiceMeshRuntime {
             `RouteMesh admission rejected peer '${received.sourceRid}' as ${result}` +
               (detail === undefined ? '.' : ` (${detail}).`)
           );
-          await this.send(received.sourceRid, [encodeReject(admissionReason(result))]);
+          await this.sendControl(received.sourceRid, [encodeReject(admissionReason(result))]);
           return 'infrastructure';
         }
         if (result === 'admitted') this.selectBilateralConnection(descriptor);
@@ -804,7 +873,7 @@ export class RawServiceMeshRuntime {
           // Hello supplies only the sender's descriptor. Even NotRequired
           // peers must receive ours before the intent owner closes transport;
           // submitting a response does not guarantee delivery before close.
-          await this.send(received.sourceRid, [
+          await this.sendControl(received.sourceRid, [
             encodeRouteMeshAdmission(M6aServiceWireCommand.admit, this.topology.localDescriptor())
           ]);
         } else if (result === 'notRequired') {
@@ -819,6 +888,8 @@ export class RawServiceMeshRuntime {
       }
       const peer = this.topology.peer(received.sourceRid);
       if (peer === undefined) return 'protocolError';
+      if (peer.liveness !== undefined && peer.liveness.connectionId === received.routeGeneration)
+        this.liveness.recordReceived(peer.liveness, nowMs);
       if (
         header.command === M6aServiceWireCommand.livenessProbe ||
         header.command === M6aServiceWireCommand.livenessAck
@@ -828,11 +899,11 @@ export class RawServiceMeshRuntime {
         if (record.command === M6aServiceWireCommand.livenessProbe) {
           const ack = this.liveness.acknowledgeProbe(
             received.sourceRid,
-            peer.connectionId,
+            received.routeGeneration,
             record.probeId
           );
           if (ack === undefined) return 'protocolError';
-          const sent = await this.send(received.sourceRid, [
+          const sent = await this.sendControl(received.sourceRid, [
             livenessCodec.encodeLivenessRecord({
               command: M6aServiceWireCommand.livenessAck,
               probeId: record.probeId
@@ -840,7 +911,12 @@ export class RawServiceMeshRuntime {
           ]);
           if (!sent) return 'dropped';
         } else {
-          this.liveness.acknowledge(received.sourceRid, peer.connectionId, record.probeId, nowMs);
+          this.liveness.acknowledge(
+            received.sourceRid,
+            received.routeGeneration,
+            record.probeId,
+            nowMs
+          );
         }
         return 'infrastructure';
       }
@@ -950,21 +1026,24 @@ export class RawServiceMeshRuntime {
     return this.operations.size;
   }
 
-  async tickLiveness(nowMs = performance.now()): Promise<ServiceLivenessTick> {
+  tickLiveness(nowMs = performance.now()): ServiceLivenessTick {
     const result = this.liveness.tick(nowMs);
     this.requireStarted();
     for (const probe of result.probes) {
-      await this.send(probe.nodeRoutingId, [
+      const parts = [
         livenessCodec.encodeLivenessRecord({
           command: M6aServiceWireCommand.livenessProbe,
           probeId: probe.probeId
         })
-      ]);
+      ];
+      this.startControlSend(probe.nodeRoutingId, M6aServiceWireCommand.livenessProbe, () =>
+        this.submitControl(probe.nodeRoutingId, parts)
+      );
     }
     for (const nodeRoutingId of result.timedOutNodes) {
       const peer = this.topology.peer(nodeRoutingId);
       if (peer === undefined) continue;
-      // Liveness timeout is a semantic peer removal of the admitted route.
+      this.requireStarted().disconnectRid!(nodeRoutingId);
       this.removePeer(peer);
     }
     return result;
@@ -990,7 +1069,7 @@ export class RawServiceMeshRuntime {
       const expected = this.expectedPeers.get(nodeRoutingId);
       if (expected !== undefined) delete expected.helloSubmittedGeneration;
       const peer = this.topology.peer(nodeRoutingId);
-      if (peer !== undefined && peer.connectionId === routeConnectionId(generation)) {
+      if (peer?.liveness?.connectionId === generation) {
         this.removePeer(peer);
       }
     }
@@ -1008,9 +1087,13 @@ export class RawServiceMeshRuntime {
 
   close(): void {
     if (this.closed) return;
+    this.closed = true;
     this.applicationJobStop.abort(
       new Error('Raw service runtime application job admission stopped.')
     );
+    this.receivePermit?.releaseAfterInternalProcessing();
+    this.receivePermit = undefined;
+    for (const send of this.controlSends.values()) delete send.next;
     this.mailbox.close();
     this.operations.close('Raw service runtime closed.');
     this.serviceIngress = undefined;
@@ -1022,7 +1105,6 @@ export class RawServiceMeshRuntime {
     if (host !== undefined) host.close();
     this.router = undefined;
     this.host = undefined;
-    this.closed = true;
   }
 
   private requestToTarget(
@@ -1107,7 +1189,8 @@ export class RawServiceMeshRuntime {
       })().catch((error) => this.operations.fail(pending.id, error));
       return pending;
     }
-    if (this.topology.peer(selectedTargetNodeRoutingId) === undefined) {
+    const selectedPeer = this.topology.peer(selectedTargetNodeRoutingId);
+    if (selectedPeer === undefined) {
       // Logical target selection is the requester's decision: an RID that is
       // not an admitted peer is not a target. Whether an admitted target's
       // route can carry the request is Core's REQUEST result.
@@ -1118,11 +1201,14 @@ export class RawServiceMeshRuntime {
       return pending;
     }
     const parts = [header, encodedPayload];
-    let router: ZLinkRawRouterPort;
     let request: Promise<readonly Uint8Array[]>;
     try {
-      router = this.requireStarted();
-      request = router.request(selectedTargetNodeRoutingId, parts, timeoutMs);
+      request = this.requireStarted().request(
+        selectedTargetNodeRoutingId,
+        parts,
+        timeoutMs,
+        selectedPeer.liveness?.recordReceived
+      );
     } catch (error) {
       this.operations.fail(pending.id, error);
       return pending;
@@ -1182,7 +1268,7 @@ export class RawServiceMeshRuntime {
 
   private admitPeer(
     descriptor: ServiceNodeDescriptor,
-    connectionId: string,
+    connectionId: string | bigint,
     nowMs: number,
     expected?: {
       readonly endpoint?: string;
@@ -1202,12 +1288,18 @@ export class RawServiceMeshRuntime {
         `RouteMesh peer '${descriptor.nodeRoutingId}' descriptor revision is stale or conflicting.`
       );
     }
-    const result = this.topology.admit(descriptor, connectionId, expected);
-    if (result === 'admitted') {
-      this.liveness.admit(descriptor.nodeRoutingId, connectionId, nowMs);
-      this.liveness.requestProbe(descriptor.nodeRoutingId, connectionId, nowMs);
-    } else if (result === 'notRequired' && previous !== undefined) {
-      this.liveness.disconnect(descriptor.nodeRoutingId, previous.connectionId);
+    const topologyConnectionId =
+      typeof connectionId === 'bigint' ? `route:${connectionId}` : connectionId;
+    const result = this.topology.admit(descriptor, topologyConnectionId, expected, () => {
+      const admission = this.liveness.admit(descriptor.nodeRoutingId, connectionId, nowMs);
+      this.liveness.recordReceived(admission, nowMs);
+      return admission;
+    });
+    if (result === 'notRequired' && previous !== undefined) {
+      this.liveness.disconnect(
+        descriptor.nodeRoutingId,
+        previous.liveness?.connectionId ?? previous.connectionId
+      );
     }
     return result;
   }
@@ -1244,7 +1336,10 @@ export class RawServiceMeshRuntime {
 
   private removePeer(peer: AdmittedServicePeer): void {
     this.topology.disconnect(peer.descriptor.nodeRoutingId, peer.connectionId);
-    this.liveness.disconnect(peer.descriptor.nodeRoutingId, peer.connectionId);
+    this.liveness.disconnect(
+      peer.descriptor.nodeRoutingId,
+      peer.liveness?.connectionId ?? peer.connectionId
+    );
     this.onPeerDisconnected?.(
       peer.descriptor.nodeRoutingId,
       peer.descriptor.advertisedEndpoint,
@@ -1257,18 +1352,129 @@ export class RawServiceMeshRuntime {
     return this.router;
   }
 
-  private async send(targetNodeRoutingId: string, parts: readonly Uint8Array[]): Promise<boolean> {
+  private async sendControl(
+    targetNodeRoutingId: string,
+    parts: readonly Uint8Array[]
+  ): Promise<boolean> {
+    return this.startControlSend(targetNodeRoutingId, decodeHeader(parts[0]!).command, () =>
+      this.submitControl(targetNodeRoutingId, parts)
+    );
+  }
+
+  private submitControl(
+    target: string,
+    parts: readonly Uint8Array[],
+    accepted?: () => void
+  ): boolean | Promise<boolean> {
+    let submission;
     try {
-      await this.requireStarted().send(targetNodeRoutingId, parts);
-      return true;
+      submission = this.requireStarted().submitSend(target, parts);
     } catch (error) {
-      if (isZLinkBackendResultError(error) && error.operation === 'submit') {
-        const terminal = submitToRequestResult(error.result, error.phase);
-        if (terminal === RequestResult.NotConnected || terminal === RequestResult.NotFound)
-          return false;
+      return this.sendFailure(error);
+    }
+    if (submission.result === SubmitResult.Ok) {
+      accepted?.();
+      return true;
+    }
+    return submission.admitted.then(
+      () => {
+        accepted?.();
+        return true;
+      },
+      (error) => this.sendFailure(error)
+    );
+  }
+
+  private startControlSend(
+    target: string,
+    command: number,
+    attempt: () => boolean | Promise<boolean>,
+    current?: () => boolean
+  ): boolean {
+    const key = JSON.stringify([target, command]);
+    const existing = this.controlSends.get(key);
+    if (
+      existing !== undefined &&
+      (command !== M6aServiceWireCommand.hello || existing.current?.() === true)
+    ) {
+      if (command !== M6aServiceWireCommand.hello) {
+        existing.next = attempt;
+        existing.current = current;
       }
-      throw error;
+      return true;
+    }
+    const first = attempt();
+    if (typeof first === 'boolean') return first;
+    const owner = this;
+    const state: {
+      task: Promise<boolean>;
+      next?: () => boolean | Promise<boolean>;
+      current?: () => boolean;
+    } = {
+      current,
+      task: first.then(function drain(accepted): boolean | Promise<boolean> {
+        if (!owner.closed && state.next !== undefined) {
+          const next = state.next;
+          delete state.next;
+          return Promise.resolve(next()).then(drain);
+        }
+        if (owner.controlSends.get(key) === state) owner.controlSends.delete(key);
+        return accepted;
+      })
+    };
+    this.controlSends.set(key, state);
+    state.task.then(
+      () => {},
+      (error) => {
+        if (this.controlSends.get(key) === state) this.controlSends.delete(key);
+        if (this.infrastructureTaskRunner === undefined) this.reportInfrastructureFailure(error);
+      }
+    );
+    this.infrastructureTaskRunner?.runDetached('RouteMesh control SEND', async () => {
+      await state.task;
+    });
+    return true;
+  }
+
+  private reportInfrastructureFailure(error: unknown): void {
+    this.infrastructureFailure = {
+      error:
+        this.infrastructureFailure === undefined
+          ? error
+          : new AggregateError(
+              [this.infrastructureFailure.error, error],
+              'RouteMesh infrastructure failed.'
+            )
+    };
+    this.onReceiveReady?.();
+  }
+
+  private observeInfrastructureFailure(): void {
+    const failure = this.infrastructureFailure;
+    this.infrastructureFailure = undefined;
+    if (failure !== undefined) throw failure.error;
+  }
+
+  private send(targetNodeRoutingId: string, parts: readonly Uint8Array[]): Promise<boolean> {
+    return this.requireStarted()
+      .send(targetNodeRoutingId, parts)
+      .then(
+        () => {
+          return true;
+        },
+        (error) => {
+          return this.sendFailure(error);
+        }
+      );
+  }
+
+  private sendFailure(error: unknown): false {
+    if (isZLinkBackendResultError(error) && error.operation === 'submit') {
+      const terminal = submitToRequestResult(error.result, error.phase);
+      if (terminal === RequestResult.NotConnected || terminal === RequestResult.NotFound)
+        return false;
     }
+    throw error;
   }
 }
 
@@ -1309,14 +1515,6 @@ function isAlreadyDisconnectedError(error: unknown): boolean {
   return (error as { readonly nativeErrno?: unknown }).nativeErrno === nativeErrnoValues.ENOENT;
 }
 
-/**
- * An admission belongs to one Core selected route. Its connection identity is
- * that route's generation, an opaque token compared only for equality.
- */
-function routeConnectionId(routeGeneration: bigint): string {
-  return `route:${routeGeneration.toString()}`;
-}
-
 function describeInvalidAdmissionDescriptor(
   descriptor: ServiceNodeDescriptor,
   expected: ServicePeerAdmissionExpectation | undefined,