@zlink-systems/framework 0.30.3 → 0.32.0

package/src/runtime/backend/node/node-raw-mesh-backend.ts

@@ -1,3 +1,6 @@
+import type { MeshOperationSubmission } from '../mesh-completion-table';
+import type { RequestRuntime } from '../../foundation/request-runtime';
+import type { PendingOperation } from '../../foundation/operation-registry';
 import { routingIdsEqual } from '../../routing-id';
 import { requestDurableService } from '../../foundation/durable-service-request';
 import { encodeRoutingIdStorageHex, toBackendRoutingId } from '../../routing-id';
@@ -126,6 +129,7 @@ import {
   decodeMultipartApplicationParts,
   decodeMultipartParts,
   encodeMultipartApplicationPayload,
+  encodeMultipartPayload,
   ServiceWireProtocolError,
   type ServiceApplicationPayload
 } from '../../foundation/service-wire-m6a-codec';
@@ -151,10 +155,6 @@ const UNSTARTED_DIAGNOSTIC_ENDPOINT = 'inproc://not-started';
 const MULTIPART_PACKET_NAME = SERVICE_FRAMEWORK_MULTIPART_PACKET_NAME;
 const MULTIPART_CONTENT_TYPE = SERVICE_FRAMEWORK_MULTIPART_CONTENT_TYPE;
 const MAX_DRAIN_RECORDS = 64;
-// The maintenance timer runs at the nearest management deadline and never
-// sooner than this. Only the binding readable handler admits receive work;
-// this timer never probes the socket.
-const MESH_BACKEND_MAINTENANCE_INTERVAL_MS = 1;
 /** The longest single timer delay Node accepts. */
 const MESH_BACKEND_MAINTENANCE_MAX_DELAY_MS = 2 ** 31 - 1;
 // Spec 08 §4.2 bounds the combined turn. Raw gets one equal share; stateful
@@ -182,6 +182,7 @@ export class ZLinkNodeRawMeshBackend implements ZLinkBackendMeshNode {
       readonly endpoint: string;
       readonly nodeRoutingId?: string;
       readonly lifecycleGeneration?: bigint;
+      readonly securityIdentity?: string;
     }
   >();
   private completionHandler?: ZLinkMeshCompletionHandler;
@@ -189,16 +190,21 @@ export class ZLinkNodeRawMeshBackend implements ZLinkBackendMeshNode {
   private readonly lifecycleGeneration: bigint;
   private bindEndpoint?: string;
   private advertiseHost?: string;
-  private runtime?: RawServiceMeshRuntime;
+  runtime?: RawServiceMeshRuntime;
   private stateful?: ServiceStatefulRuntime;
   private readyHandler?: (domains: number) => number;
   private maintenanceTimer?: NodeJS.Timeout;
   private maintenanceDueMs = Number.POSITIVE_INFINITY;
   private pumpTask?: Promise<void>;
   private readable = 0;
-  private readonly onReadable = (receiveReady: boolean, routeReady: boolean): void => {
+  private readonly onReadable = (
+    receiveReady: boolean,
+    routeReady: boolean,
+    connectReady: boolean
+  ): void => {
     if (receiveReady) this.readable |= 1;
     if (routeReady) this.readable |= 2;
+    if (connectReady) this.readable |= 4;
     this.pump();
   };
   private readonly onMaintenance = (): void => {
@@ -238,8 +244,8 @@ export class ZLinkNodeRawMeshBackend implements ZLinkBackendMeshNode {
     record: import('../../foundation/service-stateful-wire-codec').ServiceMessageFollowRecord
   ) => void;
   private dispatchErrors?: ZLinkDispatchErrorReporter;
-  private sessionOwnerDescriptors?: ServiceSessionOwnerDescriptors;
   private requestSourceAcceptance?: import('../contracts').ZLinkBackendRequestSourceAcceptance;
+  private sessionOwnerDescriptors?: ServiceSessionOwnerDescriptors;
   private readonly peerDisconnectedHandlers = new Set<(endpoint: string) => void>();
   constructor(
     private readonly meshName: string,
@@ -248,7 +254,11 @@ export class ZLinkNodeRawMeshBackend implements ZLinkBackendMeshNode {
     private readonly applicationJobQueue?: ApplicationJobQueuePort,
     private readonly applicationJobReceiveFlowFailureSink?: (error: unknown) => void,
     private readonly peerAdmissionSealed?: () => boolean,
-    private readonly infrastructureTaskRunner?: import('../../spots/spot-actor-join-dispatch').ZLinkDetachedTaskRunner
+    private readonly infrastructureTaskRunner?: import('../../spots/spot-actor-join-dispatch').ZLinkDetachedTaskRunner,
+    runtimeDependencies?: {
+      readonly raw?: RawServiceMeshRuntime;
+      readonly stateful?: ServiceStatefulRuntime;
+    }
   ) {
     if (meshName.length === 0) throw new TypeError('MeshName must be non-empty.');
     if (routingId === undefined || routingId.length === 0) {
@@ -256,6 +266,8 @@ export class ZLinkNodeRawMeshBackend implements ZLinkBackendMeshNode {
     }
     this.routingId = routingId;
     this.lifecycleGeneration = createLifecycleGeneration();
+    this.runtime = runtimeDependencies?.raw;
+    this.stateful = runtimeDependencies?.stateful;
   }
 
   configureObjectPlacement(options: {
@@ -365,17 +377,18 @@ export class ZLinkNodeRawMeshBackend implements ZLinkBackendMeshNode {
     deadlineUnixMs: bigint,
     sourceSpotId?: string,
     metadata?: ReadonlyMap<string, string>,
-    onAdmitted?: () => void
+    onAdmitted?: () => void,
+    requestOperation?: PendingOperation<unknown>
   ): MeshOperationId {
-    return this.observeStateful(
-      OperationKind.InstanceSpotRequest,
+    return this.observeStatefulRequest(OperationKind.InstanceSpotRequest, requestOperation, () =>
       this.requireStateful().requestToMissingInstanceSpotFrame(
         target,
         encodeMultipartApplicationFrame(parts),
         deadlineUnixMs,
         sourceSpotId,
         metadata === undefined ? undefined : encodeServiceMetadataFrame(metadata),
-        onAdmitted
+        onAdmitted,
+        requestOperation
       )
     );
   }
@@ -398,6 +411,13 @@ export class ZLinkNodeRawMeshBackend implements ZLinkBackendMeshNode {
     this.advertiseHost = host;
   }
 
+  private requests?: RequestRuntime;
+
+  setRequestRuntime(requests: RequestRuntime): void {
+    this.requireNotStarted();
+    this.requests = requests;
+  }
+
   start(): void {
     if (this.runtime !== undefined) return;
     if (this.closed) throw new Error('MeshNode is closed.');
@@ -405,12 +425,13 @@ export class ZLinkNodeRawMeshBackend implements ZLinkBackendMeshNode {
       throw new Error('MeshNode bind endpoint is not configured.');
     const descriptor = this.createDescriptor();
     const runtime = new RawServiceMeshRuntime({
+      requests: this.requests,
       onPendingOperationsChanged: this.onPendingOperationsChanged,
       descriptor,
       resolveAdvertisedEndpoint: (boundEndpoint) => this.resolveAdvertisedEndpoint(boundEndpoint),
       bindingPort: this.bindingPort,
       applicationJobQueue: this.requireApplicationJobQueue(),
-      onReceiveReady: () => this.onReadable(true, false),
+      onReceiveReady: () => this.onReadable(true, false, false),
       infrastructureTaskRunner: this.infrastructureTaskRunner,
       peerAdmissionSealed: this.peerAdmissionSealed,
       onReceiveFlowConfigFailure: this.applicationJobReceiveFlowFailureSink,
@@ -444,7 +465,8 @@ export class ZLinkNodeRawMeshBackend implements ZLinkBackendMeshNode {
       runtime,
       descriptor.nodeRoutingId,
       descriptor.lifecycleGeneration,
-      this.onPendingOperationsChanged
+      this.onPendingOperationsChanged,
+      this.requests
     );
     if (this.dispatchErrors !== undefined) {
       this.stateful.setDispatchErrorReporter(this.dispatchErrors, this.meshName);
@@ -550,19 +572,34 @@ export class ZLinkNodeRawMeshBackend implements ZLinkBackendMeshNode {
     const runtime = this.requireRuntime();
     const nodeRoutingId =
       options.expectedRid === undefined ? undefined : String(options.expectedRid);
+    const securityIdentity =
+      options.expectedSecurityIdentity ?? runtime.topology.localDescriptor().securityIdentity;
+    for (const [id, existing] of this.peerIntents) {
+      if (existing.endpoint !== options.endpoint) continue;
+      if (
+        existing.nodeRoutingId === nodeRoutingId &&
+        existing.securityIdentity === securityIdentity &&
+        existing.lifecycleGeneration === options.expectedLifecycleGeneration
+      ) {
+        return id;
+      }
+      runtime.disconnectPeerEndpoint(existing.endpoint);
+      this.peerIntents.delete(id);
+    }
     if (nodeRoutingId === undefined) {
       runtime.connectPeerEndpoint(options.endpoint);
     } else {
       runtime.connectPeerByRoutingId(
         options.endpoint,
         nodeRoutingId,
-        options.expectedSecurityIdentity ?? runtime.topology.localDescriptor().securityIdentity,
+        securityIdentity,
         options.expectedLifecycleGeneration
       );
     }
     const intent = this.nextPeerIntent++;
     this.peerIntents.set(intent, {
       endpoint: options.endpoint,
+      securityIdentity,
       ...(nodeRoutingId === undefined
         ? {}
         : {
@@ -592,6 +629,7 @@ export class ZLinkNodeRawMeshBackend implements ZLinkBackendMeshNode {
       peer.securityIdentity,
       peer.lifecycleGeneration
     );
+    this.scheduleMaintenance();
   }
 
   removePeerConnection(intentId: bigint): void {
@@ -741,63 +779,106 @@ export class ZLinkNodeRawMeshBackend implements ZLinkBackendMeshNode {
     frames: readonly Uint8Array[],
     options?: {
       readonly timeoutMs?: number;
+      readonly requestOperation?: PendingOperation<unknown>;
       readonly signal?: AbortSignal;
       readonly onSubmitted?: (signal: AbortSignal) => Promise<void>;
     }
   ): Promise<readonly Uint8Array[]> {
-    return requestDurableService(
-      this.requireRuntime(),
+    const runtime = this.requireRuntime();
+    const timeoutMs = options?.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
+    const pending =
+      options?.requestOperation ?? runtime.beginInfrastructureRequest(timeoutMs, options?.signal);
+    if (options?.requestOperation !== undefined) {
+      return requestDurableService(
+        runtime,
+        String(targetRid),
+        frames,
+        pending,
+        'relocation control',
+        options.signal,
+        options.onSubmitted
+      );
+    }
+    const completed = (reply: readonly Buffer[]) => {
+      pending.complete(reply);
+    };
+    const failed = (error: unknown) => {
+      pending.fail(error);
+    };
+    void requestDurableService(
+      runtime,
       String(targetRid),
       frames,
-      performance.now() + (options?.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS),
+      pending,
       'relocation control',
       options?.signal,
       options?.onSubmitted
-    );
+    ).then(completed, failed);
+    return pending.promise as Promise<readonly Uint8Array[]>;
   }
 
   requestToNode(
     targetRid: unknown,
     parts: MessageLike | readonly MessageLike[],
-    options?: { readonly timeoutMs?: number; readonly onAdmitted?: () => void }
+    options?: {
+      readonly timeoutMs?: number;
+      readonly requestOperation?: PendingOperation<unknown>;
+      readonly onAdmitted?: () => void;
+    },
+    completion?: import('../mesh-completion-table').ZLinkMeshCompletionHandler
   ): MeshOperationId {
     const pending = this.requireRuntime().requestToNode(
       String(targetRid),
       encodeMultipartApplicationFrame(parts),
       options?.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS,
       options?.onAdmitted,
-      this.requestObserver,
-      OperationKind.NodeRequest
+      this.requestObserverFor(completion),
+      OperationKind.NodeRequest,
+      options?.requestOperation
     );
     return this.observedOperation(pending, OperationKind.NodeRequest);
   }
 
-  async sendToChannel(
+  sendToChannel(
     channelName: string,
     parts: MessageLike | readonly MessageLike[]
   ): Promise<SubmitResultValue> {
-    return this.requireRuntime().sendToChannel(channelName, encodeMultipartApplicationFrame(parts));
+    try {
+      return Promise.resolve(
+        this.requireRuntime().sendToChannel(channelName, encodeMultipartApplicationFrame(parts))
+      );
+    } catch (error) {
+      return Promise.reject(error);
+    }
   }
 
   requestToChannel(
     channelName: string,
     parts: MessageLike | readonly MessageLike[],
-    options?: { readonly timeoutMs?: number; readonly onAdmitted?: () => void }
+    options?: {
+      readonly timeoutMs?: number;
+      readonly requestOperation?: PendingOperation<unknown>;
+      readonly onAdmitted?: () => void;
+    },
+    completion?: import('../mesh-completion-table').ZLinkMeshCompletionHandler
   ): MeshOperationId {
     const pending = this.requireRuntime().requestToChannel(
       channelName,
       encodeMultipartApplicationFrame(parts),
       options?.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS,
       options?.onAdmitted,
-      this.requestObserver,
-      OperationKind.ChannelRequest
+      this.requestObserverFor(completion),
+      OperationKind.ChannelRequest,
+      options?.requestOperation
     );
     return this.observedOperation(pending, OperationKind.ChannelRequest);
   }
 
   status() {
     const descriptor = this.runtime?.topology.localDescriptor() ?? this.createDescriptor();
-    const peers = this.runtime?.topology.peers() ?? [];
+    const peers = (this.runtime?.topology.peers() ?? []).filter(
+      (peer) => this.runtime?.isPeerRouteReady(peer.descriptor.nodeRoutingId) === true
+    );
     return {
       state: stateCode(descriptor.state),
       routingId: descriptor.nodeRoutingId as RoutingId,
@@ -1255,19 +1336,26 @@ export class ZLinkNodeRawMeshBackend implements ZLinkBackendMeshNode {
     sourceSpotId?: string,
     metadata?: ReadonlyMap<string, string>,
     instanceIntent = false,
-    onAdmitted?: () => void
+    onAdmitted?: () => void,
+    requestOperation?: PendingOperation<unknown>,
+    completion?: import('../mesh-completion-table').ZLinkMeshCompletionHandler
   ): MeshOperationId {
-    return this.observeStateful(
+    return this.observeStatefulRequest(
       OperationKind.InstanceSpotRequest,
-      this.requireStateful().requestToInstanceSpot(
-        route,
-        encodeMultipart(parts),
-        timeoutMs,
-        sourceSpotId,
-        metadata === undefined ? undefined : encodeServiceMetadataFrame(metadata),
-        instanceIntent,
-        onAdmitted
-      )
+      requestOperation,
+      (receiver) =>
+        this.requireStateful().requestToInstanceSpot(
+          route,
+          encodeMultipart(parts),
+          timeoutMs,
+          sourceSpotId,
+          metadata === undefined ? undefined : encodeServiceMetadataFrame(metadata),
+          instanceIntent,
+          onAdmitted,
+          requestOperation,
+          receiver
+        ),
+      completion
     );
   }
 
@@ -1418,21 +1506,28 @@ export class ZLinkNodeRawMeshBackend implements ZLinkBackendMeshNode {
     parts: MessageLike | readonly MessageLike[],
     options?: {
       readonly timeoutMs?: number;
+      readonly requestOperation?: PendingOperation<unknown>;
       readonly onAdmitted?: () => void;
       readonly routeFence?: import('../../foundation/service-stateful-wire-codec').ServiceActorRouteFence;
-    }
+    },
+    completion?: import('../mesh-completion-table').ZLinkMeshCompletionHandler
   ): MeshOperationId {
-    return this.observeStateful(
+    return this.observeStatefulRequest(
       OperationKind.ActorRequest,
-      this.requireStateful().requestToActor(
-        actor,
-        encodeMultipart(parts),
-        options?.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS,
-        undefined,
-        undefined,
-        options?.onAdmitted,
-        options?.routeFence
-      )
+      options?.requestOperation,
+      (receiver) =>
+        this.requireStateful().requestToActor(
+          actor,
+          encodeMultipart(parts),
+          options?.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS,
+          undefined,
+          undefined,
+          options?.onAdmitted,
+          options?.routeFence,
+          options?.requestOperation,
+          receiver
+        ),
+      completion
     );
   }
 
@@ -1460,17 +1555,22 @@ export class ZLinkNodeRawMeshBackend implements ZLinkBackendMeshNode {
     source: ZLinkBackendActorRef,
     target: ZLinkBackendActorRef,
     parts: MessageLike | readonly MessageLike[],
-    options?: { readonly timeoutMs?: number; readonly onAdmitted?: () => void }
+    options?: {
+      readonly timeoutMs?: number;
+      readonly requestOperation?: PendingOperation<unknown>;
+      readonly onAdmitted?: () => void;
+    }
   ): MeshOperationId {
-    return this.observeStateful(
-      OperationKind.ActorRequest,
+    return this.observeStatefulRequest(OperationKind.ActorRequest, options?.requestOperation, () =>
       this.requireStateful().requestToActor(
         target,
         encodeMultipart(parts),
         options?.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS,
         source,
         undefined,
-        options?.onAdmitted
+        options?.onAdmitted,
+        undefined,
+        options?.requestOperation
       )
     );
   }
@@ -1535,17 +1635,24 @@ export class ZLinkNodeRawMeshBackend implements ZLinkBackendMeshNode {
     target: ServiceDirectSpotRouteFence,
     parts: MessageLike | readonly MessageLike[],
     timeoutMs: number,
-    onAdmitted?: () => void
+    onAdmitted?: () => void,
+    requestOperation?: PendingOperation<unknown>,
+    completion?: import('../mesh-completion-table').ZLinkMeshCompletionHandler
   ): MeshOperationId {
-    return this.observeStateful(
+    return this.observeStatefulRequest(
       OperationKind.SpotRequest,
-      this.requireStateful().requestToSpot(
-        source.ref.spotId,
-        target,
-        encodeMultipart(parts),
-        timeoutMs,
-        onAdmitted
-      )
+      requestOperation,
+      (receiver) =>
+        this.requireStateful().requestToSpot(
+          source.ref.spotId,
+          target,
+          encodeMultipart(parts),
+          timeoutMs,
+          onAdmitted,
+          requestOperation,
+          receiver
+        ),
+      completion
     );
   }
 
@@ -1584,7 +1691,7 @@ export class ZLinkNodeRawMeshBackend implements ZLinkBackendMeshNode {
     this.requireStateful().clearSubscriptions(state.ref.spotId);
   }
 
-  private createDescriptor(): ServiceNodeDescriptor {
+  createDescriptor(): ServiceNodeDescriptor {
     const channels: ServiceChannelDescriptor[] = [...this.channels]
       .map(([name, weight]) => ({ name, weight }))
       .sort((left, right) => left.name.localeCompare(right.name));
@@ -1623,22 +1730,17 @@ export class ZLinkNodeRawMeshBackend implements ZLinkBackendMeshNode {
   /**
    * Arms the one maintenance timer for the nearest management deadline
    * (spec 08 §4.2): request expiry, liveness, or a pending peer handshake.
-   * An earlier deadline re-arms it; a later one keeps the armed time.
+   * The timer follows changes to the index visit or the other management deadlines.
    */
   private scheduleMaintenance(): void {
     if (this.closed) return;
     const nowMs = performance.now();
-    const dueMs = Math.max(
-      nowMs + MESH_BACKEND_MAINTENANCE_INTERVAL_MS,
-      Math.min(
-        this.runtime?.nextMaintenanceDeadlineMs(nowMs) ?? Number.POSITIVE_INFINITY,
-        this.stateful?.nextMaintenanceDeadlineMs() ?? Number.POSITIVE_INFINITY
-      )
+    const dueMs = Math.min(
+      this.runtime?.nextMaintenanceDeadlineMs(nowMs) ?? Number.POSITIVE_INFINITY,
+      this.stateful?.nextMaintenanceDeadlineMs() ?? Number.POSITIVE_INFINITY
     );
     if (dueMs === Number.POSITIVE_INFINITY) {
-      if (this.maintenanceTimer !== undefined) clearTimeout(this.maintenanceTimer);
-      this.maintenanceTimer = undefined;
-      this.maintenanceDueMs = dueMs;
+      this.maintenanceTimer?.unref();
       return;
     }
     if (this.maintenanceTimer === undefined || dueMs < this.maintenanceDueMs) {
@@ -1661,29 +1763,92 @@ export class ZLinkNodeRawMeshBackend implements ZLinkBackendMeshNode {
 
   private pump(): void {
     if (this.closed || this.pumpTask !== undefined) return;
-    this.pumpTask = this.runPump(this.requireRuntime());
+    const runtime = this.requireRuntime();
+    let complete!: () => void;
+    let fail!: (error: unknown) => void;
+    // Publish the join owner before receive can signal readiness or start close.
+    // The receive turn itself needs no startup Promise continuation.
+    this.pumpTask = new Promise<void>((resolve, reject) => {
+      complete = resolve;
+      fail = reject;
+    });
+    this.runPump(runtime, complete, fail);
   }
 
-  private async runPump(runtime: RawServiceMeshRuntime): Promise<void> {
-    // Publish pumpTask before any callback can signal readiness or begin close.
-    await Promise.resolve();
+  private runPump(
+    runtime: RawServiceMeshRuntime,
+    complete: () => void,
+    fail: (error: unknown) => void
+  ): void {
     try {
       do {
-        const ready = this.readable;
-        this.readable = 0;
-        const more = await runtime.pumpBatch((ready & 1) !== 0, (ready & 2) !== 0);
+        const batch = this.pumpReceiveBatch(runtime);
+        if (batch !== false) {
+          // Only suspended work needs an async continuation. A completed receive
+          // rotation uses the published join without allocating another Promise.
+          void this.resumePump(runtime, batch, complete, fail);
+          return;
+        }
+      } while (this.hasPumpReadiness());
+    } catch (error) {
+      this.finishPump(complete, fail, runtime, error);
+      return;
+    }
+    this.finishPump(complete, fail);
+  }
+
+  private async resumePump(
+    runtime: RawServiceMeshRuntime,
+    batch: boolean | Promise<boolean>,
+    complete: () => void,
+    fail: (error: unknown) => void
+  ): Promise<void> {
+    try {
+      for (;;) {
+        const more = batch instanceof Promise ? await batch : batch;
         if (more) {
           // Yield only after actual progress, retaining readiness across the
           // existing batch limits until receive reports no data.
           this.readable |= 1;
           await yieldToIO();
         }
-      } while (!this.closed && this.readable !== 0);
+        if (!this.hasPumpReadiness()) break;
+        batch = this.pumpReceiveBatch(runtime);
+      }
     } catch (error) {
-      runtime.reportInfrastructureFailure(error);
-    } finally {
-      this.pumpTask = undefined;
-      this.scheduleMaintenance();
+      this.finishPump(complete, fail, runtime, error);
+      return;
+    }
+    this.finishPump(complete, fail);
+  }
+
+  private pumpReceiveBatch(runtime: RawServiceMeshRuntime): boolean | Promise<boolean> {
+    const ready = this.readable;
+    this.readable = 0;
+    return runtime.pumpBatch((ready & 1) !== 0, (ready & 2) !== 0, (ready & 4) !== 0);
+  }
+
+  private hasPumpReadiness(): boolean {
+    return !this.closed && this.readable !== 0;
+  }
+
+  private finishPump(
+    complete: () => void,
+    fail: (error: unknown) => void,
+    failedRuntime?: RawServiceMeshRuntime,
+    error?: unknown
+  ): void {
+    try {
+      try {
+        if (failedRuntime !== undefined) failedRuntime.reportInfrastructureFailure(error);
+      } finally {
+        this.pumpTask = undefined;
+        this.scheduleMaintenance();
+      }
+      complete();
+    } catch (error) {
+      // Forward reporting and maintenance failures to the published join owner.
+      fail(error);
     }
   }
 
@@ -1700,57 +1865,121 @@ export class ZLinkNodeRawMeshBackend implements ZLinkBackendMeshNode {
   }
 
   /**
-   * The raw registry takes a remote request's terminal and hands it here,
-   * where the Framework completion owner receives it in the same turn.
+   * Publishes native results after submission returns and the completion slot has its id.
+   * A caller-owned entry is completed after the result is mapped.
    */
   private readonly requestObserver: RawServiceRequestObserver = {
     operationCompleted: (id, result, operationKind) =>
-      this.deliverCompletion({ high: 1n, low: id }, operationKind ?? 0, result),
+      queueMicrotask(() =>
+        this.deliverCompletion({ high: 1n, low: id }, operationKind ?? 0, result)
+      ),
     operationFailed: (id, error, operationKind) =>
-      this.deliverCompletion({ high: 1n, low: id }, operationKind ?? 0, requestFailureResult(error))
+      queueMicrotask(() =>
+        this.deliverCompletion(
+          { high: 1n, low: id },
+          operationKind ?? 0,
+          requestFailureResult(error)
+        )
+      )
   };
 
+  private requestObserverFor(
+    completion?: import('../mesh-completion-table').ZLinkMeshCompletionHandler
+  ): RawServiceRequestObserver {
+    if (completion === undefined) return this.requestObserver;
+    return {
+      operationCompleted: (id, result, operationKind) =>
+        queueMicrotask(() =>
+          this.deliverCompletion({ high: 1n, low: id }, operationKind ?? 0, result, completion)
+        ),
+      operationFailed: (id, error, operationKind) =>
+        queueMicrotask(() =>
+          this.deliverCompletion(
+            { high: 1n, low: id },
+            operationKind ?? 0,
+            requestFailureResult(error),
+            completion
+          )
+        )
+    };
+  }
+
   private observedOperation(
     pending: RawServiceObservedRequest,
     operationKind: number
-  ): MeshOperationId {
-    return pending.promise === undefined
-      ? { high: 1n, low: pending.id }
-      : this.observeCompletion(pending.id, operationKind, pending.promise);
-  }
-
-  private observeCompletion(
-    low: bigint,
-    operationKind: number,
-    promise: Promise<RawServiceRequestResult>
-  ): MeshOperationId {
-    const id = { high: 1n, low };
-    void promise.then(
-      (result) => this.deliverCompletion(id, operationKind, result),
-      (error) => this.deliverCompletion(id, operationKind, requestFailureResult(error))
-    );
-    return id;
+  ): MeshOperationSubmission {
+    const id = { high: 1n, low: pending.id };
+    if (pending.promise !== undefined && !pending.hasObserver()) {
+      void pending.promise.then(
+        (result) => this.deliverCompletion(id, operationKind, result),
+        (error) => this.deliverCompletion(id, operationKind, requestFailureResult(error))
+      );
+    }
+    return this.requestSubmission(id, pending);
+  }
+
+  private observeStatefulRequest(
+    kind: number,
+    entry: PendingOperation<unknown> | undefined,
+    submit: (
+      receiver?: import('../../foundation/operation-registry').OperationObserver<unknown, unknown>
+    ) => ServiceStatefulPendingOperation,
+    completion?: import('../mesh-completion-table').ZLinkMeshCompletionHandler
+  ): MeshOperationSubmission {
+    if (entry === undefined) return this.observeStateful(kind, submit(), completion);
+    const id = { high: ZLINK_NATIVE_CORRELATION_OPERATION_NAMESPACE, low: entry.id };
+    const receiver: import('../../foundation/operation-registry').OperationObserver<
+      unknown,
+      unknown
+    > = {
+      operationCompleted: (_operationId, value) =>
+        queueMicrotask(() =>
+          this.deliverCompletion(id, kind, value as ServiceStatefulResult, completion)
+        ),
+      operationFailed: (_operationId, error) =>
+        queueMicrotask(() =>
+          this.deliverCompletion(id, kind, requestFailureResult(error), completion)
+        )
+    };
+    if (completion === undefined) {
+      entry.setObserver(receiver);
+      submit();
+    } else submit(receiver);
+    return this.requestSubmission(id, entry);
   }
 
   private observeStateful(
     operationKind: number,
-    pending: ServiceStatefulPendingOperation
-  ): MeshOperationId {
+    pending: ServiceStatefulPendingOperation,
+    completion?: import('../mesh-completion-table').ZLinkMeshCompletionHandler
+  ): MeshOperationSubmission {
     const id = { high: ZLINK_NATIVE_CORRELATION_OPERATION_NAMESPACE, low: pending.id };
     void pending.promise.then(
-      (result) => this.deliverCompletion(id, operationKind, result),
-      (error) => this.deliverCompletion(id, operationKind, requestFailureResult(error))
+      (result) => this.deliverCompletion(id, operationKind, result, completion),
+      (error) => this.deliverCompletion(id, operationKind, requestFailureResult(error), completion)
     );
-    return id;
+    return this.requestSubmission(id, pending);
+  }
+
+  private requestSubmission(
+    id: MeshOperationId,
+    requestOperation: RawServiceObservedRequest | PendingOperation<unknown>
+  ): MeshOperationSubmission {
+    return Object.defineProperty({ ...id }, 'requestOperation', {
+      value: requestOperation,
+      configurable: true
+    });
   }
 
   private deliverCompletion(
     operationId: MeshOperationId,
     operationKind: number,
-    result: RawServiceRequestResult | ServiceStatefulResult
+    result: RawServiceRequestResult | ServiceStatefulResult,
+    completion: import('../mesh-completion-table').ZLinkMeshCompletionHandler | undefined = this
+      .completionHandler
   ): void {
-    if (this.completionHandler !== undefined) {
-      this.completionHandler({ operationId, terminalResult: result.terminalResult }, () =>
+    if (completion !== undefined) {
+      completion({ operationId, terminalResult: result.terminalResult }, () =>
         completionRecord(operationId, operationKind, result)
       );
       return;
@@ -1844,14 +2073,18 @@ class RawServiceSpot implements ServiceSpot {
     };
   }
 
-  async sendToChannel(
+  sendToChannel(
     channelName: string,
     parts: MessageLike | readonly MessageLike[],
     options?: { readonly flags?: number }
   ): Promise<SubmitResultValue> {
-    this.requireOpen();
-    void options;
-    return this.backend.sendToChannel(channelName, parts);
+    try {
+      this.requireOpen();
+      void options;
+      return this.backend.sendToChannel(channelName, parts);
+    } catch (error) {
+      return Promise.reject(error);
+    }
   }
 
   requestToChannel(
@@ -1860,6 +2093,7 @@ class RawServiceSpot implements ServiceSpot {
     options?: {
       readonly flags?: number;
       readonly timeoutMs?: number;
+      readonly requestOperation?: PendingOperation<unknown>;
       readonly onAdmitted?: () => void;
     }
   ): MeshOperationId {
@@ -1897,10 +2131,12 @@ class RawServiceSpot implements ServiceSpot {
     parts: MessageLike | readonly MessageLike[],
     options?: {
       readonly timeoutMs?: number;
+      readonly requestOperation?: PendingOperation<unknown>;
       readonly onAdmitted?: () => void;
       readonly routeFence?: ServiceDirectSpotRouteFence;
       readonly entrySpot?: boolean;
-    }
+    },
+    completion?: import('../mesh-completion-table').ZLinkMeshCompletionHandler
   ): MeshOperationId {
     this.requireOpen();
     const routeFence = requireDirectSpotRouteFence(
@@ -1917,7 +2153,9 @@ class RawServiceSpot implements ServiceSpot {
       routeFence,
       parts,
       options?.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS,
-      options?.onAdmitted
+      options?.onAdmitted,
+      options?.requestOperation,
+      completion
     );
   }
 
@@ -2322,14 +2560,17 @@ function completionRecord(
     kindData: 'kindData' in result ? (result.kindData ?? null) : null,
     terminalResult: result.terminalResult,
     failureErrno: result.failureCode,
+    ...('failureMessage' in result && result.failureMessage !== undefined
+      ? { failureMessage: result.failureMessage }
+      : {}),
     parts:
       payload === undefined
         ? []
         : payload.contentType === MULTIPART_CONTENT_TYPE
           ? decodeMultipart(payload.payload)
           : [ZLinkBufferMessage.from(payload.payload)],
-    reply: () => SubmitResult.InvalidState,
-    replyActorJoin: () => SubmitResult.NotSupported
+    reply: replyInvalidState,
+    replyActorJoin: replyActorJoinNotSupported
   };
 }
 
@@ -2380,17 +2621,12 @@ function decodeMultipartRecord(
         const values = Array.isArray(parts) ? parts : [parts];
         if (values.length === 0) return SubmitResult.InvalidState;
         runtime.replyService(
-          {
-            sourceRoutingId: record.sourceRoutingId!,
-            ...(record.sourceRoute === undefined ? {} : { sourceRoute: record.sourceRoute }),
-            requestSequence: record.requestSequence!,
-            ...(record.reply === undefined ? {} : { reply: record.reply })
-          },
-          values.map((value) => Buffer.from(messageBytesView(value)))
+          record as Parameters<RawServiceMeshRuntime['replyService']>[0],
+          values.map(messagePayloadView)
         );
         return SubmitResult.Ok;
       },
-      replyActorJoin: () => SubmitResult.NotSupported
+      replyActorJoin: replyActorJoinNotSupported
     };
   }
   const header = record.parts[0]!;
@@ -2463,6 +2699,10 @@ function releaseRecordApplicationJob(this: {
   )?.close();
 }
 
+function replyInvalidState(): SubmitResultValue {
+  return SubmitResult.InvalidState;
+}
+
 function replyActorJoinNotSupported(): SubmitResultValue {
   return SubmitResult.NotSupported;
 }
@@ -2572,9 +2812,9 @@ function decodeStatefulRecord(
       });
       return accepted ? SubmitResult.Ok : SubmitResult.InvalidState;
     },
-    replyFailure(terminalResult, failureCode) {
+    replyFailure(terminalResult, failureCode, tail) {
       if (stateful.reply === undefined) return SubmitResult.InvalidState;
-      return stateful.reply(terminalResult, failureCode)
+      return stateful.reply(terminalResult, failureCode, undefined, tail)
         ? SubmitResult.Ok
         : SubmitResult.InvalidState;
     }
@@ -2582,16 +2822,20 @@ function decodeStatefulRecord(
 }
 
 function receiveIngressLifecycle(record: ServiceMailboxRecord): {
+  readonly receivedAt?: Date;
   readonly applicationJobPermit?: import('../../application-jobs/contracts').ApplicationJobPermitPort;
   readonly releaseRetainedIngress?: () => void;
 } {
   const applicationJob = record.applicationJob;
-  return applicationJob === undefined
-    ? {}
-    : {
-        applicationJobPermit: applicationJob,
-        releaseRetainedIngress: () => applicationJob.close()
-      };
+  return {
+    ...(record.receivedAt === undefined ? {} : { receivedAt: record.receivedAt }),
+    ...(applicationJob === undefined
+      ? {}
+      : {
+          applicationJobPermit: applicationJob,
+          releaseRetainedIngress: () => applicationJob.close()
+        })
+  };
 }
 
 function prepareStatefulMailboxDispatch(
@@ -2696,24 +2940,10 @@ export function requestFailureResult(error: unknown): RawServiceRequestResult {
 function encodeMultipart(parts: MessageLike | readonly MessageLike[]) {
   const values = Array.isArray(parts) ? parts : [parts];
   if (values.length === 0) throw new TypeError('Multipart payload must contain at least one part.');
-  //  messageBytesView aliases the caller's buffer; the parts are consumed
-  //  synchronously by the copy loop below and never retained.
-  const bytes = values.map(messageBytesView);
-  let size = 4;
-  for (const part of bytes) size += 4 + part.byteLength;
-  const payload = Buffer.alloc(size);
-  payload.writeUInt32BE(bytes.length, 0);
-  let offset = 4;
-  for (const part of bytes) {
-    payload.writeUInt32BE(part.byteLength, offset);
-    offset += 4;
-    payload.set(part, offset);
-    offset += part.byteLength;
-  }
   return {
     packetName: MULTIPART_PACKET_NAME,
     contentType: MULTIPART_CONTENT_TYPE,
-    payload
+    payload: encodeMultipartPayload(values.map(messagePayloadView))
   };
 }
 
@@ -2721,7 +2951,7 @@ function encodeMultipartApplicationFrame(parts: MessageLike | readonly MessageLi
   const values = Array.isArray(parts) ? parts : [parts];
   if (values.length === 0) throw new TypeError('Multipart payload must contain at least one part.');
   return encodeMultipartApplicationPayload(
-    values.map(messageBytesView),
+    values.map(messagePayloadView),
     MULTIPART_PACKET_NAME,
     MULTIPART_CONTENT_TYPE
   );
@@ -2739,11 +2969,10 @@ function borrowMultipartPart(part: Buffer): Buffer {
   return part;
 }
 
-function messageBytesView(value: MessageLike): Uint8Array {
+function messagePayloadView(value: MessageLike): Uint8Array | string {
   if (typeof value === 'object' && 'data' in value) {
     return value.data();
   }
-  if (typeof value === 'string') return Buffer.from(value);
   return value;
 }