@zlink-systems/framework 0.31.0 → 0.32.0

package/src/runtime/channels/channel-receive-loops.ts

@@ -19,8 +19,9 @@ import {
 } from './channel-receive-task-tracker';
 import { ApplicationJobQueue, type ApplicationJobQueuePermit } from '../host/application-job-queue';
 import {
-  bindApplicationJobPermit,
-  runWithApplicationJobPermit
+  enterApplicationJobPermitScope,
+  leaveApplicationJobPermitScope,
+  releaseApplicationJobPermitTurn
 } from '../application-jobs/application-job-queue-scope';
 import {
   ZLinkSerialExecutionQueue,
@@ -29,20 +30,45 @@ import {
 
 export function createChannelApplicationDispatchQueue(): ZLinkSerialExecutionQueue {
   return new ZLinkSerialExecutionQueue(async (record: ZLinkSerialWorkRecord<unknown>) => {
+    const permit = record.context as ApplicationJobQueuePermit;
+    const previousPermit = enterApplicationJobPermitScope(permit);
     try {
       void Promise.resolve(record.operation()).then(
         (value) => {
-          record.resolve(value);
-          record.release();
+          try {
+            record.resolve(value);
+          } finally {
+            try {
+              record.release();
+            } finally {
+              releaseApplicationJobPermitTurn(permit);
+            }
+          }
         },
         (error) => {
-          record.reject(error);
-          record.release();
+          try {
+            record.reject(error);
+          } finally {
+            try {
+              record.release();
+            } finally {
+              releaseApplicationJobPermitTurn(permit);
+            }
+          }
         }
       );
     } catch (error) {
-      record.reject(error);
-      record.release();
+      try {
+        record.reject(error);
+      } finally {
+        try {
+          record.release();
+        } finally {
+          releaseApplicationJobPermitTurn(permit);
+        }
+      }
+    } finally {
+      leaveApplicationJobPermitScope(previousPermit);
     }
   });
 }
@@ -79,7 +105,8 @@ interface ZLinkChannelRequestDispatchLoop {
       reply(routingId: unknown, replyToken: unknown): ZLinkMultipartReplyOperation;
     },
     signal?: AbortSignal,
-    decodedHeader?: ZLinkChannelEnvelopeHeader
+    decodedHeader?: ZLinkChannelEnvelopeHeader,
+    receivedAt?: Date
   ): Promise<boolean | void>;
 }
 
@@ -87,7 +114,8 @@ interface ZLinkChannelPublishDispatchLoop {
   dispatch(
     topicMessage: { readonly topic: string; readonly parts: readonly Message[] },
     signal?: AbortSignal,
-    decodedHeader?: ZLinkChannelEnvelopeHeader
+    decodedHeader?: ZLinkChannelEnvelopeHeader,
+    receivedAt?: Date
   ): Promise<void>;
 }
 
@@ -118,7 +146,8 @@ interface ZLinkRoutePacketDispatchLoop {
     },
     signal?: AbortSignal,
     decodedHeader?: ZLinkChannelEnvelopeHeader,
-    infrastructureChecked?: boolean
+    infrastructureChecked?: boolean,
+    receivedAt?: Date
   ): Promise<boolean | void>;
 }
 
@@ -240,7 +269,8 @@ export class ZLinkChannelReceiveLoop {
     private readonly applicationJobQueue: ApplicationJobQueue,
     reportError?: ZLinkReceiveTaskErrorReporter,
     private readonly roundRobin?: ZLinkReceiveRoundRobinCoordinator,
-    private readonly applicationDispatchQueue = createChannelApplicationDispatchQueue()
+    private readonly applicationDispatchQueue = createChannelApplicationDispatchQueue(),
+    private readonly receivedAtClock: () => Date = () => new Date()
   ) {
     this.inFlight = new ZLinkReceiveTaskTracker(reportError);
     this.receiveOwner = roundRobin?.register() ?? Symbol('channel-receive-owner');
@@ -304,6 +334,7 @@ export class ZLinkChannelReceiveLoop {
                 this.roundRobin?.setReady(this.receiveOwner, false);
                 break;
               }
+              const receivedAt = this.receivedAtClock();
               const receivedBytes = messageBytes(received.parts);
               let classification: ReturnType<ZLinkChannelReceiveLoop['classify']>;
               try {
@@ -322,12 +353,18 @@ export class ZLinkChannelReceiveLoop {
                 break;
               } else {
                 applicationJobPermit.markApplicationQueued();
-                const task = runWithApplicationJobPermit(applicationJobPermit, () =>
-                  this.applicationDispatchQueue.submitPreAdmitted(
-                    bindApplicationJobPermit(() =>
-                      this.dispatchAndClose(received, signal, classification.decodedHeader, true)
-                    )
-                  )
+                const task = this.applicationDispatchQueue.submitPreAdmitted(
+                  () =>
+                    this.dispatchAndClose(
+                      received,
+                      signal,
+                      classification.decodedHeader,
+                      true,
+                      undefined,
+                      receivedAt
+                    ),
+                  undefined,
+                  applicationJobPermit
                 );
                 this.inFlight.track(task);
               }
@@ -374,7 +411,8 @@ export class ZLinkChannelReceiveLoop {
     signal?: AbortSignal,
     decodedHeader?: ZLinkChannelEnvelopeHeader,
     infrastructureChecked = false,
-    releaseRawReceive?: () => void
+    releaseRawReceive?: () => void,
+    receivedAt: Date = this.receivedAtClock()
   ): Promise<void> {
     let closeReceived = true;
     try {
@@ -386,7 +424,13 @@ export class ZLinkChannelReceiveLoop {
         }
         decodedHeader = classification.decodedHeader;
       }
-      const consumed = await this.dispatcher.dispatch(received, this.router, signal, decodedHeader);
+      const consumed = await this.dispatcher.dispatch(
+        received,
+        this.router,
+        signal,
+        decodedHeader,
+        receivedAt
+      );
       if (consumed === true) {
         closeReceived = false;
       }
@@ -452,7 +496,8 @@ export class ZLinkSubscriberReceiveLoop {
     ) => boolean | ZLinkSubscriberInfrastructureResult,
     private readonly roundRobin?: ZLinkReceiveRoundRobinCoordinator,
     reportError?: ZLinkReceiveTaskErrorReporter,
-    private readonly applicationDispatchQueue = createChannelApplicationDispatchQueue()
+    private readonly applicationDispatchQueue = createChannelApplicationDispatchQueue(),
+    private readonly receivedAtClock: () => Date = () => new Date()
   ) {
     this.inFlight = new ZLinkReceiveTaskTracker(reportError);
     this.poller = adapter.createReadablePoller(subscriber);
@@ -523,6 +568,7 @@ export class ZLinkSubscriberReceiveLoop {
         if (!(await this.poller.waitForReadable(capacitySignal))) break;
         continue;
       }
+      const receivedAt = this.receivedAtClock();
       const receivedBytes = messageBytes(topicMessage.parts as readonly Message[]);
       let classification: ReturnType<ZLinkSubscriberReceiveLoop['classify']>;
       try {
@@ -551,12 +597,18 @@ export class ZLinkSubscriberReceiveLoop {
         break;
       }
       applicationJobPermit.markApplicationQueued();
-      const task = runWithApplicationJobPermit(applicationJobPermit, () =>
-        this.applicationDispatchQueue.submitPreAdmitted(
-          bindApplicationJobPermit(() =>
-            this.dispatchAndClose(topicMessage, signal, classification.decodedHeader, true)
-          )
-        )
+      const task = this.applicationDispatchQueue.submitPreAdmitted(
+        () =>
+          this.dispatchAndClose(
+            topicMessage,
+            signal,
+            classification.decodedHeader,
+            true,
+            undefined,
+            receivedAt
+          ),
+        undefined,
+        applicationJobPermit
       );
       releaseRawReceive();
       this.inFlight.track(task);
@@ -593,7 +645,8 @@ export class ZLinkSubscriberReceiveLoop {
     signal?: AbortSignal,
     decodedHeader?: ZLinkChannelEnvelopeHeader,
     infrastructureChecked = false,
-    releaseRawReceive?: () => void
+    releaseRawReceive?: () => void,
+    receivedAt: Date = this.receivedAtClock()
   ): Promise<void> {
     try {
       if (!infrastructureChecked) {
@@ -601,7 +654,7 @@ export class ZLinkSubscriberReceiveLoop {
         if (classification.kind === 'consumed') return;
         decodedHeader = classification.decodedHeader;
       }
-      await this.dispatcher.dispatch(topicMessage, signal, decodedHeader);
+      await this.dispatcher.dispatch(topicMessage, signal, decodedHeader, receivedAt);
     } finally {
       releaseRawReceive?.();
       closeMessages(topicMessage.parts as readonly Message[]);
@@ -650,7 +703,8 @@ export class ZLinkRouteReceiveLoop {
     private readonly applicationJobQueue: ApplicationJobQueue,
     private readonly roundRobin?: ZLinkReceiveRoundRobinCoordinator,
     reportError?: ZLinkReceiveTaskErrorReporter,
-    private readonly applicationDispatchQueue = createChannelApplicationDispatchQueue()
+    private readonly applicationDispatchQueue = createChannelApplicationDispatchQueue(),
+    private readonly receivedAtClock: () => Date = () => new Date()
   ) {
     this.inFlight = new ZLinkReceiveTaskTracker(reportError);
     this.receiveOwner = roundRobin?.register() ?? Symbol('route-receive-owner');
@@ -712,6 +766,7 @@ export class ZLinkRouteReceiveLoop {
         if (!(await this.poller.waitForReadable(capacitySignal))) break;
         continue;
       }
+      const receivedAt = this.receivedAtClock();
       const receivedBytes = messageBytes(received.parts);
       let infrastructure: boolean | Promise<void> | undefined;
       try {
@@ -763,10 +818,10 @@ export class ZLinkRouteReceiveLoop {
         break;
       }
       applicationJobPermit.markApplicationQueued();
-      const task = runWithApplicationJobPermit(applicationJobPermit, () =>
-        this.applicationDispatchQueue.submitPreAdmitted(
-          bindApplicationJobPermit(() => this.dispatchAndClose(received, signal, true))
-        )
+      const task = this.applicationDispatchQueue.submitPreAdmitted(
+        () => this.dispatchAndClose(received, signal, true, undefined, receivedAt),
+        undefined,
+        applicationJobPermit
       );
       releaseRawReceive();
       this.inFlight.track(task);
@@ -808,7 +863,8 @@ export class ZLinkRouteReceiveLoop {
     },
     signal?: AbortSignal,
     infrastructureChecked = false,
-    releaseRawReceive?: () => void
+    releaseRawReceive?: () => void,
+    receivedAt: Date = this.receivedAtClock()
   ): Promise<void> {
     let closeReceived = true;
     try {
@@ -817,7 +873,8 @@ export class ZLinkRouteReceiveLoop {
         this.router,
         signal,
         tryDecodeChannelHeader(received.parts, this.dispatcher.flowEnabled?.() ?? true),
-        infrastructureChecked
+        infrastructureChecked,
+        receivedAt
       );
       if (consumed === true) {
         closeReceived = false;

package/src/runtime/channels/channel-runtime-lifecycle.ts

@@ -104,7 +104,7 @@ export class ZLinkChannelRuntimeLifecycle {
   private readonly receiveRoundRobin = new ZLinkReceiveRoundRobinCoordinator();
   private readonly channelReceiveLoops: ZLinkChannelReceiveLoop[] = [];
   private readonly subscriberReceiveLoops: ZLinkSubscriberReceiveLoop[] = [];
-  private readonly manualFanoutSubscribers = new Map<string, ManualFanoutSubscriberState>();
+  readonly manualFanoutSubscribers = new Map<string, ManualFanoutSubscriberState>();
   private readonly manualFanoutSubscriberOwners = new Set<object>();
   private readonly routeReceiveLoops: Array<{ stop(): Promise<void> }> = [];
   private readonly meshChannelDispatchers = new Map<string, ZLinkChannelRequestDispatcher>();
@@ -664,7 +664,7 @@ export class ZLinkChannelRuntimeLifecycle {
     return state;
   }
 
-  private requestManualFanoutConnect(
+  requestManualFanoutConnect(
     channelName: string,
     endpoint: string,
     taskRunner: ZLinkRuntimeTaskRunner,

package/src/runtime/channels/channel-socket-registry.ts

@@ -156,7 +156,7 @@ export class ZLinkChannelSocketRegistry {
       readonly lifecycleGeneration: bigint;
     }
   >();
-  private readonly clientServerConnections = new Map<string, ClientServerPhysicalConnection>();
+  readonly clientServerConnections = new Map<string, ClientServerPhysicalConnection>();
   private readonly clientServerDiscovery = new ServiceDiscoveryRegistry();
   private readonly clientServerReadyIdentities = new Map<
     string,
@@ -1455,7 +1455,7 @@ export class ZLinkChannelSocketRegistry {
     }
   }
 
-  private applyClientServerDescriptorUpdate(
+  applyClientServerDescriptorUpdate(
     connectionId: string,
     connection: { readonly channelName: string; readonly physicalConnectionId: symbol },
     admission: ZLinkClientServerAdmission
@@ -1518,7 +1518,7 @@ export class ZLinkChannelSocketRegistry {
     this.clientServerLivenessTimer.unref();
   }
 
-  private drainSocketMonitors(): void {
+  drainSocketMonitors(): void {
     for (const monitor of this.ownedResources) if ('drain' in monitor) monitor.drain();
   }
 

package/src/runtime/diagnostics/runtime-metrics.ts

@@ -574,7 +574,8 @@ export class ZLinkRuntimeMetrics {
   startRelocation(
     meshName: string,
     objectKind: 'actor' | 'user_spot' | 'instance_spot',
-    policy: 'recreate' | 'snapshot'
+    policy: 'recreate' | 'snapshot',
+    onComplete?: (outcome: string) => void
   ): ZLinkRelocationMetricOperation {
     const started = process.hrtime.bigint();
     const startAttributes = { mesh_name: meshName, object_kind: objectKind, policy };
@@ -595,6 +596,7 @@ export class ZLinkRuntimeMetrics {
           Number(process.hrtime.bigint() - started) / NANOSECONDS_PER_SECOND,
           terminal
         );
+        onComplete?.(outcome);
       }
     };
   }

package/src/runtime/execution/serial-execution-queue.ts

@@ -72,9 +72,9 @@ interface ZLinkSerialAdmissionLane {
  * Application work carries the host permit through the queue until the handler starts.
  */
 export class ZLinkSerialExecutionQueue {
-  private readonly application: ZLinkSerialAdmissionLane;
-  private readonly lifecycle: ZLinkSerialAdmissionLane;
-  private activeLifecycle?: ZLinkSerialWorkRecord<unknown>;
+  readonly application: ZLinkSerialAdmissionLane;
+  readonly lifecycle: ZLinkSerialAdmissionLane;
+  activeLifecycle?: ZLinkSerialWorkRecord<unknown>;
   private activeRecord?: ZLinkSerialWorkRecord<unknown>;
   /** The admitted node itself owns ready membership; no second readiness flag. */
   private ownerMembership?: SerialWorkRecord<unknown>;
@@ -86,10 +86,10 @@ export class ZLinkSerialExecutionQueue {
   private lifecycleDebt = false;
   private claimStartedAt?: number;
   private closed = false;
-  private readonly idleWaiters: Array<() => void> = [];
+  readonly idleWaiters: Array<() => void> = [];
 
   constructor(
-    private readonly executeRecord: (record: ZLinkSerialWorkRecord<unknown>) => Promise<void>,
+    readonly executeRecord: (record: ZLinkSerialWorkRecord<unknown>) => Promise<void>,
     options: ZLinkSerialSchedulerOptions = {},
     private readonly sharedOwner?: ZLinkSerialExecutionQueue
   ) {
@@ -217,7 +217,7 @@ export class ZLinkSerialExecutionQueue {
     return this.closed;
   }
 
-  private admit<T>(
+  admit<T>(
     operation: () => Promise<T> | T,
     options: ZLinkSerialWorkOptions | undefined,
     context?: unknown,
@@ -291,7 +291,7 @@ export class ZLinkSerialExecutionQueue {
     return result;
   }
 
-  private scheduleDrain(): void {
+  scheduleDrain(): void {
     if (this.sharedOwner !== undefined) {
       if (this.activeRecord !== undefined) return;
       // Ready membership in the existing owner queue is the only notification state.
@@ -320,7 +320,7 @@ export class ZLinkSerialExecutionQueue {
   }
 
   /** Notifications carry no payload, admission sequence, result Promise, or host permit. */
-  private enqueueMailboxTurn(operation: () => void): void {
+  enqueueMailboxTurn(operation: () => void): void {
     const entry: SerialWorkRecord<unknown> = {
       acceptedSequence: 0n,
       lane: 'application',
@@ -384,7 +384,7 @@ export class ZLinkSerialExecutionQueue {
     });
   }
 
-  private async drain(): Promise<void> {
+  async drain(): Promise<void> {
     const consumer = this.consumer!;
     if (consumer.running) return;
     consumer.running = true;
@@ -461,7 +461,7 @@ export class ZLinkSerialExecutionQueue {
     return record.preparationState !== 'pending' && record.preparationState !== 'canceling';
   }
 
-  private selectNext(): SerialWorkRecord<unknown> | undefined {
+  selectNext(): SerialWorkRecord<unknown> | undefined {
     const lifecycleOwner = this.activeLifecycle?.context as ZLinkSerialLifecycleContext | undefined;
     let applicationRecord: SerialWorkRecord<unknown> | undefined;
     if (lifecycleOwner === undefined) {
@@ -488,7 +488,7 @@ export class ZLinkSerialExecutionQueue {
     return applicationRecord!;
   }
 
-  private takeSelected(selected: SerialWorkRecord<unknown>): ZLinkSerialWorkRecord<unknown> {
+  takeSelected(selected: SerialWorkRecord<unknown>): ZLinkSerialWorkRecord<unknown> {
     const record =
       selected.lane === 'lifecycle' ? this.takeLifecycle() : this.takeApplication(selected);
     if (record !== selected) {
@@ -508,7 +508,7 @@ export class ZLinkSerialExecutionQueue {
     return record;
   }
 
-  private takeApplication(selected: SerialWorkRecord<unknown>): ZLinkSerialWorkRecord<unknown> {
+  takeApplication(selected: SerialWorkRecord<unknown>): ZLinkSerialWorkRecord<unknown> {
     this.lifecycleStreak = 0;
     this.lifecycleDebt = false;
     if (this.application.records[0] === selected) return this.application.records.shift()!;
@@ -577,7 +577,7 @@ export class ZLinkSerialExecutionQueue {
     return performance.now() - (this.claimStartedAt ?? performance.now()) >= this.ownerTimeBudget;
   }
 
-  private resolveIdleWaiters(): void {
+  resolveIdleWaiters(): void {
     for (let index = 0; index < this.idleWaiters.length; index++) this.idleWaiters[index]!();
     this.idleWaiters.length = 0;
   }

package/src/runtime/foundation/raw-service-mesh-runtime.ts

@@ -22,6 +22,7 @@ import type {
   ZLinkRawRouterPort
 } from '../backend/raw-binding-port';
 import { submitToRequestResult } from '../messaging/submission-result';
+import { copyPayloadBytes } from '../../contracts/Common/payload-slab';
 import { RequestResult, SubmitResult, isZLinkBackendResultError } from '../backend/runtime-values';
 import { enumWireRejectReason } from '../protocol/service_wire_codec.generated';
 import { SERVICE_WIRE_PREFIX_SIZE } from './service-wire-binary-primitives';
@@ -46,9 +47,11 @@ import {
   SERVICE_WIRE_MAGIC,
   SERVICE_WIRE_MAJOR,
   ServiceWireCommand,
-  ServiceWireFrameworkErrorCode
+  ServiceWireFrameworkErrorCode,
+  recoverRequestCorrelation
 } from './service-wire-constants.generated';
 import {
+  decodeApplicationPayload,
   decodeApplicationPayloadView,
   decodeChannelRequestHeader,
   decodeChannelSendHeader,
@@ -204,7 +207,7 @@ export class RawServiceMeshRuntime {
   private readonly releaseOperations: () => void;
 
   private readonly operations: OperationRegistry<RawServiceRequestResult, number>;
-  private readonly expectedPeers = new Map<
+  readonly expectedPeers = new Map<
     string,
     {
       readonly meshName: string;
@@ -221,7 +224,7 @@ export class RawServiceMeshRuntime {
    * The last observed Core selected route of each RID (Core ROUTER §10.1):
    * RID -> route generation. Only `observeSelectedRoutes` replaces it.
    */
-  private selectedRoutes = new Map<string, bigint>();
+  selectedRoutes = new Map<string, bigint>();
   private connectionAttempts: readonly ConnectionAttempt[] = [];
   private readonly bindingPort: ZLinkRawBindingPort;
   private readonly applicationJobQueue: ApplicationJobQueuePort;
@@ -231,7 +234,7 @@ export class RawServiceMeshRuntime {
   private receiveAcquisition?: Promise<ApplicationJobPermitPort>;
   private receivePermit?: ApplicationJobPermitPort;
   private receiveRotation?: ApplicationJobReceiveRotationPort;
-  private readonly controlSends = new Map<string, ControlSendState>();
+  readonly controlSends = new Map<string, ControlSendState>();
   private infrastructureFailure?: { readonly error: unknown };
   private readonly onReceiveReady?: () => void;
   private readonly infrastructureTaskRunner?: RawServiceMeshRuntimeOptions['infrastructureTaskRunner'];
@@ -242,7 +245,7 @@ export class RawServiceMeshRuntime {
   private readonly resolveAdvertisedEndpoint?: RawServiceMeshRuntimeOptions['resolveAdvertisedEndpoint'];
   private descriptor: ServiceNodeDescriptor;
   private host?: ZLinkRawHostPort;
-  private router?: ZLinkRawRouterPort;
+  router?: ZLinkRawRouterPort;
   private nextCorrelation = 1n;
   private serviceIngress?: RawServiceIngressHandler;
   private serviceIngressAccepts?: (command: number) => boolean;
@@ -1189,7 +1192,7 @@ export class RawServiceMeshRuntime {
     }
   }
 
-  private reportProtocolError(
+  reportProtocolError(
     received: import('../backend/node/node-raw-binding-port').ZLinkRawReceivedRecord
   ): void {
     const request = received.requestSeq !== undefined;
@@ -1206,32 +1209,19 @@ export class RawServiceMeshRuntime {
     received: import('../backend/node/node-raw-binding-port').ZLinkRawReceivedRecord
   ): boolean {
     if (received.requestSeq === undefined || received.parts.length === 0) return false;
-    try {
-      const header = decodeHeader(received.parts[0]!);
-      const correlation =
-        header.command === M6aServiceWireCommand.nodeRequest
-          ? decodeNodeRequestHeader(received.parts[0]!)
-          : header.command === M6aServiceWireCommand.channelRequest
-            ? decodeChannelRequestHeader(received.parts[0]!).correlation
-            : undefined;
-      if (correlation === undefined) return false;
-      const reply = [
-        encodeReplyHeader(
-          correlation,
-          RequestResult.ProtocolError,
-          ServiceWireFrameworkErrorCode.requestProtocolError
-        )
-      ];
-      if (received.reply === undefined) return false;
-      received.reply(reply);
-      return true;
-    } catch (error) {
-      if (error instanceof ServiceWireProtocolError) return false;
-      throw error;
-    }
+    const correlation = recoverRequestCorrelation(received.parts[0]!);
+    if (correlation === undefined || received.reply === undefined) return false;
+    received.reply([
+      encodeReplyHeader(
+        correlation,
+        RequestResult.ProtocolError,
+        ServiceWireFrameworkErrorCode.requestProtocolError
+      )
+    ]);
+    return true;
   }
 
-  private processReceived(
+  processReceived(
     received: ZLinkRawReceivedRecord,
     nowMs: number,
     applicationJobOwner: ApplicationIngressRecordOwner
@@ -1604,11 +1594,24 @@ export class RawServiceMeshRuntime {
         this.submitControl(probe.nodeRoutingId, parts)
       );
     }
+    const failures: unknown[] = [];
     for (const nodeRoutingId of result.timedOutNodes) {
       const peer = this.topology.peer(nodeRoutingId);
       if (peer === undefined) continue;
-      this.removePeer(peer, () => this.requireStarted().disconnectRid!(nodeRoutingId));
+      try {
+        this.removePeer(peer, () => {
+          try {
+            this.requireStarted().disconnectRid!(nodeRoutingId);
+          } catch (error) {
+            failures.push(error);
+          }
+        });
+      } catch (error) {
+        failures.push(error);
+      }
     }
+    if (failures.length === 1) throw failures[0];
+    if (failures.length > 1) throw new AggregateError(failures, 'Liveness peer cleanup failed.');
     return result;
   }
 
@@ -1698,7 +1701,7 @@ export class RawServiceMeshRuntime {
       channelName === undefined
         ? encodeNodeRequestHeader(correlation)
         : encodeChannelRequestHeader(correlation, channelName);
-    const encodedPayload = this.applicationFrame(payload);
+    let encodedPayload = this.applicationFrame(payload);
     let selectedTargetNodeRoutingId = targetNodeRoutingId;
     if (
       selectedTargetNodeRoutingId !== this.descriptor.nodeRoutingId &&
@@ -1726,6 +1729,7 @@ export class RawServiceMeshRuntime {
       }
     }
     if (selectedTargetNodeRoutingId === this.descriptor.nodeRoutingId) {
+      encodedPayload = copyPayloadBytes(encodedPayload);
       // The local admission path aborts its capacity wait on the terminal
       // Promise, so a local request always keeps the Promise entry.
       const pending =
@@ -1916,10 +1920,10 @@ export class RawServiceMeshRuntime {
   }
 
   private applicationPayload(payload: ServiceApplicationPayloadInput): ServiceApplicationPayload {
-    return Buffer.isBuffer(payload) ? decodeApplicationPayloadView(payload) : payload;
+    return Buffer.isBuffer(payload) ? decodeApplicationPayload(payload) : payload;
   }
 
-  private admitPeer(
+  admitPeer(
     descriptor: ServiceNodeDescriptor,
     connectionId: string | bigint,
     nowMs: number,
@@ -2021,10 +2025,7 @@ export class RawServiceMeshRuntime {
     return this.router;
   }
 
-  private async sendControl(
-    targetNodeRoutingId: string,
-    parts: readonly Uint8Array[]
-  ): Promise<boolean> {
+  async sendControl(targetNodeRoutingId: string, parts: readonly Uint8Array[]): Promise<boolean> {
     return this.startControlSend(targetNodeRoutingId, decodeHeader(parts[0]!).command, () =>
       this.submitControl(targetNodeRoutingId, parts)
     );
@@ -2044,7 +2045,7 @@ export class RawServiceMeshRuntime {
     );
   }
 
-  private startControlSend(
+  startControlSend(
     target: string,
     command: number,
     attempt: () => boolean | Promise<boolean>,

package/src/runtime/foundation/service-liveness-registry.ts

@@ -60,7 +60,7 @@ const NO_LIVENESS_WORK: ServiceLivenessTick = Object.freeze({
 });
 
 export class ServiceLivenessRegistry {
-  private readonly peers = new Map<string, ServiceLivenessConnection>();
+  readonly peers = new Map<string, ServiceLivenessConnection>();
   private nextProbeId = 1n;
   /**
    * The earliest probe or timeout among admitted peers. Received traffic only

package/src/runtime/foundation/service-mailbox.ts

@@ -9,6 +9,7 @@ export interface ServiceMailboxDispatchOwner {
 }
 
 export interface ServiceMailboxRecord {
+  receivedAt?: Date;
   readonly sourcePreparationOwner?: ServiceMailboxDispatchOwner;
   readonly ingress?: RawServiceIngressRecord;
   readonly owner: string;
@@ -82,7 +83,10 @@ export class ServiceMailbox {
   private nextRelocationSerial = 1n;
   private closed = false;
 
-  constructor(private readonly onReady?: (domain: ServiceMailboxDomain) => void) {
+  constructor(
+    private readonly onReady?: (domain: ServiceMailboxDomain) => void,
+    private readonly receivedAtClock: () => Date = () => new Date()
+  ) {
     // Host ApplicationJobQueue permits own ordinary-record admission. The
     // Framework-owned payload lifetime does not retain Core HWM credit. A
     // second mailbox cap would reject already-admitted work before the shared
@@ -105,6 +109,7 @@ export class ServiceMailbox {
     const target = this.domain(record.domain);
     const bytes = retainedBytes(record);
     if (this.closed) return false;
+    record.receivedAt ??= this.receivedAtClock();
     if (record.domain === 'application') {
       if (this.relocatedOwners.has(record.owner)) return false;
       const relocation = this.relocationSeals.get(record.owner);

package/src/runtime/foundation/service-runtime-contracts.ts

@@ -253,6 +253,7 @@ export interface ZLinkMessageFollowOrigin {
 }
 
 export interface ReceiveRecord {
+  readonly receivedAt?: Date;
   /** Resolve record prerequisites in owner order, outside an application handler turn. */
   readonly prepareDispatch?: (
     this: ReceiveRecord,
@@ -276,6 +277,7 @@ export interface ReceiveRecord {
   readonly kindData: ReceiveKindData | null;
   readonly terminalResult: number;
   readonly failureErrno: number;
+  readonly failureMessage?: string;
   readonly parts: Message[];
   /** Returns whether the local operation can still accept a lifecycle reply. */
   readonly isPending?: () => boolean;
@@ -299,7 +301,11 @@ export interface ReceiveRecord {
     replyContentType?: string
   ): SubmitResult;
   /** Emits a typed command-20 terminal for a lifecycle request before admission. */
-  replyFailure?(terminalResult: number, failureCode: number): SubmitResult;
+  replyFailure?(
+    terminalResult: number,
+    failureCode: number,
+    tail?: import('./service-stateful-wire-codec').ServiceStatefulReplyTail
+  ): SubmitResult;
 }
 
 export interface ClaimReceiveResult {