@zlink-systems/framework 0.27.0 → 0.29.0

package/src/runtime/host/spot-address-transport.ts

@@ -55,12 +55,6 @@ export interface ZLinkHostSpotAddressTransportOptions {
   readonly completions: (meshName: string) => ZLinkMeshCompletionTable | undefined;
   readonly codecs?: ZLinkChannelEnvelopeCodecRegistry;
   readonly defaultRequestTimeoutMs: number;
-  /** Internal one-way deadline policy; public calls do not expose this option. */
-  readonly defaultSendTimeoutMs?: number;
-  /** Resolves the send timeout for a configured Spot Mesh. */
-  readonly sendTimeoutMsForMesh?: (meshName: string) => number;
-  /** Resolves the send timeout for a resolved Spot route channel. */
-  readonly sendTimeoutMsForRouteChannel?: (routeChannelId: string) => number;
   readonly dispatchErrors?: ZLinkDispatchErrorReporter;
 }
 
@@ -93,7 +87,7 @@ export class ZLinkHostSpotAddressTransport implements ZLinkSpotAddressTransport
   sendToSpotAddress(
     spotId: RoutingId,
     message: unknown,
-    call: ZLinkSpotAddressCallOptions
+    call: Omit<ZLinkSpotAddressCallOptions, 'timeoutMs' | 'signal'>
   ): Promise<ZLinkSubmitResult> {
     // Call-scoped flow (spec 27 §4): the envelope encoders and the
     // traceInstanceAddress points share one ambient flow that does not
@@ -107,157 +101,136 @@ export class ZLinkHostSpotAddressTransport implements ZLinkSpotAddressTransport
   private async sendToSpotAddressScoped(
     spotId: RoutingId,
     message: unknown,
-    call: ZLinkSpotAddressCallOptions
+    call: Omit<ZLinkSpotAddressCallOptions, 'timeoutMs' | 'signal'>
   ): Promise<ZLinkSubmitResult> {
-    const timeoutMs = initialSendTimeoutMs(this.options, call);
-    const deadline = createSpotAddressDeadline(timeoutMs, call.signal);
-    try {
-      const existing = await this.resolveExisting(spotId, deadline.signal);
-      deadline.requireRemaining();
-      if (existing !== undefined) {
-        this.validateExisting(existing, call);
-        deadline.setOwnerTimeout(this.sendTimeoutMsForRouteChannel(existing.routerChannelId));
-        try {
-          const result = await awaitWithAbort(
-            this.options.routed.sendToSpot(existing, message, {
-              instanceSpot: call.instanceSpot,
-              timeoutMs: deadline.requireRemaining(),
-              signal: deadline.signal,
-              metadata: call.metadata
-            }),
-            deadline.signal
+    const activationDeadlineUnixMs = BigInt(
+      Math.round(Date.now() + this.options.defaultRequestTimeoutMs)
+    );
+    const existing = await this.resolveExisting(spotId);
+    if (existing !== undefined) {
+      this.validateExisting(existing, call);
+      try {
+        const result = await this.options.routed.sendToSpot(existing, message, {
+          instanceSpot: call.instanceSpot,
+          metadata: call.metadata
+        });
+        if (result.status === ZLinkSubmitStatus.Submitted) {
+          this.traceInstanceAddress(
+            ZLinkMessageFlowOutcome.Sent,
+            ZLinkDispatchMessageKind.Send,
+            spotId,
+            message,
+            existing.routerChannelId,
+            existing.stableType,
+            existing.targetNodeRid
           );
-          if (result.status === ZLinkSubmitStatus.Submitted) {
-            this.traceInstanceAddress(
-              ZLinkMessageFlowOutcome.Sent,
-              ZLinkDispatchMessageKind.Send,
-              spotId,
-              message,
-              existing.routerChannelId,
-              existing.stableType,
-              existing.targetNodeRid
-            );
-          } else {
-            this.traceInstanceAddress(
-              submitResultFlowOutcome(result.status),
-              ZLinkDispatchMessageKind.Send,
-              spotId,
-              message,
-              existing.routerChannelId,
-              existing.stableType,
-              existing.targetNodeRid,
-              submitResultReason(result.status)
-            );
-          }
-          if (
-            result.status === ZLinkSubmitStatus.TargetNotFound ||
-            result.status === ZLinkSubmitStatus.RouteNotConnected
-          ) {
-            this.options.resolver()?.invalidate?.(spotId);
-          }
-          return result;
-        } catch (error) {
-          if (isSpotRouteRefreshError(error)) {
-            this.options.resolver()?.invalidate?.(spotId);
-          }
-          const reason = addressedInstanceErrorReason(error);
+        } else {
           this.traceInstanceAddress(
-            reason === ZLinkDispatchErrorReason.Backpressure
-              ? ZLinkMessageFlowOutcome.Backpressured
-              : ZLinkMessageFlowOutcome.Dropped,
+            submitResultFlowOutcome(result.status),
             ZLinkDispatchMessageKind.Send,
             spotId,
             message,
             existing.routerChannelId,
             existing.stableType,
             existing.targetNodeRid,
-            reason
+            submitResultReason(result.status)
           );
-          throw error;
         }
-      }
-      if (!call.instanceSpot) {
-        return { status: ZLinkSubmitStatus.TargetNotFound };
-      }
-      const selected = this.selectMissingTarget(spotId, call);
-      if (selected.kind === 'unsupported') {
-        this.traceInstanceAddress(
-          ZLinkMessageFlowOutcome.Dropped,
-          ZLinkDispatchMessageKind.Send,
-          spotId,
-          message,
-          call.initialMeshName,
-          call.instanceSpotType,
-          undefined,
-          ZLinkDispatchErrorReason.StaleTarget
-        );
-        return { status: ZLinkSubmitStatus.TargetNotFound };
-      }
-      if (selected.kind === 'capacity') {
-        const error = missingInstancePlacementCapacity(spotId, call.instanceSpotType);
+        if (
+          result.status === ZLinkSubmitStatus.TargetNotFound ||
+          result.status === ZLinkSubmitStatus.RouteNotConnected
+        ) {
+          this.options.resolver()?.invalidate?.(spotId);
+        }
+        return result;
+      } catch (error) {
+        if (isSpotRouteRefreshError(error)) {
+          this.options.resolver()?.invalidate?.(spotId);
+        }
+        const reason = addressedInstanceErrorReason(error);
         this.traceInstanceAddress(
-          ZLinkMessageFlowOutcome.Backpressured,
+          reason === ZLinkDispatchErrorReason.Backpressure
+            ? ZLinkMessageFlowOutcome.Backpressured
+            : ZLinkMessageFlowOutcome.Dropped,
           ZLinkDispatchMessageKind.Send,
           spotId,
           message,
-          call.initialMeshName,
-          call.instanceSpotType,
-          undefined,
-          ZLinkDispatchErrorReason.Backpressure
+          existing.routerChannelId,
+          existing.stableType,
+          existing.targetNodeRid,
+          reason
         );
         throw error;
       }
-      if (selected.kind === 'unavailable') {
-        this.traceInstanceAddress(
-          ZLinkMessageFlowOutcome.Dropped,
-          ZLinkDispatchMessageKind.Send,
-          spotId,
-          message,
-          call.initialMeshName,
-          call.instanceSpotType,
-          undefined,
-          ZLinkDispatchErrorReason.StaleTarget
-        );
-        return { status: ZLinkSubmitStatus.RouteNotConnected };
-      }
-      deadline.setOwnerTimeout(this.sendTimeoutMsForMesh(selected.meshName));
-      const encoded = this.encode(ZLinkChannelMessageKind.Command, selected.meshName, message);
-      const sourceSpotId =
-        call.sourceSpot === undefined ? undefined : String(call.sourceSpot.routingId);
-      const mapped = classifySubmitResult(
-        await awaitWithAbort(
-          selected.node.sendToMissingInstanceSpot(
-            selected.target,
-            encoded,
-            BigInt(deadline.deadlineUnixMs),
-            sourceSpotId,
-            call.metadata
-          ),
-          deadline.signal
-        ),
-        'Instance Spot submission'
+    }
+    if (!call.instanceSpot) {
+      return { status: ZLinkSubmitStatus.TargetNotFound };
+    }
+    const selected = this.selectMissingTarget(spotId, call);
+    if (selected.kind === 'unsupported') {
+      this.traceInstanceAddress(
+        ZLinkMessageFlowOutcome.Dropped,
+        ZLinkDispatchMessageKind.Send,
+        spotId,
+        message,
+        call.initialMeshName,
+        call.instanceSpotType,
+        undefined,
+        ZLinkDispatchErrorReason.StaleTarget
       );
+      return { status: ZLinkSubmitStatus.TargetNotFound };
+    }
+    if (selected.kind === 'capacity') {
+      const error = missingInstancePlacementCapacity(spotId, call.instanceSpotType);
       this.traceInstanceAddress(
-        submitResultFlowOutcome(mapped.status),
+        ZLinkMessageFlowOutcome.Backpressured,
         ZLinkDispatchMessageKind.Send,
         spotId,
         message,
-        selected.meshName,
-        selected.target.stableType,
-        selected.target.targetNodeRid,
-        mapped.status === ZLinkSubmitStatus.Submitted
-          ? undefined
-          : submitResultReason(mapped.status)
+        call.initialMeshName,
+        call.instanceSpotType,
+        undefined,
+        ZLinkDispatchErrorReason.Backpressure
       );
-      return mapped;
-    } catch (error) {
-      if (deadline.expired()) {
-        return { status: ZLinkSubmitStatus.TimedOut };
-      }
       throw error;
-    } finally {
-      deadline.close();
     }
+    if (selected.kind === 'unavailable') {
+      this.traceInstanceAddress(
+        ZLinkMessageFlowOutcome.Dropped,
+        ZLinkDispatchMessageKind.Send,
+        spotId,
+        message,
+        call.initialMeshName,
+        call.instanceSpotType,
+        undefined,
+        ZLinkDispatchErrorReason.StaleTarget
+      );
+      return { status: ZLinkSubmitStatus.RouteNotConnected };
+    }
+    const encoded = this.encode(ZLinkChannelMessageKind.Command, selected.meshName, message);
+    const sourceSpotId =
+      call.sourceSpot === undefined ? undefined : String(call.sourceSpot.routingId);
+    const mapped = classifySubmitResult(
+      await selected.node.sendToMissingInstanceSpot(
+        selected.target,
+        encoded,
+        activationDeadlineUnixMs,
+        sourceSpotId,
+        call.metadata
+      ),
+      'Instance Spot submission'
+    );
+    this.traceInstanceAddress(
+      submitResultFlowOutcome(mapped.status),
+      ZLinkDispatchMessageKind.Send,
+      spotId,
+      message,
+      selected.meshName,
+      selected.target.stableType,
+      selected.target.targetNodeRid,
+      mapped.status === ZLinkSubmitStatus.Submitted ? undefined : submitResultReason(mapped.status)
+    );
+    return mapped;
   }
 
   requestToSpotAddress<TReply = unknown>(
@@ -603,7 +576,7 @@ export class ZLinkHostSpotAddressTransport implements ZLinkSpotAddressTransport
     if (stableType === undefined) {
       if (distinctTypes.length === 0) return { kind: 'unsupported' };
       throw createInternalFrameworkException(
-        ZLinkFrameworkInternalErrorKind.InvalidConfiguration,
+        ZLinkFrameworkInternalErrorKind.InvalidOperation,
         'Instance Spot type is required when multiple types are registered.'
       );
     }
@@ -642,22 +615,6 @@ export class ZLinkHostSpotAddressTransport implements ZLinkSpotAddressTransport
     return { kind: 'unavailable' };
   }
 
-  private sendTimeoutMsForMesh(meshName: string): number {
-    return (
-      this.options.sendTimeoutMsForMesh?.(meshName) ??
-      this.options.defaultSendTimeoutMs ??
-      this.options.defaultRequestTimeoutMs
-    );
-  }
-
-  private sendTimeoutMsForRouteChannel(routeChannelId: string): number {
-    return (
-      this.options.sendTimeoutMsForRouteChannel?.(routeChannelId) ??
-      this.options.defaultSendTimeoutMs ??
-      this.options.defaultRequestTimeoutMs
-    );
-  }
-
   private validateExisting(
     target: import('../spots/spot-routing-internal').ZLinkSpotRouteTarget,
     call: ZLinkSpotAddressCallOptions
@@ -845,21 +802,9 @@ function addressedInstanceErrorReason(error: unknown): ZLinkDispatchErrorReason
   return ZLinkDispatchErrorReason.HandlerException;
 }
 
-function initialSendTimeoutMs(
-  options: ZLinkHostSpotAddressTransportOptions,
-  call: ZLinkSpotAddressCallOptions
-): number {
-  return call.initialMeshName === undefined
-    ? (options.defaultSendTimeoutMs ?? options.defaultRequestTimeoutMs)
-    : (options.sendTimeoutMsForMesh?.(call.initialMeshName) ??
-        options.defaultSendTimeoutMs ??
-        options.defaultRequestTimeoutMs);
-}
-
 interface ZLinkSpotAddressDeadline {
   readonly deadlineUnixMs: number;
   readonly signal: AbortSignal;
-  setOwnerTimeout(timeoutMs: number): void;
   requireRemaining(): number;
   expired(): boolean;
   close(): void;
@@ -871,7 +816,7 @@ function createSpotAddressDeadline(
 ): ZLinkSpotAddressDeadline {
   const startedAtMs = performance.now();
   const startedAtUnixMs = Date.now();
-  let deadlineMs = startedAtMs + Math.max(0, timeoutMs);
+  const deadlineMs = startedAtMs + Math.max(0, timeoutMs);
   const controller = new AbortController();
   let expired = false;
   let timeout: ReturnType<typeof setTimeout> | undefined;
@@ -897,10 +842,6 @@ function createSpotAddressDeadline(
       return Math.round(startedAtUnixMs + (deadlineMs - startedAtMs));
     },
     signal: controller.signal,
-    setOwnerTimeout(ownerTimeoutMs: number) {
-      deadlineMs = startedAtMs + Math.max(0, ownerTimeoutMs);
-      arm();
-    },
     requireRemaining() {
       const remainingMs = deadlineMs - performance.now();
       if (remainingMs <= 0) {

package/src/runtime/host/stateful-authority-route-runtime.ts

@@ -1,3 +1,4 @@
+import { encodeAuthorityKey } from '../locations/authority-key-codec';
 import { createHash } from 'node:crypto';
 import type {
   ZLinkAuthorityKey,
@@ -175,12 +176,25 @@ export class ZLinkStatefulAuthorityRouteRuntime {
 
       const status = sink.status();
       for (const pending of snapshot.pending) {
+        if (pending.targetMeshName !== meshName) continue;
         if (
-          pending.targetMeshName === meshName &&
           routingIdsEqual(status.routingId, pending.targetNodeRid) &&
           status.lifecycleGeneration === pending.targetNodeGeneration
         ) {
           await this.recoverPendingInstanceActivation(sink, pending, signal);
+        } else {
+          const release = this.options.creationStore?.releaseEndedReservation;
+          if (release === undefined)
+            throw new Error('Reservation lifecycle recovery requires a Location repository.');
+          if (
+            await release.call(
+              this.options.creationStore,
+              encodeAuthorityKey('instance_spot', pending.spotId),
+              pending.pending.storeVersion,
+              signal
+            )
+          )
+            await this.clearRecoveryRoot(pending.pending.requestReference, signal);
         }
       }
 
@@ -386,14 +400,7 @@ export class ZLinkStatefulAuthorityRouteRuntime {
         signal
       );
       if (aborted.kind === 'aborted' || aborted.kind === 'alreadyAborted') {
-        try {
-          await this.options.relocationStore?.delete(
-            relocationBlobReference(recovery.pending.requestReference)
-          );
-        } catch {
-          // The authority no longer publishes this root. Retention owns a
-          // failed best-effort orphan cleanup.
-        }
+        await this.clearRecoveryRoot(recovery.pending.requestReference, signal);
       }
       return;
     }
@@ -401,6 +408,12 @@ export class ZLinkStatefulAuthorityRouteRuntime {
       return;
   }
 
+  private async clearRecoveryRoot(reference: string, signal?: AbortSignal): Promise<void> {
+    if (this.options.relocationStore === undefined)
+      throw new Error('Instance activation recovery requires a Relocation Store.');
+    await this.options.relocationStore.delete(relocationBlobReference(reference), signal);
+  }
+
   private async readRecoveryEnvelope(
     recovery: ServiceActivationRecoveryState,
     signal?: AbortSignal

package/src/runtime/locations/domain-store-contract.ts

@@ -140,6 +140,11 @@ export interface ZLinkDomainLocationStore {
     request: ZLinkObjectCreationCompleteRequest,
     signal?: AbortSignal
   ): Promise<ZLinkObjectCreationCompleteResult>;
+  releaseEndedReservation?(
+    key: ZLinkAuthorityKey,
+    expectedStoreVersion: string,
+    signal?: AbortSignal
+  ): Promise<boolean>;
   abort(
     request: ZLinkObjectAbortRequest,
     signal?: AbortSignal,

package/src/runtime/locations/in-memory-authority-store.ts

@@ -276,6 +276,42 @@ export class ZLinkInMemoryAuthorityStore {
     return BigInt(removed);
   }
 
+  async releaseEndedReservation(
+    key: ZLinkAuthorityKey,
+    expectedStoreVersion: string,
+    signal?: AbortSignal
+  ): Promise<boolean> {
+    signal?.throwIfAborted();
+    return this.tryReleaseEndedReservation(key, expectedStoreVersion);
+  }
+
+  private tryReleaseEndedReservation(
+    key: ZLinkAuthorityKey,
+    expectedStoreVersion: string
+  ): boolean {
+    const current = this.rows.get(key.value);
+    if (
+      current === undefined ||
+      current.snapshot.storeVersion.value !== expectedStoreVersion ||
+      current.snapshot.allocation.state !== 'reserved' ||
+      current.creation === undefined ||
+      this.validation.isTargetLive(
+        current.snapshot.allocation.descriptor,
+        current.snapshot.allocation.descriptorLifecycleGeneration,
+        {
+          ownerId: current.snapshot.ownerId,
+          leaseGeneration: current.snapshot.ownerLeaseGeneration
+        }
+      )
+    )
+      return false;
+    this.adjustCapacity(this.pendingCapacity, current.snapshot.allocation, -1);
+    this.rows.delete(key.value);
+    this.creationTerminals.set(current.creation.reservationId, 'aborted');
+    this.scanRevision++;
+    return true;
+  }
+
   async reserve(
     request: ZLinkObjectReserveRequest,
     signal?: AbortSignal
@@ -297,15 +333,13 @@ export class ZLinkInMemoryAuthorityStore {
       if (current.snapshot.allocation.state === 'active') {
         return { kind: 'alreadyExists', current: this.snapshot(current.snapshot) };
       }
-      // An unfinished creation whose owner lease ended is cancellable.
-      // Reclaim it so the key does not stay blocked forever.
-      if (this.isOwnerLive(current.snapshot) || current.creation === undefined) {
+      if (
+        !this.tryReleaseEndedReservation(
+          encodeAuthorityKey(request.key.kind, request.key.globalId),
+          current.snapshot.storeVersion.value
+        )
+      )
         return { kind: 'conflict', current: this.read(key) };
-      }
-      this.adjustCapacity(this.pendingCapacity, current.snapshot.allocation, -1);
-      this.rows.delete(key);
-      this.creationTerminals.set(current.creation.reservationId, 'aborted');
-      this.scanRevision++;
     }
     const target = creationTarget(request);
     if (

package/src/runtime/locations/in-memory-location-store.ts

@@ -201,6 +201,14 @@ export class ZLinkInMemoryLocationStore
     return this.authority.readCreationTerminal(operation, signal);
   }
 
+  async releaseEndedReservation(
+    key: ZLinkAuthorityKey,
+    expectedStoreVersion: string,
+    signal?: AbortSignal
+  ): Promise<boolean> {
+    return this.authority.releaseEndedReservation(key, expectedStoreVersion, signal);
+  }
+
   async abort(
     request: ZLinkObjectAbortRequest,
     signal?: AbortSignal

package/src/runtime/locations/internal-store-contracts.ts

@@ -32,6 +32,7 @@ export type ZLinkObjectCreationStore = Pick<
   | 'commit'
   | 'completeCreation'
   | 'abort'
+  | 'releaseEndedReservation'
   | 'prepareAggregate'
   | 'commitAggregate'
   | 'abortAggregate'