@zlink-systems/framework 0.28.0 → 0.29.1

package/src/runtime/foundation/service-stateful-wire-codec.ts

@@ -167,8 +167,6 @@ export type ServiceWireRelocationObject =
       readonly objectGeneration: bigint;
     };
 
-/** Shared CRC-32C wire bounds for the direct relocation payload transfer. */
-export const RELOCATION_PAYLOAD_TOTAL_LENGTH_MAX = 274_877_906_944n;
 export const RELOCATION_PAYLOAD_CHUNK_COUNT_MAX = 4096;
 export { RELOCATION_STATE_CHUNK_DATA_MAX_BYTES } from '../../contracts/Configuration/InternalDefaults';
 
@@ -2658,29 +2656,6 @@ export function decodeServiceWireFrozenRecord(bytes: Uint8Array): ServiceWireFro
   return { ...record, canonicalBytes: Buffer.from(bytes) };
 }
 
-/**
- * Consumes exactly one canonical frozen record at `offset`.  Relocation
- * saved-work is a vector of back-to-back records, so its boundary is the
- * record grammar itself rather than an enclosing length field.
- */
-export function decodeServiceWireFrozenRecordPrefix(
-  bytes: Uint8Array,
-  offset = 0
-): { readonly record: ServiceWireFrozenRecord; readonly length: number } {
-  if (!Number.isSafeInteger(offset) || offset < 0 || offset >= bytes.byteLength) {
-    fail('Frozen record offset is outside the encoded bytes.');
-  }
-  const reader = new FrozenReader(bytes.subarray(offset));
-  const record = decodeServiceWireFrozenRecordFrom(reader);
-  return {
-    record: {
-      ...record,
-      canonicalBytes: Buffer.from(bytes.subarray(offset, offset + reader.position))
-    },
-    length: reader.position
-  };
-}
-
 function decodeServiceWireFrozenRecordFrom(reader: FrozenReader): ServiceWireFrozenRecord {
   const recordKind = reader.u8('recordKind');
   if (recordKind < 1 || recordKind > 14) fail('Invalid frozen record kind.');

package/src/runtime/foundation/service-topology-registry.ts

@@ -305,35 +305,46 @@ export class ServiceTopologyRegistry {
     stableType: string,
     isReady: (descriptor: ServiceNodeDescriptor) => boolean = () => true
   ): ServiceNodeDescriptor | undefined {
+    return this.selectObjectPlacementWithStatus(stableType, isReady).descriptor;
+  }
+
+  selectObjectPlacementWithStatus(
+    stableType: string,
+    isReady: (descriptor: ServiceNodeDescriptor) => boolean = () => true
+  ): {
+    readonly status: ServiceObjectPlacementStatus;
+    readonly descriptor?: ServiceNodeDescriptor;
+  } {
     requireText(stableType, 'stableType');
-    return this.selectWeightedCycle(
+    const candidates = this.objectPlacementCandidates(stableType, isReady);
+    if (candidates.status !== 'available') return { status: candidates.status };
+    const descriptor = this.selectWeightedCycle(
       `object:${stableType}`,
-      () => {
-        const capability = `object-type:${stableType}`;
-        return [this.local, ...this.peersByRid.values()]
-          .map((value) => ('descriptor' in value ? value.descriptor : value))
-          .filter(
-            (descriptor) =>
-              descriptor.state === 'serving' &&
-              descriptor.objectRole === 'server' &&
-              descriptor.placementWeight > 0 &&
-              descriptor.activeCapacityUsed < descriptor.activeCapacityLimit &&
-              descriptor.pendingCapacityUsed < descriptor.pendingCapacityLimit &&
-              descriptor.protocolCapabilities.includes(capability)
-          )
-          .map(cloneDescriptor);
-      },
+      () => this.objectPlacementCandidates(stableType).eligible.map(cloneDescriptor),
       (descriptor) => descriptor.placementWeight,
       (descriptor) => descriptor.nodeRoutingId,
       (left, right) => compareOrdinal(left.nodeRoutingId, right.nodeRoutingId),
       isReady
     );
+    return descriptor === undefined
+      ? { status: 'unavailable' }
+      : { status: 'available', descriptor };
   }
 
   objectPlacementStatus(
     stableType: string,
     isReady: (descriptor: ServiceNodeDescriptor) => boolean = () => true
   ): ServiceObjectPlacementStatus {
+    return this.objectPlacementCandidates(stableType, isReady).status;
+  }
+
+  private objectPlacementCandidates(
+    stableType: string,
+    isReady: (descriptor: ServiceNodeDescriptor) => boolean = () => true
+  ): {
+    readonly status: ServiceObjectPlacementStatus;
+    readonly eligible: readonly ServiceNodeDescriptor[];
+  } {
     requireText(stableType, 'stableType');
     const capability = `object-type:${stableType}`;
     const supported = [
@@ -345,16 +356,15 @@ export class ServiceTopologyRegistry {
         descriptor.objectRole === 'server' &&
         descriptor.protocolCapabilities.includes(capability)
     );
-    if (supported.length === 0) return 'unsupported';
+    if (supported.length === 0) return { status: 'unsupported', eligible: [] };
     const withCapacity = supported.filter(
       (descriptor) =>
         descriptor.activeCapacityUsed < descriptor.activeCapacityLimit &&
         descriptor.pendingCapacityUsed < descriptor.pendingCapacityLimit
     );
-    if (withCapacity.length === 0) return 'capacity';
+    if (withCapacity.length === 0) return { status: 'capacity', eligible: [] };
     const weighted = withCapacity.filter((descriptor) => descriptor.placementWeight > 0);
-    if (weighted.length === 0) return 'unsupported';
-    return weighted.some(isReady) ? 'available' : 'unavailable';
+    return { status: weighted.some(isReady) ? 'available' : 'unavailable', eligible: weighted };
   }
 
   instanceSpotPlacementTypes(): readonly string[] {

package/src/runtime/handlers/handler-instance-scope.ts

@@ -212,7 +212,8 @@ class LifecycleHandlerInstanceScope {
   }
 
   async dispose(detachedTaskRunner: ZLinkDetachedTaskRunner): Promise<void> {
-    const disposeFromActiveInvocation = activeLifecycleScope.getStore() === this;
+    const disposeFromActiveInvocation =
+      this.activeInvocations > 0 && activeLifecycleScope.getStore() === this;
     const disposal = this.beginDisposeCore();
     if (disposal.deferred !== undefined) {
       startOutsideLifecycleInvocation(() => {

package/src/runtime/host/actor-placement-coordinator.ts

@@ -11,7 +11,8 @@ import type {
   ZLinkAuthoritySnapshot,
   ZLinkCreationOperationIdentity,
   ZLinkLocationOwnerToken,
-  ZLinkObjectReserveResult
+  ZLinkObjectReserveResult,
+  ZLinkObjectReserveRequest
 } from '../locations/internal-location-contracts';
 import type {
   ZLinkAuthorityStore,
@@ -57,6 +58,10 @@ export interface ZLinkActorPlacementTarget {
 }
 
 export interface ZLinkActorPlacementCoordinatorOptions {
+  readonly actorRelocationPolicy?: (
+    meshName: string,
+    stableType: string
+  ) => ZLinkObjectReserveRequest['actorRelocationPolicy'];
   readonly store: ZLinkObjectCreationStore & ZLinkAuthorityStore;
   readonly target: (
     meshName: string | undefined,
@@ -130,7 +135,8 @@ export class ZLinkActorPlacementCoordinator {
             },
             target: reservationTarget(target),
             creatingPayload,
-            capacity: { actors: 1, spots: 0 }
+            capacity: { actors: 1, spots: 0 },
+            actorRelocationPolicy: this.options.actorRelocationPolicy?.(target.meshName, stableType)
           },
           deadline.signal,
           BigInt(deadlineUnixMs)

package/src/runtime/host/actor-runtime-options-factory.ts

@@ -34,6 +34,10 @@ import type { ZLinkRemoteActorPacketTarget } from '../actors';
 import type { ZLinkActorJoinRelocation } from '../actors/actor-join-relocation';
 
 export interface ZLinkActorRuntimeOptionsFactoryOptions {
+  readonly terminateActorActivation: (
+    actor: ZLinkActor,
+    terminal: () => Promise<void>
+  ) => Promise<void>;
   readonly registration: ZLinkFrameworkRegistration;
   /** Message flow tracer, so a failed deferred Join records its cause on the flow. */
   readonly messageFlow?: () =>
@@ -101,6 +105,7 @@ export class ZLinkActorRuntimeOptionsFactory {
     | 'actorRefResolver'
     | 'actorCreatedNotifier'
     | 'actorDestroyedCleanup'
+    | 'terminateActorActivation'
     | 'publishActorAuthority'
     | 'locationLifecycle'
     | 'boundSessionFactory'
@@ -111,6 +116,7 @@ export class ZLinkActorRuntimeOptionsFactory {
   > {
     const actorTransferRegistry = this.options.actorTransferRegistry;
     return {
+      terminateActorActivation: this.options.terminateActorActivation,
       actorMeshNameProvider: this.options.actorMeshName,
       actorLeaveSpot: (meshName, spotId, actor, signal) => {
         const spotManager = this.options.spotManager();

package/src/runtime/host/index.ts

@@ -2016,6 +2016,7 @@ export class ZLinkFrameworkRuntimeHost
     | 'actorRefResolver'
     | 'actorCreatedNotifier'
     | 'actorDestroyedCleanup'
+    | 'terminateActorActivation'
     | 'locationLifecycle'
     | 'boundSessionFactory'
     | 'shutdownSignal'
@@ -2139,6 +2140,9 @@ export class ZLinkFrameworkRuntimeHost
     }
     this.actorPlacement ??= new ZLinkActorPlacementCoordinator({
       store: locationStore,
+      actorRelocationPolicy: (meshName, stableType) =>
+        this.options.registration.spotNodes.get(meshName)?.actorFactoryRegistrations?.[stableType]
+          ?.relocation.kind,
       remoteCreate: (meshName, targetNodeRid, request, timeoutMs) => {
         const node = this.spotNodeRuntime?.meshNode(meshName);
         if (node === undefined) {
@@ -2194,14 +2198,12 @@ export class ZLinkFrameworkRuntimeHost
             const capability = descriptor.objectCapabilities.find(
               (candidate) => candidate.objectKind === 'actor' && candidate.stableType === stableType
             );
-            const actors = descriptor.populationCapacity.actors;
             return (
               descriptor.state === ZLinkFrameworkRuntimeState.Serving &&
               descriptor.objectRole === 'server' &&
               descriptor.placementWeight > 0 &&
               descriptor.entrySpotId !== undefined &&
               excludedNodeRids?.has(String(descriptor.rid)) !== true &&
-              (actors.limit === 0 || actors.active + actors.reserved < actors.limit) &&
               capability !== undefined
             );
           }
@@ -2592,6 +2594,8 @@ export class ZLinkFrameworkRuntimeHost
 
   private actorRuntimeOptionsFactory(): ZLinkActorRuntimeOptionsFactory {
     return new ZLinkActorRuntimeOptionsFactory({
+      terminateActorActivation: (actor, terminal) =>
+        this.spotNodeRuntime?.terminateActorActivation(actor, terminal) ?? terminal(),
       registration: this.options.registration,
       messageFlow: () => this.createDispatchErrorReporter(this.runtimeOrPreStartErrorSink).flow,
       routeTransport: this.routeTransport,

package/src/runtime/host/instance-activation-authority.ts

@@ -16,7 +16,7 @@ import type {
   ZLinkAuthorityStore,
   ZLinkObjectCreationStore
 } from '../locations/internal-store-contracts';
-import { ZLinkFrameworkException } from '../../contracts';
+import { ZLinkFrameworkErrorKind, ZLinkFrameworkException } from '../../contracts';
 import {
   decodeServiceInstanceAuthorityPayload,
   decodeServiceReadySpotAuthority,
@@ -46,7 +46,6 @@ import {
 
 const CREATION_REQUEST_RETENTION_MS = 24 * 60 * 60 * 1_000;
 const MAX_CREATION_REQUEST_BYTES = 1024 * 1024;
-const ACTIVATION_JOIN_POLL_INTERVAL_MS = 10;
 
 interface PendingReservation {
   readonly reservationId: string;
@@ -115,87 +114,81 @@ export class ZLinkInstanceActivationAuthority implements ServiceAsyncInstanceAct
     }
     let reserved: Awaited<ReturnType<ZLinkObjectCreationStore['reserve']>>;
     try {
-      for (;;) {
-        reserved = await this.options.store.reserve(
-          {
-            key: { kind: 'instance_spot', globalId: target.targetSpotId },
-            intent: {
-              stableType: target.stableType,
-              requestContentReference: stored.reference.value,
-              requestSha256: requestHash,
-              requestEncodedSize: BigInt(requestBytes.byteLength)
-            },
-            target: {
-              meshName: this.options.meshName,
-              nodeRid: target.targetNodeRid,
-              nodeLifecycleGeneration: target.targetNodeGeneration,
-              owner
-            },
-            creatingPayload: encodeServiceInstanceAuthorityPayload({
-              state: 'coldActivating',
+      reserved = await this.options.store.reserve(
+        {
+          key: { kind: 'instance_spot', globalId: target.targetSpotId },
+          intent: {
+            stableType: target.stableType,
+            requestContentReference: stored.reference.value,
+            requestSha256: requestHash,
+            requestEncodedSize: BigInt(requestBytes.byteLength)
+          },
+          target: {
+            meshName: this.options.meshName,
+            nodeRid: target.targetNodeRid,
+            nodeLifecycleGeneration: target.targetNodeGeneration,
+            owner
+          },
+          creatingPayload: encodeServiceInstanceAuthorityPayload({
+            state: 'coldActivating',
+            stableType: target.stableType,
+            spotId: target.targetSpotId,
+            ownerId: owner.ownerId,
+            ownerLeaseGeneration: owner.leaseGeneration,
+            ownerMeshName: this.options.meshName,
+            ownerNodeRid: target.targetNodeRid,
+            ownerNodeGeneration: target.targetNodeGeneration
+          }),
+          capacity: {
+            actors: 0,
+            spots: 1,
+            spotType: {
+              objectKind: 'instance_spot',
               stableType: target.stableType,
-              spotId: target.targetSpotId,
-              ownerId: owner.ownerId,
-              ownerLeaseGeneration: owner.leaseGeneration,
-              ownerMeshName: this.options.meshName,
-              ownerNodeRid: target.targetNodeRid,
-              ownerNodeGeneration: target.targetNodeGeneration
-            }),
-            capacity: {
-              actors: 0,
-              spots: 1,
-              spotType: {
-                objectKind: 'instance_spot',
-                stableType: target.stableType,
-                count: 1
-              }
+              count: 1
             }
-          },
-          undefined,
-          activation.deadlineUnixMs
-        );
-        if (reserved.kind !== 'alreadyExists') break;
-        this.options.metrics?.recordInstanceSpotClaimConflict(
-          this.options.meshName,
-          target.stableType,
-          'already_exists'
-        );
-        await this.deleteOrphan(stored.reference);
-        return readyExisting(reserved.current, target);
-      }
+          }
+        },
+        undefined,
+        activation.deadlineUnixMs
+      );
     } catch (error) {
       await this.deleteOrphan(stored.reference);
       throw error;
     }
-    if (
-      reserved.kind === 'conflict' &&
-      reserved.current.kind === 'snapshot' &&
-      reserved.current.allocation.objectKind === 'instance_spot' &&
-      reserved.current.allocation.stableType === target.stableType &&
-      reserved.current.allocation.state === 'reserved'
-    ) {
-      this.options.metrics?.recordInstanceSpotClaimConflict(
-        this.options.meshName,
-        target.stableType,
-        'activation_in_progress'
-      );
-      await this.deleteOrphan(stored.reference);
-      return await this.awaitConcurrentActivation(target, activation.deadlineUnixMs);
-    }
     if (reserved.kind !== 'reserved') {
+      const conflictReason =
+        reserved.kind === 'alreadyExists'
+          ? 'already_exists'
+          : reserved.kind === 'conflict' &&
+              reserved.current.kind === 'snapshot' &&
+              reserved.current.allocation.objectKind === 'instance_spot' &&
+              reserved.current.allocation.stableType === target.stableType &&
+              reserved.current.allocation.state === 'reserved'
+            ? 'activation_in_progress'
+            : reserved.kind;
       this.options.metrics?.recordInstanceSpotClaimConflict(
         this.options.meshName,
         target.stableType,
-        reserved.kind
+        conflictReason
       );
       await this.deleteOrphan(stored.reference);
+      if (reserved.kind === 'typeMismatch') {
+        throw createInternalFrameworkException(
+          ZLinkFrameworkInternalErrorKind.SpotTypeMismatch,
+          `Instance activation reservation failed: ${reserved.kind}.`
+        );
+      }
       if (reserved.kind === 'placementCapacityExhausted') {
         throw createInternalFrameworkException(
           ZLinkFrameworkInternalErrorKind.PlacementCapacityExhausted,
           `Instance Spot placement capacity is exhausted for '${String(target.targetSpotId)}'.`
         );
       }
-      throw new Error(`Instance activation reservation failed: ${reserved.kind}.`);
+      throw new ZLinkFrameworkException(
+        ZLinkFrameworkErrorKind.Unavailable,
+        `Instance activation reservation failed: ${reserved.kind}.`
+      );
     }
     const reservation: ServiceInstanceActivationReservation = {
       attempt: reserved.creating.objectGeneration,
@@ -455,34 +448,6 @@ export class ZLinkInstanceActivationAuthority implements ServiceAsyncInstanceAct
     }
   }
 
-  private async awaitConcurrentActivation(
-    target: ServiceInstanceActivationTarget,
-    deadlineUnixMs: bigint
-  ): Promise<Extract<ServiceInstanceAuthorityRead, { readonly kind: 'ready' }>> {
-    const key = authorityKey(target.targetSpotId);
-    const deadlineMs = performance.now() + Number(deadlineUnixMs - BigInt(Date.now()));
-    while (performance.now() <= deadlineMs) {
-      const current = await this.options.store.readAuthority(key);
-      if (current.kind === 'snapshot') {
-        if (
-          current.allocation.objectKind !== 'instance_spot' ||
-          current.allocation.stableType !== target.stableType
-        ) {
-          throw new Error('Concurrent Instance activation resolved to a different Spot type.');
-        }
-        if (current.allocation.state === 'active') {
-          return readyExisting(current, target);
-        }
-      } else {
-        throw new Error('Concurrent Instance activation ended before the Ready barrier.');
-      }
-      await waitForActivationJoin(deadlineMs);
-    }
-    throw new Error(
-      `Concurrent Instance activation for '${target.targetSpotId}' did not reach Ready before deadline.`
-    );
-  }
-
   private requirePending(reservation: ServiceInstanceActivationReservation): PendingReservation {
     const pending = this.pending.get(reservation.token);
     if (pending === undefined || pending.creating.objectGeneration !== reservation.attempt) {
@@ -539,38 +504,6 @@ function readyRead(
   return { kind: 'ready', route: routeFromSnapshot(snapshot), authority: snapshot };
 }
 
-function readyExisting(
-  snapshot: ZLinkAuthoritySnapshot,
-  target: ServiceInstanceActivationTarget
-): Extract<ServiceInstanceAuthorityRead, { readonly kind: 'ready' }> {
-  const decoded =
-    decodeServiceReadySpotAuthority(snapshot.payload) ??
-    decodeServiceClosingSpotAuthority(snapshot.payload);
-  if (
-    decoded?.kind !== 'instance_spot' ||
-    decoded.stableType !== target.stableType ||
-    decoded.spotId !== target.targetSpotId ||
-    snapshot.allocation.objectKind !== 'instance_spot' ||
-    snapshot.allocation.state !== 'active' ||
-    decoded.ownerId !== snapshot.ownerId ||
-    decoded.ownerLeaseGeneration !== snapshot.ownerLeaseGeneration ||
-    decoded.ownerMeshName !== snapshot.allocation.descriptor.meshName ||
-    decoded.ownerNodeGeneration !== snapshot.allocation.descriptorLifecycleGeneration ||
-    !routingIdsEqual(decoded.ownerNodeRid, snapshot.allocation.descriptor.rid)
-  ) {
-    throw new Error('Existing Instance authority is not a matching Ready allocation.');
-  }
-  return { kind: 'ready', route: routeFromSnapshot(snapshot), authority: snapshot };
-}
-
-async function waitForActivationJoin(deadlineMs: number): Promise<void> {
-  const remaining = deadlineMs - performance.now();
-  if (remaining <= 0) return;
-  await new Promise<void>((resolve) => {
-    setTimeout(resolve, Math.min(ACTIVATION_JOIN_POLL_INTERVAL_MS, remaining));
-  });
-}
-
 function requireCommitIdentity(
   target: ServiceInstanceActivationTarget,
   reservation: ServiceInstanceActivationReservation,

package/src/runtime/host/spot-address-transport.ts

@@ -576,7 +576,7 @@ export class ZLinkHostSpotAddressTransport implements ZLinkSpotAddressTransport
     if (stableType === undefined) {
       if (distinctTypes.length === 0) return { kind: 'unsupported' };
       throw createInternalFrameworkException(
-        ZLinkFrameworkInternalErrorKind.InvalidConfiguration,
+        ZLinkFrameworkInternalErrorKind.InvalidOperation,
         'Instance Spot type is required when multiple types are registered.'
       );
     }

package/src/runtime/host/stateful-authority-route-runtime.ts

@@ -1,3 +1,4 @@
+import { encodeAuthorityKey } from '../locations/authority-key-codec';
 import { createHash } from 'node:crypto';
 import type {
   ZLinkAuthorityKey,
@@ -175,12 +176,25 @@ export class ZLinkStatefulAuthorityRouteRuntime {
 
       const status = sink.status();
       for (const pending of snapshot.pending) {
+        if (pending.targetMeshName !== meshName) continue;
         if (
-          pending.targetMeshName === meshName &&
           routingIdsEqual(status.routingId, pending.targetNodeRid) &&
           status.lifecycleGeneration === pending.targetNodeGeneration
         ) {
           await this.recoverPendingInstanceActivation(sink, pending, signal);
+        } else {
+          const release = this.options.creationStore?.releaseEndedReservation;
+          if (release === undefined)
+            throw new Error('Reservation lifecycle recovery requires a Location repository.');
+          if (
+            await release.call(
+              this.options.creationStore,
+              encodeAuthorityKey('instance_spot', pending.spotId),
+              pending.pending.storeVersion,
+              signal
+            )
+          )
+            await this.clearRecoveryRoot(pending.pending.requestReference, signal);
         }
       }
 
@@ -386,14 +400,7 @@ export class ZLinkStatefulAuthorityRouteRuntime {
         signal
       );
       if (aborted.kind === 'aborted' || aborted.kind === 'alreadyAborted') {
-        try {
-          await this.options.relocationStore?.delete(
-            relocationBlobReference(recovery.pending.requestReference)
-          );
-        } catch {
-          // The authority no longer publishes this root. Retention owns a
-          // failed best-effort orphan cleanup.
-        }
+        await this.clearRecoveryRoot(recovery.pending.requestReference, signal);
       }
       return;
     }
@@ -401,6 +408,12 @@ export class ZLinkStatefulAuthorityRouteRuntime {
       return;
   }
 
+  private async clearRecoveryRoot(reference: string, signal?: AbortSignal): Promise<void> {
+    if (this.options.relocationStore === undefined)
+      throw new Error('Instance activation recovery requires a Relocation Store.');
+    await this.options.relocationStore.delete(relocationBlobReference(reference), signal);
+  }
+
   private async readRecoveryEnvelope(
     recovery: ServiceActivationRecoveryState,
     signal?: AbortSignal

package/src/runtime/locations/domain-store-contract.ts

@@ -37,9 +37,9 @@ import type {
   ZLinkObjectCommitResult,
   ZLinkObjectCreationCompleteRequest,
   ZLinkObjectCreationCompleteResult,
-  ZLinkObjectReserveRequest,
   ZLinkObjectReserveResult
 } from '../../contracts/Locations/Authority';
+import type { ZLinkObjectReserveRequest } from './internal-location-contracts';
 
 /**
  * Framework-private domain repository. Provider implementations never
@@ -140,6 +140,11 @@ export interface ZLinkDomainLocationStore {
     request: ZLinkObjectCreationCompleteRequest,
     signal?: AbortSignal
   ): Promise<ZLinkObjectCreationCompleteResult>;
+  releaseEndedReservation?(
+    key: ZLinkAuthorityKey,
+    expectedStoreVersion: string,
+    signal?: AbortSignal
+  ): Promise<boolean>;
   abort(
     request: ZLinkObjectAbortRequest,
     signal?: AbortSignal,