@zlink-systems/framework 0.29.0 → 0.29.1

package/src/runtime/locations/in-memory-location-store.ts

@@ -99,11 +99,14 @@ export class ZLinkInMemoryLocationStore
   private readonly authority: ZLinkInMemoryAuthorityStore;
 
   constructor(private readonly now: () => Date = () => new Date()) {
+    const isOwnerLive = (owner: ZLinkLocationOwnerToken): boolean =>
+      this.liveOwnerLease(owner.ownerId, this.now())?.token.leaseGeneration ===
+      owner.leaseGeneration;
     this.authority = new ZLinkInMemoryAuthorityStore(
       {
+        isOwnerLive,
         isTargetLive: (key, lifecycleGeneration, owner) => {
           const descriptor = this.meshNodes.rows.get(meshNodeKey(key.meshName, key.rid));
-          const lease = this.leases.get(owner.ownerId);
           return (
             descriptor !== undefined &&
             descriptor.lifecycleGeneration === lifecycleGeneration &&
@@ -111,9 +114,7 @@ export class ZLinkInMemoryLocationStore
             descriptor.leaseGeneration === owner.leaseGeneration &&
             descriptor.objectRole === ZLinkObjectRole.Server &&
             descriptor.state === ZLinkFrameworkRuntimeState.Serving &&
-            lease !== undefined &&
-            lease.token.leaseGeneration === owner.leaseGeneration &&
-            lease.leaseExpiresAt.getTime() > this.now().getTime()
+            isOwnerLive(owner)
           );
         },
         placementCapacityAvailable: (key, requested, reserved, active) => {
@@ -566,7 +567,8 @@ export class ZLinkInMemoryLocationStore
     const storeNow = this.now();
     return pageRows(
       this.fanoutPublishers,
-      (row) => row.channelName === channelName && this.isOwnerLive(row.ownerId, storeNow),
+      (row) =>
+        row.channelName === channelName && this.liveOwnerLease(row.ownerId, storeNow) !== undefined,
       page
     );
   }
@@ -628,7 +630,7 @@ export class ZLinkInMemoryLocationStore
     if (
       intent === ZLinkLocationWriteIntent.NewClaim &&
       current !== undefined &&
-      this.isOwnerLive(current.ownerId, updatedAt)
+      this.liveOwnerLease(current.ownerId, updatedAt) !== undefined
     ) {
       return rejectedConflict();
     }
@@ -706,7 +708,7 @@ export class ZLinkInMemoryLocationStore
     if (
       intent === ZLinkLocationWriteIntent.NewClaim &&
       current !== undefined &&
-      this.isOwnerLive(current.ownerId, updatedAt)
+      this.liveOwnerLease(current.ownerId, updatedAt) !== undefined
     ) {
       return rejectedConflict();
     }
@@ -1155,7 +1157,7 @@ export class ZLinkInMemoryLocationStore
     if (
       intent === ZLinkLocationWriteIntent.NewClaim &&
       current !== undefined &&
-      this.isOwnerLive(ownerOf(current), updatedAt)
+      this.liveOwnerLease(ownerOf(current), updatedAt) !== undefined
     ) {
       return rejectedConflict();
     }
@@ -1252,9 +1254,11 @@ export class ZLinkInMemoryLocationStore
     return removed;
   }
 
-  private isOwnerLive(ownerId: string, now: Date): boolean {
+  private liveOwnerLease(ownerId: string, now: Date): InMemoryOwnerLease | undefined {
     const lease = this.leases.get(ownerId);
-    return lease !== undefined && lease.leaseExpiresAt.getTime() > now.getTime();
+    return lease !== undefined && lease.leaseExpiresAt.getTime() > now.getTime()
+      ? lease
+      : undefined;
   }
 
   private bump(kind: ZLinkLocationKind, meshName: string | undefined): void {

package/src/runtime/locations/internal-location-contracts.ts

@@ -1,4 +1,10 @@
 export * from '../../contracts/Locations';
+import type { ZLinkObjectReserveRequest as ReserveRequest } from '../../contracts/Locations/Authority';
+import type { ZLinkRelocationConfiguration } from '../../contracts/Configuration/RegistrationTypes';
+
+export interface ZLinkObjectReserveRequest extends ReserveRequest {
+  readonly actorRelocationPolicy?: ZLinkRelocationConfiguration<unknown>['kind'];
+}
 export type * from '../../contracts/Locations/Authority';
 export type {
   ZLinkActorLocation,

package/src/runtime/locations/location-store-repository.ts

@@ -1130,19 +1130,31 @@ export class ZLinkLocationStoreRepository extends ZLinkInMemoryLocationStore {
     expectedStoreVersion: string,
     signal?: AbortSignal
   ): Promise<boolean> {
+    return (
+      (await this.tryReleaseEndedReservation(key, expectedStoreVersion, signal)) === 'reclaimed'
+    );
+  }
+
+  private async tryReleaseEndedReservation(
+    key: ZLinkAuthorityKey,
+    expectedStoreVersion: string,
+    signal?: AbortSignal,
+    request?: ZLinkObjectReserveRequest
+  ): Promise<'notReclaimable' | 'conflict' | 'reclaimed'> {
     const rowKey = authorityKey(key.value);
     const current = await this.provider.read(rowKey, signal);
     if (current.kind === 'missing' || current.value.version.value !== expectedStoreVersion)
-      return false;
+      return 'conflict';
     const record = decodeAuthorityRecord(current.value.bytes);
     const snapshot = record.snapshot;
     if (
-      snapshot.allocation.state !== 'reserved' ||
-      snapshot.pendingCreation === undefined ||
+      (snapshot.allocation.state === 'active'
+        ? request?.key.kind !== 'actor'
+        : snapshot.pendingCreation === undefined) ||
       record.aggregate !== undefined ||
       hasRelocationAuthority(snapshot.payload)
     )
-      return false;
+      return 'notReclaimable';
     const leaseKey = ownerKey(snapshot.ownerId);
     const descriptorKey = meshKey(
       snapshot.allocation.descriptor.meshName,
@@ -1159,27 +1171,56 @@ export class ZLinkLocationStoreRepository extends ZLinkInMemoryLocationStore {
     ]);
     if (lease.kind === 'found' && lease.value.expiresAt === undefined)
       throw new Error('Location Store owner lease record is invalid.');
-    if (
-      sameLiveOwner(lease, snapshot) &&
+    const ownerLive = sameLiveOwner(lease, snapshot);
+    if (snapshot.allocation.state === 'active') {
+      if (ownerLive) return 'notReclaimable';
+      if (request?.actorRelocationPolicy !== 'disabled')
+        throw new ZLinkFrameworkException(
+          ZLinkFrameworkErrorKind.Unavailable,
+          'Actor owner lease is unavailable.'
+        );
+    } else if (
+      ownerLive &&
       descriptor.kind === 'found' &&
       reviveMeshDescriptor(
         decodeCanonicalDescriptorRecord<ZLinkMeshNodeDescriptor>(descriptor.value.bytes).descriptor
       ).lifecycleGeneration === snapshot.allocation.descriptorLifecycleGeneration
     )
-      return false;
+      return 'notReclaimable';
     const mutations: ZLinkStoreWriteRequest['mutations'][number][] = [
       { kind: 'delete', key: rowKey }
     ];
     if (capacity.kind === 'found') {
       const stored = decodeJson<CapacityRecord>(capacity.value.bytes);
-      mutations.push({
-        kind: 'put',
-        key: capacityRowKey,
-        bytes: encodeJson({
-          active: stored.active,
-          pending: subtractCapacity(stored.pending, snapshot.allocation.capacity)
-        } satisfies CapacityRecord)
-      });
+      try {
+        mutations.push({
+          kind: 'put',
+          key: capacityRowKey,
+          bytes: encodeJson({
+            active:
+              snapshot.allocation.state === 'active'
+                ? subtractCapacity(stored.active, snapshot.allocation.capacity)
+                : stored.active,
+            pending:
+              snapshot.allocation.state === 'reserved'
+                ? subtractCapacity(stored.pending, snapshot.allocation.capacity)
+                : stored.pending
+          } satisfies CapacityRecord)
+        });
+      } catch (error) {
+        if (
+          !(error instanceof ZLinkFrameworkException) ||
+          error.kind !== ZLinkFrameworkErrorKind.DataLost
+        )
+          throw error;
+        const observed = await this.provider.read(rowKey, signal);
+        if (
+          observed.kind === 'missing' ||
+          observed.value.version.value !== current.value.version.value
+        )
+          return 'conflict';
+        throw error;
+      }
     }
     const result = await this.provider.write(
       {
@@ -1193,7 +1234,7 @@ export class ZLinkLocationStoreRepository extends ZLinkInMemoryLocationStore {
       },
       signal
     );
-    return result.kind === 'applied';
+    return result.kind === 'applied' ? 'reclaimed' : 'conflict';
   }
 
   override async reserve(
@@ -1203,6 +1244,7 @@ export class ZLinkLocationStoreRepository extends ZLinkInMemoryLocationStore {
   ): Promise<ZLinkObjectReserveResult> {
     const encodedAuthorityKey = encodeAuthorityKey(request.key.kind, request.key.globalId);
     const reservationId = randomUUID();
+    let reclaimVersion: string | undefined;
     for (;;) {
       throwIfOperationExpired(signal, deadlineUnixMs);
       const rowKey = authorityKey(encodedAuthorityKey.value);
@@ -1223,19 +1265,23 @@ export class ZLinkLocationStoreRepository extends ZLinkInMemoryLocationStore {
         ) {
           return { kind: 'typeMismatch', current: snapshot };
         }
-        if (snapshot.allocation.state === 'active') {
-          return { kind: 'alreadyExists', current: snapshot };
-        }
-        if (
-          !(await this.releaseEndedReservation(
-            encodedAuthorityKey,
-            current.value.version.value,
-            signal
-          ))
-        )
+        if (reclaimVersion !== undefined && snapshot.storeVersion.value !== reclaimVersion)
           return { kind: 'conflict', current: snapshot };
+        const reclaim = await this.tryReleaseEndedReservation(
+          encodedAuthorityKey,
+          current.value.version.value,
+          signal,
+          request
+        );
+        if (reclaim === 'notReclaimable')
+          return snapshot.allocation.state === 'active'
+            ? { kind: 'alreadyExists', current: snapshot }
+            : { kind: 'conflict', current: snapshot };
+        reclaimVersion = reclaim === 'conflict' ? snapshot.storeVersion.value : undefined;
         continue;
       }
+      if (reclaimVersion !== undefined)
+        return { kind: 'conflict', current: { kind: 'missing', storeNow: current.storeNow } };
       const [descriptorRead, leaseRead] = await Promise.all([
         this.provider.read(descriptorKey, signal),
         this.provider.read(leaseKey, signal)
@@ -4040,13 +4086,21 @@ function adjustCapacity(
   if (delta.spotType !== undefined) {
     const key = capacityTypeKey(delta.spotType);
     const next = (spotTypes[key] ?? 0) + direction * delta.spotType.count;
-    if (next < 0) throw new Error('Provider capacity counter underflow.');
+    if (next < 0)
+      throw new ZLinkFrameworkException(
+        ZLinkFrameworkErrorKind.DataLost,
+        'Provider capacity counter underflow.'
+      );
     if (next === 0) delete spotTypes[key];
     else spotTypes[key] = next;
   }
   const actors = usage.actors + direction * delta.actors;
   const spots = usage.spots + direction * delta.spots;
-  if (actors < 0 || spots < 0) throw new Error('Provider capacity counter underflow.');
+  if (actors < 0 || spots < 0)
+    throw new ZLinkFrameworkException(
+      ZLinkFrameworkErrorKind.DataLost,
+      'Provider capacity counter underflow.'
+    );
   return { actors, spots, spotTypes };
 }
 

package/src/runtime/spots/spot-entry-activation.ts

@@ -378,6 +378,10 @@ export class ZLinkEntrySpotActivation {
     return this.spotSerialExecutor.executeActor(actorId, operation);
   }
 
+  terminateActor(actorId: string, terminal: () => Promise<void>): Promise<void> {
+    return this.spotSerialExecutor.terminateActor(actorId, terminal);
+  }
+
   /**
    * Entry Spot membership does not have an application admission callback.
    * The shared core round-trip therefore accepts a valid returning Actor and

package/src/runtime/spots/spot-node-runtime-manager.ts

@@ -783,6 +783,13 @@ export class ZLinkSpotNodeRuntimeManager {
     return activation?.notifyJoinActor(actor, signal) ?? Promise.resolve();
   }
 
+  terminateActorActivation(actor: ZLinkActor, terminal: () => Promise<void>): Promise<void> {
+    const activation = this.entryActivations.get(actor.context.meshName);
+    return activation === undefined
+      ? terminal()
+      : activation.terminateActor(actor.context.actorId, terminal);
+  }
+
   notifyPrimaryEntrySpotActorLeft(
     actor: ZLinkActor,
     signal?: AbortSignal,

package/src/runtime/spots/spot-serial-executor.ts

@@ -51,10 +51,8 @@ export class ZLinkSpotSerialExecutor {
     operation: (serial: ZLinkSpotSerialTurnExecutor) => Promise<T> | T,
     workOptions?: ZLinkSerialWorkOptions
   ): Promise<T> {
-    return this.actorExecutor(actorId).execute(
-      () => operation(this.actorSerial(actorId)),
-      workOptions
-    );
+    const serial = this.actorSerial(actorId);
+    return this.actorExecutor(actorId).execute(() => operation(serial), workOptions);
   }
 
   /** @internal Restores a durable Actor FIFO prefix without starting it. */
@@ -102,6 +100,14 @@ export class ZLinkSpotSerialExecutor {
     return this.spotSerial.execute(operation, { ...workOptions, lane: 'lifecycle' });
   }
 
+  async terminateActor(actorId: string, terminal: () => Promise<void>): Promise<void> {
+    const executor = this.actorExecutor(actorId);
+    await executor.close();
+    await terminal();
+    if (this.actorExecutors.get(actorId) === executor) this.actorExecutors.delete(actorId);
+    this.actorSerials.delete(actorId);
+  }
+
   /**
    * Makes all child queues unreachable and completes them in one synchronous
    * ownership turn. Already accepted work remains drainable.