@zlink-systems/framework 0.25.0 → 0.27.0

package/src/runtime/locations/relocation-blob.ts

@@ -1,11 +1,11 @@
 import { randomUUID } from 'node:crypto';
+import { awaitWithAbort, macrotaskBoundary } from '../abort';
 import type {
   ZLinkBlobReference,
+  ZLinkBlobPutResult,
   ZLinkRelocationStore
 } from '../../contracts/Locations/RelocationStore';
 
-const MAX_REFERENCE_GENERATION_ATTEMPTS = 4;
-
 export function relocationBlobReference(value: string): ZLinkBlobReference {
   return { value } as ZLinkBlobReference;
 }
@@ -20,10 +20,21 @@ export async function putNewRelocationBlob(
   readonly expiresAt: Date;
   readonly storeNow: Date;
 }> {
-  for (let attempt = 0; attempt < MAX_REFERENCE_GENERATION_ATTEMPTS; attempt += 1) {
+  let reference = relocationBlobReference(randomUUID());
+  for (; ; await macrotaskBoundary()) {
     signal?.throwIfAborted();
-    const reference = relocationBlobReference(randomUUID());
-    const result = await store.put(reference, payload, retentionMs, signal);
+    let result: ZLinkBlobPutResult;
+    try {
+      result = await awaitWithAbort(store.put(reference, payload, retentionMs, signal), signal);
+    } catch {
+      signal?.throwIfAborted();
+      const read = await awaitWithAbort(store.read(reference, signal), signal);
+      if (read.kind === 'found' && Buffer.compare(read.bytes, payload) === 0) {
+        return { reference, expiresAt: read.expiresAt, storeNow: read.storeNow };
+      }
+      if (read.kind === 'missing') continue;
+      result = { kind: 'conflict', storeNow: read.storeNow };
+    }
     if (result.kind !== 'conflict') {
       return {
         reference,
@@ -31,6 +42,6 @@ export async function putNewRelocationBlob(
         storeNow: result.storeNow
       };
     }
+    reference = relocationBlobReference(randomUUID());
   }
-  throw new Error('Unable to allocate a unique Relocation Store reference.');
 }

package/src/runtime/locations/resolvers.ts

@@ -1,26 +1,53 @@
+import { listAllMeshNodeDescriptors } from './location-store-pages';
+import type { ActorRef, RoutingId, SpotId } from '../../contracts/Common';
+import { ZLinkFrameworkErrorKind, ZLinkFrameworkException } from '../../contracts/Errors';
+import { zlinkDefaultLocationOptions } from '../../contracts/Locations/Options';
+import { ZLinkSpotKind } from '../../contracts/Spots';
+import { decodeActorAuthorityIdentity } from '../actors/actor-authority-publication';
+import { waitActorReply } from '../actors/actor-request-deadline';
+import { emitActorOwnerLeaseObservation, isRelocationDebugEnabled } from '../diagnostics';
+import { ZLinkStateLane } from '../execution/state-lane';
+import {
+  decodeServiceClosingSpotAuthority,
+  decodeServiceReadySpotAuthority
+} from '../foundation/service-authority-payload-codec';
+import { serviceRelocationAuthorityApplicationPayload } from '../foundation/service-relocation-runtime';
 import {
   ZLinkFrameworkInternalErrorKind,
   createInternalFrameworkException,
   internalFrameworkErrorKind
 } from '../framework-errors-internal';
-import type { ActorRef, RoutingId, SpotId } from '../../contracts/Common';
+import { routingIdsEqual } from '../routing-id';
 import {
+  type SpotHandle,
+  type ZLinkActorSpotHandleResolver,
+  type ZLinkSpotHandleResolver,
+  createSpotHandle,
+  type ResolvedSpotHandle
+} from '../spots/spot-handle';
+
+import type { ZLinkSpotRouteResolver, ZLinkSpotRouteTarget } from '../spots/spot-routing-internal';
+import { encodeAuthorityKey } from './authority-key-codec';
+import { isKnownZLinkLocationAutoConnectType, isKnownZLinkLocationRole } from './canonical-codec';
+import {
+  type ZLinkAuthoritySnapshot,
+  ZLinkFrameworkRuntimeState,
   ZLinkLocationRole,
   ZLinkLocationTopologyState,
-  ZLinkFrameworkRuntimeState,
   ZLinkObjectRole,
-  type ZLinkLocationReadiness,
-  type ZLinkLocationRuntimeQuery,
-  type ZLinkPeerLocationResolver,
   type ZLinkActorLocation,
   type ZLinkActorLocationKey,
+  type ZLinkLocationReadiness,
+  type ZLinkLocationRuntimeQuery,
   type ZLinkPeerLocation,
   type ZLinkPeerLocationFilter,
+  type ZLinkPeerLocationResolver,
   type ZLinkRouteLocation,
   type ZLinkRouteLocationKey,
   type ZLinkSpotLocation,
   type ZLinkSpotLocationKey
 } from './internal-location-contracts';
+
 import type {
   ZLinkActorLocationStore,
   ZLinkAuthorityStore,
@@ -29,25 +56,7 @@ import type {
   ZLinkRouteLocationStore,
   ZLinkSpotLocationStore
 } from './internal-store-contracts';
-import { decodeServiceReadySpotAuthority } from '../foundation/service-authority-payload-codec';
-import { serviceRelocationAuthorityApplicationPayload } from '../foundation/service-relocation-runtime';
-import { decodeActorAuthorityIdentity } from '../actors/actor-authority-publication';
-import { encodeAuthorityKey } from './authority-key-codec';
-import { ZLinkSpotKind } from '../../contracts/Spots';
-import type { ZLinkAuthoritySnapshot } from './internal-location-contracts';
-import type {
-  SpotHandle,
-  ZLinkActorSpotHandleResolver,
-  ZLinkSpotHandleResolver
-} from '../spots/spot-handle';
-import { ZLinkFrameworkErrorKind, ZLinkFrameworkException } from '../../contracts/Errors';
-import type { ZLinkSpotRouteResolver, ZLinkSpotRouteTarget } from '../spots/spot-routing-internal';
-import { createSpotHandle, type ResolvedSpotHandle } from '../spots/spot-handle';
-import { isKnownZLinkLocationAutoConnectType, isKnownZLinkLocationRole } from './canonical-codec';
 import { ZLinkLiveRowFilter, ZLinkOwnerLeaseTracker } from './lease-tracker';
-import { routingIdsEqual } from '../routing-id';
-import { ZLinkStateLane } from '../execution/state-lane';
-import { emitActorOwnerLeaseObservation } from '../diagnostics';
 
 export interface ZLinkStoreLocationResolverStores {
   readonly authorityStore: ZLinkAuthorityStore;
@@ -174,7 +183,7 @@ export class ZLinkStoreLocationResolvers
   private nextActorPlacement = 0n;
 
   constructor(private readonly options: ZLinkStoreLocationResolversOptions) {
-    this.liveRows = new ZLinkLiveRowFilter(options.leaseTracker);
+    this.liveRows = new ZLinkLiveRowFilter();
     this.monotonicNowMs = options.monotonicNowMs ?? (() => performance.now());
     this.authoritySpotResolver = new ZLinkAuthoritySpotRouteResolver(
       options.stores.authorityStore,
@@ -193,7 +202,7 @@ export class ZLinkStoreLocationResolvers
     const rows = await this.options.stores.peerStore.listPeers(filter, signal);
     return await this.liveRows.filter(
       rows,
-      (row) => row.ownerId,
+      (row, signal) => this.options.leaseTracker.isOwnerLive(row.ownerId, signal),
       signal,
       (row) =>
         isKnownZLinkLocationAutoConnectType(row.autoConnectType) &&
@@ -218,8 +227,8 @@ export class ZLinkStoreLocationResolvers
   > {
     for (const meshName of meshNames) {
       const descriptors = await this.liveRows.filter(
-        (await this.options.stores.locationStore.listMeshNodes(meshName, undefined, signal)).items,
-        (descriptor) => descriptor.ownerId,
+        await listAllMeshNodeDescriptors(this.options.stores.locationStore, meshName, signal),
+        (descriptor, signal) => this.options.leaseTracker.isOwnerTokenLive(descriptor, signal),
         signal
       );
       const descriptor = descriptors.find(
@@ -252,12 +261,14 @@ export class ZLinkStoreLocationResolvers
     signal?: AbortSignal,
     excludedCandidateRids: ReadonlySet<string> = new Set()
   ): Promise<RoutingId | undefined> {
-    const descriptors = (
-      await this.options.stores.locationStore.listMeshNodes(meshName, undefined, signal)
-    ).items;
+    const descriptors = await listAllMeshNodeDescriptors(
+      this.options.stores.locationStore,
+      meshName,
+      signal
+    );
     const liveDescriptors = await this.liveRows.filter(
       descriptors,
-      (descriptor) => descriptor.ownerId,
+      (descriptor, signal) => this.options.leaseTracker.isOwnerTokenLive(descriptor, signal),
       signal
     );
     const candidates = liveDescriptors.filter((descriptor) => {
@@ -294,7 +305,7 @@ export class ZLinkStoreLocationResolvers
   ): Promise<ZLinkRouteLocation | undefined> {
     const row = await this.liveRows.resolve(
       await this.options.stores.routeStore.resolveRoute(key, signal),
-      (candidate) => candidate.ownerId,
+      (candidate, signal) => this.options.leaseTracker.isOwnerLive(candidate.ownerId, signal),
       signal
     );
     if (row === undefined) {
@@ -400,10 +411,23 @@ export class ZLinkStoreLocationResolvers
 
   async resolveDirectActorRoute(
     actorId: string,
-    signal?: AbortSignal
+    signal?: AbortSignal,
+    deadlineMs?: number
   ): Promise<ZLinkDirectActorRouteResolution> {
     const cached = await this.lane.run(() => this.getCachedCore(this.directActorRoutes, actorId));
     if (cached !== undefined) return { kind: 'ready', route: cached };
+    return await waitActorReply(
+      this.resolveUncachedDirectActorRoute(actorId, signal),
+      actorId,
+      deadlineMs,
+      signal
+    );
+  }
+
+  private async resolveUncachedDirectActorRoute(
+    actorId: string,
+    signal?: AbortSignal
+  ): Promise<ZLinkDirectActorRouteResolution> {
     const current = await this.options.stores.authorityStore.readAuthority(
       encodeAuthorityKey('actor', actorId),
       signal
@@ -429,12 +453,14 @@ export class ZLinkStoreLocationResolvers
         authorityGeneration: current.authorityOwnerGeneration,
         remainingLeaseMs
       };
-      emitActorOwnerLeaseObservation({
-        actorId,
-        authorityGeneration: unavailable.authorityGeneration,
-        remainingLeaseMs,
-        decision: 'owner_unavailable'
-      });
+      if (isRelocationDebugEnabled()) {
+        emitActorOwnerLeaseObservation({
+          actorId,
+          authorityGeneration: unavailable.authorityGeneration,
+          remainingLeaseMs,
+          decision: 'owner_unavailable'
+        });
+      }
       return unavailable;
     }
     if (
@@ -492,7 +518,8 @@ export class ZLinkStoreLocationResolvers
   ): Promise<ZLinkResolvedActorRoute | undefined> {
     return await this.lane.run(() => {
       if (route === undefined) return undefined;
-      const maxAgeMs = this.options.routeCacheMaxAgeMs ?? 15000;
+      const maxAgeMs =
+        this.options.routeCacheMaxAgeMs ?? zlinkDefaultLocationOptions.routeCacheMaxAgeMs;
       if (maxAgeMs > 0) {
         this.directActorRoutes.set(actorId, {
           row: route,
@@ -532,7 +559,7 @@ export class ZLinkStoreLocationResolvers
     if (cached !== undefined) return cached;
     const row = await this.liveRows.resolve(
       await this.options.stores.spotStore.resolveSpot(key, signal),
-      (candidate) => candidate.ownerId,
+      (candidate, signal) => this.options.leaseTracker.isOwnerTokenLive(candidate, signal),
       signal
     );
     if (row === undefined) {
@@ -576,8 +603,8 @@ export class ZLinkStoreLocationResolvers
     signal?: AbortSignal
   ): Promise<ZLinkFrameworkRuntimeState | undefined> {
     const descriptor = (
-      await this.options.stores.locationStore.listMeshNodes(meshName, undefined, signal)
-    ).items.find(
+      await listAllMeshNodeDescriptors(this.options.stores.locationStore, meshName, signal)
+    ).find(
       (candidate) =>
         routingIdsEqual(candidate.rid, nodeRid) &&
         (expectedLifecycleGeneration === undefined ||
@@ -608,7 +635,7 @@ export class ZLinkStoreLocationResolvers
     }
     const row = await this.liveRows.resolve(
       stored,
-      (candidate) => candidate.ownerId,
+      (candidate, signal) => this.options.leaseTracker.isOwnerTokenLive(candidate, signal),
       signal,
       (candidate) =>
         candidate.actorRef.objectGeneration > 0n &&
@@ -729,7 +756,8 @@ export class ZLinkStoreLocationResolvers
     ownerLeaseGeneration: bigint,
     signal?: AbortSignal
   ): Promise<boolean> {
-    const maxAgeMs = this.options.routeCacheMaxAgeMs ?? 15000;
+    const maxAgeMs =
+      this.options.routeCacheMaxAgeMs ?? zlinkDefaultLocationOptions.routeCacheMaxAgeMs;
     const remainingLeaseMs = await this.options.leaseTracker.remainingOwnerTokenLeaseMs(
       { ownerId, leaseGeneration: ownerLeaseGeneration },
       signal
@@ -884,6 +912,7 @@ export class ZLinkLocationSpotRouteResolver implements ZLinkSpotRouteResolver {
         targetNodeRid: entrySpot.nodeRid,
         spotId: entrySpot.spotId,
         spotKind: ZLinkSpotKind.Entry,
+        targetSpotGeneration: entrySpot.targetNodeGeneration,
         targetNodeGeneration: entrySpot.targetNodeGeneration,
         targetOwnerId: entrySpot.targetOwnerId,
         ownerLeaseGeneration: entrySpot.ownerLeaseGeneration,
@@ -912,7 +941,7 @@ export class ZLinkAuthoritySpotRouteResolver implements ZLinkSpotRouteResolver {
     private readonly routerChannelIdForMesh: (meshName: string) => string,
     private readonly fallback?: ZLinkSpotRouteResolver,
     private readonly leaseTracker?: ZLinkOwnerLeaseTracker,
-    private readonly routeCacheMaxAgeMs = 15000,
+    private readonly routeCacheMaxAgeMs = zlinkDefaultLocationOptions.routeCacheMaxAgeMs,
     private readonly monotonicNowMs: () => number = () => performance.now(),
     private readonly targetNodeStateResolver?: (
       meshName: string,
@@ -930,9 +959,10 @@ export class ZLinkAuthoritySpotRouteResolver implements ZLinkSpotRouteResolver {
 
       const current = await this.store.readAuthority(encodeAuthorityKey('user_spot', key), signal);
       if (current.kind === 'snapshot' && current.allocation.state === 'active') {
-        const decoded = decodeServiceReadySpotAuthority(
-          serviceRelocationAuthorityApplicationPayload(current.payload)
-        );
+        const authorityPayload = serviceRelocationAuthorityApplicationPayload(current.payload);
+        const ready = decodeServiceReadySpotAuthority(authorityPayload);
+        const closing = decodeServiceClosingSpotAuthority(authorityPayload);
+        const decoded = ready ?? closing;
         if (
           decoded !== undefined &&
           decoded.spotId === String(spotId) &&
@@ -975,15 +1005,17 @@ export class ZLinkAuthoritySpotRouteResolver implements ZLinkSpotRouteResolver {
               signal
             )
           };
-          await this.lane.run(() =>
-            this.cacheResolutionCore(
-              key,
-              prepared.epoch,
-              target,
-              remainingLeaseMs,
-              current.storeVersion.value
-            )
-          );
+          if (ready !== undefined) {
+            await this.lane.run(() =>
+              this.cacheResolutionCore(
+                key,
+                prepared.epoch,
+                target,
+                remainingLeaseMs,
+                current.storeVersion.value
+              )
+            );
+          }
           return target;
         }
       }