@zlink-systems/framework 0.26.0 → 0.27.0

package/dist/runtime/locations/runtime.js

@@ -1,6 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ZLinkLocationRuntime = exports.ZLinkOwnerLeaseClaimError = exports.ZLinkOwnerCleanupError = void 0;
+const location_store_pages_1 = require("./location-store-pages");
+const runtime_metrics_1 = require("../diagnostics/runtime-metrics");
+const Stores_1 = require("../../contracts/Locations/Stores");
 const Options_1 = require("../../contracts/Locations/Options");
 const EndpointNotation_1 = require("../../contracts/Configuration/EndpointNotation");
 const node_crypto_1 = require("node:crypto");
@@ -10,6 +13,7 @@ const authority_key_codec_1 = require("./authority-key-codec");
 const canonical_codec_1 = require("./canonical-codec");
 const key_codec_1 = require("./key-codec");
 const lease_tracker_1 = require("./lease-tracker");
+const abort_1 = require("../abort");
 class ZLinkOwnerCleanupError extends Error {
     constructor(cause) {
         super(`Owner cleanup failed: ${errorMessage(cause)}`, { cause });
@@ -35,6 +39,7 @@ class ZLinkLocationRuntime {
     clearTimer;
     monotonicNowMs;
     liveRows;
+    leaseTracker;
     ownershipLostHandlers = new Set();
     ownerLeaseRenewedHandlers = new Set();
     ownerLeaseRenewalFailedHandlers = new Set();
@@ -74,13 +79,14 @@ class ZLinkLocationRuntime {
             runtimeOptions.setTimer ?? ((callback, delayMs) => setTimeout(callback, delayMs));
         this.clearTimer =
             runtimeOptions.clearTimer ?? ((handle) => clearTimeout(handle));
-        const leaseTracker = runtimeOptions.leaseTracker ??
-            new lease_tracker_1.ZLinkOwnerLeaseTracker({
-                store: this.stores.ownerLeaseStore,
-                options: this.options,
-                monotonicNowMs: this.monotonicNowMs
-            });
-        this.liveRows = new lease_tracker_1.ZLinkLiveRowFilter(leaseTracker);
+        this.leaseTracker =
+            runtimeOptions.leaseTracker ??
+                new lease_tracker_1.ZLinkOwnerLeaseTracker({
+                    store: this.stores.ownerLeaseStore,
+                    options: this.options,
+                    monotonicNowMs: this.monotonicNowMs
+                });
+        this.liveRows = new lease_tracker_1.ZLinkLiveRowFilter();
     }
     get isStarted() {
         return this.started;
@@ -145,7 +151,7 @@ class ZLinkLocationRuntime {
             let cursor;
             let expired = false;
             do {
-                const page = await this.stores.authorityStore.listAuthorities('', cursor, 1000, signal);
+                const page = await this.stores.authorityStore.listAuthorities('', cursor, Stores_1.ZLINK_PROVIDER_MAX_PAGE_SIZE, signal);
                 if (page.kind === 'scanExpired') {
                     expired = true;
                     break;
@@ -175,15 +181,17 @@ class ZLinkLocationRuntime {
     removeOwnerLeaseRenewalFailedHandler(handler) {
         this.ownerLeaseRenewalFailedHandlers.delete(handler);
     }
-    async start(nodeRid, signal) {
+    async start(nodeRid, signal, shutdownSignal) {
         if (this.started) {
             return;
         }
         this.started = true;
         this.ownerCleanupComplete = false;
         this.nodeRidValue = nodeRid;
+        const claim = this.renewOwnerLeaseOnce(signal, shutdownSignal);
+        this.heartbeatRenewal = claim;
         try {
-            const claimed = await this.renewOwnerLeaseOnce(signal);
+            const claimed = await claim;
             if (claimed) {
                 this.scheduleHeartbeat();
                 return;
@@ -194,13 +202,10 @@ class ZLinkLocationRuntime {
         catch (error) {
             this.started = false;
             this.ownerCleanupComplete = true;
-            throw error;
+            throw signal?.aborted === true ? signal.reason : error;
         }
     }
     async stop(signal) {
-        if (!this.started && this.ownerCleanupComplete) {
-            return;
-        }
         this.started = false;
         if (this.heartbeatTimer !== undefined) {
             this.clearTimer(this.heartbeatTimer);
@@ -208,10 +213,21 @@ class ZLinkLocationRuntime {
         }
         this.nextLeaseRenewAtMs = undefined;
         this.ownerLeaseDeadlineMs = undefined;
-        await this.heartbeatRenewal;
         await this.cleanupOwner(signal);
     }
     async cleanupOwner(signal) {
+        const pending = this.heartbeatRenewal;
+        try {
+            await withTimeout(() => pending ?? Promise.resolve(), undefined, signal);
+        }
+        catch (error) {
+            if (!(0, abort_1.isAbortError)(error))
+                throw error;
+        }
+        finally {
+            if (this.heartbeatRenewal === pending)
+                this.heartbeatRenewal = undefined;
+        }
         if (this.ownerCleanupComplete) {
             return;
         }
@@ -221,11 +237,13 @@ class ZLinkLocationRuntime {
                 ownerId: this.ownerId,
                 leaseGeneration: 0n
             };
-            await this.stores.locationStore.removeAllByOwner(owner, signal);
-            const currentLease = await this.stores.ownerLeaseStore.readOwnerLease(this.ownerId, signal);
-            if (currentLease.kind === 'found') {
-                await this.stores.ownerLeaseStore.releaseOwnerLease(currentLease.token, signal);
-            }
+            await withTimeout(async (cleanupSignal) => {
+                await this.stores.locationStore.removeAllByOwner(owner, cleanupSignal);
+                const currentLease = await this.stores.ownerLeaseStore.readOwnerLease(this.ownerId, cleanupSignal);
+                if (currentLease.kind === 'found') {
+                    await this.stores.ownerLeaseStore.releaseOwnerLease(currentLease.token, cleanupSignal);
+                }
+            }, undefined, signal);
             this.ownerToken = undefined;
             this.lastOwnerToken = undefined;
             this.ownerCleanupComplete = true;
@@ -235,12 +253,12 @@ class ZLinkLocationRuntime {
             throw new ZLinkOwnerCleanupError(error);
         }
     }
-    async renewOwnerLeaseOnce(signal) {
+    async renewOwnerLeaseOnce(signal, shutdownSignal) {
         if (!this.started) {
             return false;
         }
         if (this.ownerToken === undefined) {
-            return await this.claimFreshOwnerLease(signal);
+            return await this.claimFreshOwnerLease(signal, shutdownSignal);
         }
         const wasOwnerLeaseUsable = this.ownerLeaseUsable;
         try {
@@ -261,7 +279,7 @@ class ZLinkLocationRuntime {
                 // The store has already told us that this owner token cannot be
                 // renewed. Claim immediately so descriptor recovery does not wait for
                 // another heartbeat interval.
-                return await this.claimFreshOwnerLease(signal);
+                return await this.claimFreshOwnerLease(signal, shutdownSignal);
             }
             this.ownerLeaseHealthy = true;
             this.ownerLeaseRenewedAt = result.storeNow;
@@ -283,12 +301,12 @@ class ZLinkLocationRuntime {
             return false;
         }
     }
-    async claimFreshOwnerLease(signal) {
+    async claimFreshOwnerLease(signal, shutdownSignal) {
         const claimStartedAtMs = this.monotonicNowMs();
         const timeoutDeadline = performance.now() + this.options.ownerLeaseRenewTimeoutMs;
         let claim;
         try {
-            claim = await this.claimOwnerLeaseWithConfirmation(timeoutDeadline, signal);
+            claim = await this.claimOwnerLeaseWithConfirmation(timeoutDeadline, signal, shutdownSignal);
         }
         catch (error) {
             if (signal?.aborted === true)
@@ -307,14 +325,7 @@ class ZLinkLocationRuntime {
         }
         try {
             if (!this.started) {
-                // stop() may have completed while the Store claim was in flight. Do
-                // not install a token into a stopped runtime or leave an orphan lease.
-                try {
-                    await withTimeout((releaseSignal) => this.stores.ownerLeaseStore.releaseOwnerLease(claim.token, releaseSignal), remainingTimeoutMs(timeoutDeadline));
-                }
-                catch (error) {
-                    this.recordFailure(errorMessage(error), 'owner_lease_release');
-                }
+                // stop owns confirmation and release under the host shutdown signal.
                 return false;
             }
             this.lastOwnerToken = this.ownerToken;
@@ -340,16 +351,32 @@ class ZLinkLocationRuntime {
             return false;
         }
     }
-    async claimOwnerLeaseWithConfirmation(timeoutDeadline, signal) {
+    async claimOwnerLeaseWithConfirmation(timeoutDeadline, signal, shutdownSignal) {
         signal?.throwIfAborted();
         let claim;
+        let request;
         try {
-            claim = await withTimeout((claimSignal) => this.stores.ownerLeaseStore.claimOwnerLease(this.ownerId, this.options.ownerLeaseTtlMs, claimSignal), remainingTimeoutMs(timeoutDeadline), signal);
+            claim = await withTimeout((claimSignal) => (request = this.stores.ownerLeaseStore.claimOwnerLease(this.ownerId, this.options.ownerLeaseTtlMs, claimSignal)), remainingTimeoutMs(timeoutDeadline), signal);
         }
         catch (error) {
             if (error instanceof OwnerLeaseRenewTimeoutError)
                 throw error;
-            const confirmed = await this.confirmOwnerLease(error, timeoutDeadline, signal);
+            if (signal?.aborted === true && request !== undefined) {
+                const cleanupSignal = shutdownSignal?.();
+                try {
+                    await withTimeout(() => request, cleanupSignal === undefined ? remainingTimeoutMs(timeoutDeadline) : undefined, cleanupSignal);
+                }
+                catch (completionError) {
+                    if (cleanupSignal?.aborted === true ||
+                        completionError instanceof OwnerLeaseRenewTimeoutError) {
+                        throw new ZLinkOwnerCleanupError(completionError);
+                    }
+                    if (!(0, abort_1.isAbortError)(completionError)) {
+                        this.recordFailure(errorMessage(completionError), 'owner_lease_claim');
+                    }
+                }
+            }
+            const confirmed = await this.confirmOwnerLease(error, timeoutDeadline, signal, shutdownSignal);
             if (confirmed !== undefined)
                 return confirmed;
             throw error;
@@ -357,29 +384,34 @@ class ZLinkLocationRuntime {
         if (claim.kind !== 'conflict')
             return claim;
         const conflict = new ZLinkOwnerLeaseClaimError(claim.kind);
-        return (await this.confirmOwnerLease(conflict, timeoutDeadline, signal)) ?? claim;
+        return ((await this.confirmOwnerLease(conflict, timeoutDeadline, signal, shutdownSignal)) ?? claim);
     }
-    async confirmOwnerLease(originalError, timeoutDeadline, signal) {
-        const confirmationTimeoutMs = remainingTimeoutMs(timeoutDeadline);
-        if (confirmationTimeoutMs <= 0)
+    async confirmOwnerLease(originalError, timeoutDeadline, signal, shutdownSignal) {
+        const cancelled = signal?.aborted === true;
+        const cleanupSignal = cancelled ? shutdownSignal?.() : undefined;
+        const confirmationTimeoutMs = cleanupSignal === undefined ? remainingTimeoutMs(timeoutDeadline) : undefined;
+        if (confirmationTimeoutMs !== undefined && confirmationTimeoutMs <= 0)
             throw originalError;
         let confirmed;
         try {
-            confirmed = await withTimeout((readSignal) => this.stores.ownerLeaseStore.readOwnerLease(this.ownerId, readSignal), confirmationTimeoutMs);
+            confirmed = await withTimeout((readSignal) => this.stores.ownerLeaseStore.readOwnerLease(this.ownerId, readSignal), confirmationTimeoutMs, cleanupSignal);
         }
         catch (confirmationError) {
             this.recordFailure(errorMessage(confirmationError), 'owner_lease_claim_confirmation');
-            throw originalError;
+            if (cancelled)
+                throw new ZLinkOwnerCleanupError(confirmationError);
+            throw confirmationError;
         }
         if (confirmed.kind !== 'found' || confirmed.token.ownerId !== this.ownerId) {
             throw originalError;
         }
         if (signal?.aborted === true) {
             try {
-                await withTimeout((releaseSignal) => this.stores.ownerLeaseStore.releaseOwnerLease(confirmed.token, releaseSignal), remainingTimeoutMs(timeoutDeadline));
+                await withTimeout((releaseSignal) => this.stores.ownerLeaseStore.releaseOwnerLease(confirmed.token, releaseSignal), cleanupSignal === undefined ? remainingTimeoutMs(timeoutDeadline) : undefined, cleanupSignal);
             }
             catch (releaseError) {
                 this.recordFailure(errorMessage(releaseError), 'owner_lease_release');
+                throw new ZLinkOwnerCleanupError(releaseError);
             }
             throw originalError;
         }
@@ -562,7 +594,7 @@ class ZLinkLocationRuntime {
     }
     async listPeerLocations(filter, signal) {
         const rows = await this.stores.peerStore.listPeers(filter, signal);
-        const live = await this.filterLive(rows, (row) => row.ownerId, signal);
+        const live = await this.liveRows.filter(rows, (row, signal) => this.leaseTracker.isOwnerLive(row.ownerId, signal), signal);
         // Location Store rows may have been written before this process adopted
         // the endpoint-notation policy, or by another language/version's writer.
         // Normalize on read so every comparison downstream sees the canonical
@@ -570,13 +602,13 @@ class ZLinkLocationRuntime {
         return live.map((row) => ({ ...row, endpoint: (0, EndpointNotation_1.normalizeEndpoint)(row.endpoint) }));
     }
     async listLiveMeshNodes(meshName, signal) {
-        const rows = (await this.stores.locationStore.listMeshNodes(meshName, undefined, signal)).items;
-        const live = await this.filterLive(rows, (row) => row.ownerId, signal);
+        const rows = await (0, location_store_pages_1.listAllMeshNodeDescriptors)(this.stores.locationStore, meshName, signal);
+        const live = await this.liveRows.filter(rows, (row, signal) => this.leaseTracker.isOwnerTokenLive(row, signal), signal);
         return live.map((row) => ({ ...row, endpoint: (0, EndpointNotation_1.normalizeEndpoint)(row.endpoint) }));
     }
     async listMeshNodeDescriptors(meshName, page, signal) {
         const rows = await this.stores.locationStore.listMeshNodes(meshName, this.pageRequest(page), signal);
-        const live = await this.filterLive(rows.items, (row) => row.ownerId, signal);
+        const live = await this.liveRows.filter(rows.items, (row, signal) => this.leaseTracker.isOwnerTokenLive(row, signal), signal);
         return {
             items: live.map((row) => ({ ...row, endpoint: (0, EndpointNotation_1.normalizeEndpoint)(row.endpoint) })),
             continuationToken: rows.continuationToken
@@ -589,7 +621,7 @@ class ZLinkLocationRuntime {
         }
         const rows = await queryStore.listSpots(filter, this.pageRequest(page), signal);
         return {
-            items: await this.filterLive(rows.items, (row) => row.ownerId, signal),
+            items: await this.liveRows.filter(rows.items, (row, signal) => this.leaseTracker.isOwnerTokenLive(row, signal), signal),
             continuationToken: rows.continuationToken
         };
     }
@@ -600,13 +632,13 @@ class ZLinkLocationRuntime {
         }
         const rows = await queryStore.listActors(filter, this.pageRequest(page), signal);
         return {
-            items: await this.filterLive(rows.items, (row) => row.ownerId, signal),
+            items: await this.liveRows.filter(rows.items, (row, signal) => this.leaseTracker.isOwnerTokenLive(row, signal), signal),
             continuationToken: rows.continuationToken
         };
     }
     async listObjectLocations(filter, page = {}, signal) {
         const normalizedPage = this.pageRequest(page);
-        const pageSize = normalizedPage.pageSize ?? this.options.listPageSize;
+        const pageSize = normalizedPage.pageSize;
         const requestedCursor = normalizedPage.continuationToken;
         let cursor = requestedCursor === undefined ? undefined : Authority_1.ZLinkAuthorityScanCursor.from(requestedCursor);
         const items = [];
@@ -624,7 +656,10 @@ class ZLinkLocationRuntime {
                     (filter.stableType !== undefined && allocation.stableType !== filter.stableType) ||
                     (filter.meshName !== undefined && allocation.descriptor.meshName !== filter.meshName))
                     continue;
-                const live = (await this.liveRows.resolve(entry, (candidate) => candidate.snapshot.ownerId, signal)) !== undefined;
+                const live = (await this.liveRows.resolve(entry, (candidate, signal) => this.leaseTracker.isOwnerTokenLive({
+                    ownerId: candidate.snapshot.ownerId,
+                    leaseGeneration: candidate.snapshot.ownerLeaseGeneration
+                }, signal), signal)) !== undefined;
                 items.push(this.objectLocationEntry(identity.globalId, entry.snapshot, live));
                 if (items.length >= pageSize)
                     break;
@@ -646,14 +681,14 @@ class ZLinkLocationRuntime {
         const found = await this.stores.authorityStore.readAuthority((0, authority_key_codec_1.encodeAuthorityKey)('user_spot', String(spotId)), signal);
         if (found.kind === 'missing' || found.allocation.objectKind === 'actor')
             return undefined;
-        const live = (await this.liveRows.resolve(found, (snapshot) => snapshot.ownerId, signal)) !== undefined;
+        const live = (await this.liveRows.resolve(found, (snapshot, signal) => this.leaseTracker.isOwnerTokenLive({ ownerId: snapshot.ownerId, leaseGeneration: snapshot.ownerLeaseGeneration }, signal), signal)) !== undefined;
         return this.objectLocationEntry(String(spotId), found, live);
     }
     async findObjectLocation(kind, globalId, signal) {
         const found = await this.stores.authorityStore.readAuthority((0, authority_key_codec_1.encodeAuthorityKey)(kind, globalId), signal);
         if (found.kind === 'missing' || found.allocation.objectKind !== kind)
             return undefined;
-        const live = (await this.liveRows.resolve(found, (snapshot) => snapshot.ownerId, signal)) !== undefined;
+        const live = (await this.liveRows.resolve(found, (snapshot, signal) => this.leaseTracker.isOwnerTokenLive({ ownerId: snapshot.ownerId, leaseGeneration: snapshot.ownerLeaseGeneration }, signal), signal)) !== undefined;
         return this.objectLocationEntry(globalId, found, live);
     }
     objectLocationEntry(globalId, snapshot, ownerLive) {
@@ -670,15 +705,15 @@ class ZLinkLocationRuntime {
     async listRouteLocations(filter, page, signal) {
         const rows = await this.stores.routeStore.listRoutes(filter, this.pageRequest(page), signal);
         return {
-            items: await this.filterLive(rows.items, (row) => row.ownerId, signal),
+            items: await this.liveRows.filter(rows.items, (row, signal) => this.leaseTracker.isOwnerLive(row.ownerId, signal), signal),
             continuationToken: rows.continuationToken
         };
     }
     async listTopology(filter, page, signal) {
         const entries = [];
         for (const meshName of this.meshNamesOf(filter.meshName)) {
-            for (const descriptor of await this.listAllMeshNodeDescriptors(meshName, signal)) {
-                const live = (await this.filterLive([descriptor], (row) => row.ownerId, signal)).length > 0;
+            for (const descriptor of await (0, location_store_pages_1.listAllMeshNodeDescriptors)(this.stores.locationStore, meshName, signal, this.options.listPageSize)) {
+                const live = await this.leaseTracker.isOwnerTokenLive(descriptor, signal);
                 const entry = {
                     meshName: descriptor.meshName,
                     nodeRid: descriptor.rid,
@@ -702,22 +737,23 @@ class ZLinkLocationRuntime {
         return this.pageInMemory(entries, page);
     }
     pageRequest(page) {
-        return page?.pageSize === undefined ? { ...page, pageSize: this.options.listPageSize } : page;
+        return page?.pageSize === undefined
+            ? { ...page, pageSize: this.options.listPageSize }
+            : page;
     }
     async listServiceSummaries(filter, page, signal) {
         const summaries = [];
         for (const meshName of this.meshNamesOf(filter.meshName)) {
-            const descriptors = await this.listAllMeshNodeDescriptors(meshName, signal);
+            const descriptors = await (0, location_store_pages_1.listAllMeshNodeDescriptors)(this.stores.locationStore, meshName, signal, this.options.listPageSize);
             if (descriptors.length === 0)
                 continue;
-            const live = await this.filterLive(descriptors, (row) => row.ownerId, signal);
-            const liveOwners = new Set(live.map((row) => row.ownerId));
+            const live = await this.liveRows.filter(descriptors, (row, signal) => this.leaseTracker.isOwnerTokenLive(row, signal), signal);
             summaries.push({
                 meshName,
                 totalCount: descriptors.length,
-                readyCount: descriptors.filter((row) => liveOwners.has(row.ownerId)).length,
+                readyCount: live.length,
                 errorCount: 0,
-                stoppedCount: descriptors.filter((row) => !liveOwners.has(row.ownerId)).length,
+                stoppedCount: descriptors.length - live.length,
                 lastUpdatedAt: descriptors.reduce((latest, row) => (row.updatedAt > latest ? row.updatedAt : latest), descriptors[0].updatedAt)
             });
         }
@@ -726,21 +762,11 @@ class ZLinkLocationRuntime {
     meshNamesOf(meshName) {
         return meshName === undefined ? this.meshNames : [meshName];
     }
-    async listAllMeshNodeDescriptors(meshName, signal) {
-        const rows = [];
-        let continuationToken;
-        do {
-            const result = await this.stores.locationStore.listMeshNodes(meshName, { pageSize: this.options.listPageSize, continuationToken }, signal);
-            rows.push(...result.items);
-            continuationToken = result.continuationToken;
-        } while (continuationToken !== undefined);
-        return rows;
-    }
     pageInMemory(entries, page) {
         const normalized = this.pageRequest(page);
         const parsedOffset = Number.parseInt(normalized.continuationToken ?? '0', 10);
         const offset = Number.isFinite(parsedOffset) && parsedOffset >= 0 ? parsedOffset : 0;
-        const pageSize = normalized.pageSize ?? this.options.listPageSize;
+        const pageSize = normalized.pageSize;
         const items = entries.slice(offset, offset + pageSize);
         const nextOffset = offset + items.length;
         return {
@@ -748,9 +774,6 @@ class ZLinkLocationRuntime {
             continuationToken: nextOffset < entries.length ? String(nextOffset) : undefined
         };
     }
-    async filterLive(rows, ownerIdOf, signal) {
-        return await this.liveRows.filter(rows, ownerIdOf, signal);
-    }
     scheduleHeartbeat() {
         if (!this.started) {
             return;
@@ -759,11 +782,11 @@ class ZLinkLocationRuntime {
         const delayMs = Math.max(0, scheduledAt - this.monotonicNowMs());
         this.heartbeatTimer = this.setTimer(() => {
             this.heartbeatTimer = undefined;
-            const lateness = Math.max(0, this.monotonicNowMs() - scheduledAt) / 1000;
+            const lateness = Math.max(0, this.monotonicNowMs() - scheduledAt) / runtime_metrics_1.MILLISECONDS_PER_SECOND;
             for (const scope of this.leaseScopes) {
                 this.metrics?.recordOwnerLeaseRenewLateness(lateness, scope.kind, scope.name);
             }
-            this.nextLeaseRenewAtMs = scheduledAt + this.options.ownerLeaseRenewIntervalMs;
+            this.nextLeaseRenewAtMs = this.monotonicNowMs() + this.options.ownerLeaseRenewIntervalMs;
             if (this.ownerLeaseDeadlineMs !== undefined &&
                 this.monotonicNowMs() >= this.ownerLeaseDeadlineMs) {
                 this.ownerLeaseHealthy = false;
@@ -838,20 +861,24 @@ class ZLinkLocationRuntime {
 }
 exports.ZLinkLocationRuntime = ZLinkLocationRuntime;
 async function withTimeout(action, timeoutMs, signal) {
-    const timeoutController = new AbortController();
+    const timeoutController = timeoutMs === undefined ? undefined : new AbortController();
     const operationSignal = signal === undefined
-        ? timeoutController.signal
-        : AbortSignal.any([signal, timeoutController.signal]);
+        ? timeoutController?.signal
+        : timeoutController === undefined
+            ? signal
+            : AbortSignal.any([signal, timeoutController.signal]);
     let timeout;
     let abort;
-    const deadline = new Promise((_resolve, reject) => {
-        timeout = setTimeout(() => {
-            const error = new OwnerLeaseRenewTimeoutError(timeoutMs);
-            timeoutController.abort(error);
-            reject(error);
-        }, timeoutMs);
-        timeout.unref();
-    });
+    const deadline = timeoutMs === undefined
+        ? undefined
+        : new Promise((_resolve, reject) => {
+            timeout = setTimeout(() => {
+                const error = new OwnerLeaseRenewTimeoutError(timeoutMs);
+                timeoutController.abort(error);
+                reject(error);
+            }, timeoutMs);
+            timeout.unref();
+        });
     const cancellation = signal === undefined
         ? undefined
         : new Promise((_resolve, reject) => {
@@ -860,9 +887,13 @@ async function withTimeout(action, timeoutMs, signal) {
         });
     try {
         signal?.throwIfAborted();
-        return await Promise.race(cancellation === undefined
-            ? [action(operationSignal), deadline]
-            : [action(operationSignal), deadline, cancellation]);
+        const operation = action(operationSignal);
+        if (deadline === undefined) {
+            return cancellation === undefined
+                ? await operation
+                : await Promise.race([operation, cancellation]);
+        }
+        return await Promise.race(cancellation === undefined ? [operation, deadline] : [operation, deadline, cancellation]);
     }
     finally {
         if (timeout !== undefined)

package/dist/runtime/locations/spot-location-claims.d.ts

@@ -1,4 +1,5 @@
 import type { RoutingId } from '../../contracts/Common';
+import type { ZLinkAuthoritySnapshot } from './internal-location-contracts';
 import { ZLinkLocationWriteStatus } from './internal-location-contracts';
 import { ZLinkSpotKind } from '../../contracts/Spots';
 import type { IZLinkLocationLifecycleRuntime, ZLinkOwnershipLostEvent } from './lifecycle-runtime';
@@ -33,4 +34,9 @@ export interface ZLinkTrackedInstanceAuthority {
 }
 export interface ZLinkInstanceClosingAuthority {
     release(): Promise<void>;
+    reincarnate?(initialize: (authority: ZLinkAuthoritySnapshot & {
+        readonly stableType: string;
+    }) => Promise<void>): Promise<ZLinkAuthoritySnapshot & {
+        readonly stableType: string;
+    }>;
 }

package/dist/runtime/locations/spot-location-claims.js

@@ -118,7 +118,48 @@ class ZLinkSpotLocationClaims {
         tracked.storeVersion = result.storeVersion.value;
         onCommitted();
         return {
-            release: async () => undefined
+            release: () => this.release(meshName, spotId),
+            reincarnate: async (initialize) => {
+                const reincarnated = await store.compareExchangeAuthority(key, result.storeVersion, {
+                    kind: 'put',
+                    generationTransition: 'reincarnate',
+                    payload: (0, service_relocation_runtime_1.replaceServiceRelocationAuthorityApplicationPayload)(result.payload, (0, service_authority_payload_codec_1.encodeServiceInstanceAuthorityPayload)({ ...decoded, state: 'coldActivating' }))
+                });
+                if (reincarnated.kind !== 'stored') {
+                    throw (0, framework_errors_internal_1.createInternalFrameworkException)(reincarnated.kind === 'generationExhausted'
+                        ? framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.InvalidOperation
+                        : framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.SpotMoving, `Instance Spot '${String(spotId)}' Closing authority changed before reincarnation.`);
+                }
+                this.trackInstanceAuthority({
+                    ...tracked,
+                    objectGeneration: reincarnated.objectGeneration,
+                    authorityOwnerGeneration: reincarnated.authorityOwnerGeneration,
+                    storeVersion: reincarnated.storeVersion.value
+                });
+                this.invalidateSpotRoute?.(spotId);
+                try {
+                    await initialize({ ...reincarnated, kind: 'snapshot', stableType: tracked.stableType });
+                    const ready = await store.compareExchangeAuthority(key, reincarnated.storeVersion, {
+                        kind: 'put',
+                        generationTransition: 'preserve',
+                        payload: (0, service_relocation_runtime_1.replaceServiceRelocationAuthorityApplicationPayload)(reincarnated.payload, (0, service_authority_payload_codec_1.encodeServiceInstanceAuthorityPayload)({ ...decoded, state: 'ready' }))
+                    });
+                    if (ready.kind !== 'stored') {
+                        throw (0, framework_errors_internal_1.createInternalFrameworkException)(ready.kind === 'generationExhausted'
+                            ? framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.InvalidOperation
+                            : framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.SpotMoving, `Instance Spot '${String(spotId)}' reincarnation authority changed before initialization completed.`);
+                    }
+                    const latest = this.spots.get(canonical);
+                    if (latest?.kind === 'authority' && latest.objectGeneration === ready.objectGeneration) {
+                        latest.storeVersion = ready.storeVersion.value;
+                    }
+                    return { ...ready, kind: 'snapshot', stableType: tracked.stableType };
+                }
+                catch (error) {
+                    await this.release(meshName, spotId, reincarnated.objectGeneration);
+                    throw error;
+                }
+            }
         };
     }
     onOwnershipLost(event) {

package/dist/runtime/messaging/payload-codec.js

@@ -9,6 +9,7 @@ exports.wrapFrameworkPayloadMessage = wrapFrameworkPayloadMessage;
 exports.selectDefaultSerializer = selectDefaultSerializer;
 exports.selectSerializer = selectSerializer;
 exports.selectSerializerWithContentType = selectSerializerWithContentType;
+const stream_wire_1 = require("@zlink-systems/stream-wire");
 const framework_errors_internal_1 = require("../framework-errors-internal");
 const runtime_message_1 = require("../backend/runtime-message");
 const contracts_1 = require("../../contracts");
@@ -20,8 +21,11 @@ const Attributes_1 = require("../../contracts/Handlers/Attributes");
 const packet_name_1 = require("./packet-name");
 const CodecContentType_1 = require("../../contracts/Configuration/CodecContentType");
 const CodecSerializerSelection_1 = require("../../contracts/Configuration/CodecSerializerSelection");
+const MAX_DECLARED_TYPE_SELECTIONS = 1024;
 const noSerializer = Symbol('noSerializer');
-const JSON_CONTENT_TYPE = 'application/json';
+const JSON_CONTENT_TYPE = stream_wire_1.ZlinkStreamContentType.Json;
+const frameworkJsonReceiveCodec = Symbol('frameworkJsonReceiveCodec');
+const defaultReceiveCodecs = new Map([[JSON_CONTENT_TYPE, frameworkJsonReceiveCodec]]);
 // Registration maps are created during host configuration and are immutable
 // for the runtime lifetime. Compile the candidate list and reverse content
 // type lookup once per map so the message path does not allocate arrays or
@@ -36,8 +40,8 @@ function encodeFrameworkPayload(payload, registry, packetName, contractPart = 'p
     if ((0, contracts_1.isZLinkMessage)(payload)) {
         if (payload.isEncoded()) {
             return {
-                message: rememberContentType(toRuntimeMessage(payload.toEncodedPayload()), 'application/octet-stream'),
-                contentType: 'application/octet-stream'
+                message: rememberContentType(toRuntimeMessage(payload.toEncodedPayload()), stream_wire_1.ZlinkStreamContentType.Raw),
+                contentType: stream_wire_1.ZlinkStreamContentType.Raw
             };
         }
         declaredType = (0, ZLinkMessage_1.readZLinkMessageDeclaredType)(payload);
@@ -72,16 +76,11 @@ function decodeFrameworkTypedPayloadMessage(message, registry, type, contentType
     return decodeFrameworkPayload(message, registry, type, contentType, packetName, contractPart);
 }
 function decodeFrameworkPayload(message, registry, type, contentType, packetName, contractPart) {
-    if (!(0, CodecContentType_1.isCanonicalCodecContentType)(contentType)) {
-        throw unsupportedContentType(contentType);
-    }
-    if (contentType !== JSON_CONTENT_TYPE) {
-        const serializer = serializerMapOf(registry)?.get(contentType);
-        if (serializer === undefined)
-            throw unsupportedContentType(contentType);
+    const codec = receiveCodecFor(contentType, registry);
+    if (codec !== frameworkJsonReceiveCodec) {
         if (message.isEmpty())
             return undefined;
-        return serializer.deserialize(encodedPayloadFromOwned(message.data()), (type ?? Object));
+        return codec.deserialize(encodedPayloadFromOwned(message.data()), (type ?? Object));
     }
     if (message.isEmpty())
         return undefined;
@@ -96,16 +95,11 @@ function wrapFrameworkPayloadMessage(message, registry, contentType = JSON_CONTE
     return (0, ZLinkMessage_1.createZLinkMessageFromEncoded)(payload, (type) => decodeFrameworkEncodedPayload(payload, registry, type, contentType, packetName, contractPart));
 }
 function decodeFrameworkEncodedPayload(payload, registry, type, contentType, packetName, contractPart) {
-    if (!(0, CodecContentType_1.isCanonicalCodecContentType)(contentType)) {
-        throw unsupportedContentType(contentType);
-    }
-    if (contentType !== JSON_CONTENT_TYPE) {
-        const serializer = serializerMapOf(registry)?.get(contentType);
-        if (serializer === undefined)
-            throw unsupportedContentType(contentType);
+    const codec = receiveCodecFor(contentType, registry);
+    if (codec !== frameworkJsonReceiveCodec) {
         if (payload.isEmpty())
             return undefined;
-        return serializer.deserialize(payload, (type ?? Object));
+        return codec.deserialize(payload, (type ?? Object));
     }
     if (payload.isEmpty())
         return undefined;
@@ -117,6 +111,18 @@ function decodeFrameworkEncodedPayload(payload, registry, type, contentType, pac
         throw (0, framework_errors_internal_1.createInternalFrameworkException)(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.PayloadDecodeFailed, 'PayloadDecodeFailed: framework payload is not valid JSON.', false, error);
     }
 }
+function receiveCodecFor(contentType, registry) {
+    if (!(0, CodecContentType_1.isCanonicalCodecContentType)(contentType)) {
+        throw unsupportedContentType(contentType);
+    }
+    const registeredCodec = serializerMapOf(registry)?.get(contentType);
+    if (registeredCodec !== undefined)
+        return registeredCodec;
+    const defaultCodec = defaultReceiveCodecs.get(contentType);
+    if (defaultCodec !== undefined)
+        return defaultCodec;
+    throw unsupportedContentType(contentType);
+}
 function schemaForDecode(type, packetName, contractPart) {
     const contract = type === undefined ? undefined : (0, packet_name_1.readFrameworkPacketJsonContract)(type, packetName);
     return (contract ??
@@ -167,7 +173,7 @@ function serializerSelectionPlanOf(registry) {
                 return cached === noSerializer ? undefined : cached;
             }
             const selected = selectSerializerFromEntries(frozenEntries, declaredType);
-            if (cachedBusinessTypeCount < 1024) {
+            if (cachedBusinessTypeCount < MAX_DECLARED_TYPE_SELECTIONS) {
                 outboundByBusinessType.set(declaredType, selected ?? noSerializer);
                 cachedBusinessTypeCount += 1;
             }