@zlink-systems/framework 0.18.1 → 0.20.0

package/dist/runtime/locations/runtime.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ZLinkLocationRuntime = exports.ZLinkOwnerCleanupError = void 0;
+exports.ZLinkLocationRuntime = exports.ZLinkOwnerLeaseClaimError = exports.ZLinkOwnerCleanupError = void 0;
 const Options_1 = require("../../contracts/Locations/Options");
 const EndpointNotation_1 = require("../../contracts/Configuration/EndpointNotation");
 const node_crypto_1 = require("node:crypto");
@@ -17,6 +17,15 @@ class ZLinkOwnerCleanupError extends Error {
     }
 }
 exports.ZLinkOwnerCleanupError = ZLinkOwnerCleanupError;
+class ZLinkOwnerLeaseClaimError extends Error {
+    kind;
+    constructor(kind) {
+        super(`Owner lease claim failed with '${kind}'.`);
+        this.kind = kind;
+        this.name = 'ZLinkOwnerLeaseClaimError';
+    }
+}
+exports.ZLinkOwnerLeaseClaimError = ZLinkOwnerLeaseClaimError;
 class ZLinkLocationRuntime {
     ownerId;
     options;
@@ -46,9 +55,9 @@ class ZLinkLocationRuntime {
     ownerLeaseRenewedAt;
     lastError;
     get ownerLeaseUsable() {
-        return this.ownerLeaseHealthy
-            && (this.ownerLeaseDeadlineMs === undefined
-                || this.monotonicNowMs() < this.ownerLeaseDeadlineMs);
+        return this.ownerToken !== undefined
+            && this.ownerLeaseDeadlineMs !== undefined
+            && this.monotonicNowMs() < this.ownerLeaseDeadlineMs;
     }
     constructor(runtimeOptions) {
         this.stores = runtimeOptions.stores;
@@ -78,6 +87,10 @@ class ZLinkLocationRuntime {
     get currentOwnerToken() {
         return this.ownerToken;
     }
+    closeOwnerLeaseAdmission(error) {
+        this.ownerLeaseDeadlineMs = undefined;
+        this.recordFailure(errorMessage(error), 'owner_lease_recovery');
+    }
     async reclaimOwnerAuthorities(signal) {
         const owner = this.ownerToken;
         if (owner === undefined) {
@@ -166,37 +179,20 @@ class ZLinkLocationRuntime {
         this.started = true;
         this.ownerCleanupComplete = false;
         this.nodeRidValue = nodeRid;
-        const claimStartedAtMs = this.monotonicNowMs();
-        let claim;
         try {
-            claim = await withTimeout((claimSignal) => this.stores.ownerLeaseStore.claimOwnerLease(this.ownerId, this.options.ownerLeaseTtlMs, claimSignal), this.options.ownerLeaseRenewTimeoutMs, signal);
-        }
-        catch (error) {
-            this.ownerLeaseHealthy = false;
-            this.ownerLeaseDeadlineMs = undefined;
-            if (signal?.aborted === true)
-                throw error;
-            this.recordFailure(errorMessage(error), 'owner_lease_claim');
-            // A Location host can still serve transport work while the Store is
-            // unavailable. Keep the runtime in degraded mode so the heartbeat can
-            // claim a lease after recovery; no descriptor or stateful authority is
-            // published until that claim succeeds.
-            this.nextLeaseRenewAtMs = this.monotonicNowMs();
+            const claimed = await this.renewOwnerLeaseOnce(signal);
+            if (claimed) {
+                this.scheduleHeartbeat();
+                return;
+            }
+            this.nextLeaseRenewAtMs = this.monotonicNowMs() + this.options.ownerLeaseRenewIntervalMs;
             this.scheduleHeartbeat();
-            return;
         }
-        const claimCompletedAtMs = this.monotonicNowMs();
-        if (claim.kind !== 'claimed') {
+        catch (error) {
             this.started = false;
             this.ownerCleanupComplete = true;
-            throw new Error(`Owner lease claim failed with '${claim.kind}'.`);
+            throw error;
         }
-        this.ownerToken = claim.token;
-        this.ownerLeaseHealthy = true;
-        this.ownerLeaseRenewedAt = claim.storeNow;
-        this.setOwnerLeaseDeadline(claimStartedAtMs, claimCompletedAtMs, claim.leaseExpiresAt, claim.storeNow);
-        this.nextLeaseRenewAtMs = this.monotonicNowMs() + this.options.ownerLeaseRenewIntervalMs;
-        this.scheduleHeartbeat();
     }
     async stop(signal) {
         if (!this.started && this.ownerCleanupComplete) {
@@ -242,6 +238,7 @@ class ZLinkLocationRuntime {
         if (this.ownerToken === undefined) {
             return await this.claimFreshOwnerLease(signal);
         }
+        const wasOwnerLeaseUsable = this.ownerLeaseUsable;
         try {
             const renewStartedAtMs = this.monotonicNowMs();
             const result = await withTimeout((renewSignal) => this.stores.ownerLeaseStore.renewOwnerLease(this.ownerToken, this.options.ownerLeaseTtlMs, renewSignal), this.options.ownerLeaseRenewTimeoutMs, signal);
@@ -266,8 +263,10 @@ class ZLinkLocationRuntime {
             this.ownerLeaseRenewedAt = result.storeNow;
             this.setOwnerLeaseDeadline(renewStartedAtMs, renewCompletedAtMs, result.leaseExpiresAt, result.storeNow);
             this.lastError = undefined;
-            for (const handler of this.ownerLeaseRenewedHandlers)
-                handler(result);
+            if (!wasOwnerLeaseUsable) {
+                for (const handler of this.ownerLeaseRenewedHandlers)
+                    handler(result);
+            }
             return true;
         }
         catch (error) {
@@ -281,18 +280,37 @@ class ZLinkLocationRuntime {
         }
     }
     async claimFreshOwnerLease(signal) {
+        const claimStartedAtMs = this.monotonicNowMs();
+        const timeoutDeadline = performance.now() + this.options.ownerLeaseRenewTimeoutMs;
+        let claim;
+        try {
+            claim = await this.claimOwnerLeaseWithConfirmation(timeoutDeadline, signal);
+        }
+        catch (error) {
+            if (signal?.aborted === true)
+                throw error;
+            if (error instanceof ZLinkOwnerLeaseClaimError)
+                throw error;
+            this.recordOwnerLeaseRenewFailure();
+            this.recordFailure(errorMessage(error), 'owner_lease_claim');
+            return false;
+        }
+        const claimCompletedAtMs = this.monotonicNowMs();
+        if (claim.kind !== 'claimed') {
+            const error = new ZLinkOwnerLeaseClaimError(claim.kind);
+            this.recordFailure(error.message, 'owner_lease_claim');
+            throw error;
+        }
         try {
-            const claimStartedAtMs = this.monotonicNowMs();
-            const claim = await withTimeout((claimSignal) => this.stores.ownerLeaseStore.claimOwnerLease(this.ownerId, this.options.ownerLeaseTtlMs, claimSignal), this.options.ownerLeaseRenewTimeoutMs, signal);
-            const claimCompletedAtMs = this.monotonicNowMs();
-            if (claim.kind !== 'claimed') {
-                this.recordFailure(`Owner lease claim failed with '${claim.kind}'.`, 'owner_lease_claim');
-                return false;
-            }
             if (!this.started) {
                 // stop() may have completed while the Store claim was in flight. Do
                 // not install a token into a stopped runtime or leave an orphan lease.
-                await this.stores.ownerLeaseStore.releaseOwnerLease(claim.token).catch(() => undefined);
+                try {
+                    await withTimeout(releaseSignal => this.stores.ownerLeaseStore.releaseOwnerLease(claim.token, releaseSignal), remainingTimeoutMs(timeoutDeadline));
+                }
+                catch (error) {
+                    this.recordFailure(errorMessage(error), 'owner_lease_release');
+                }
                 return false;
             }
             this.lastOwnerToken = this.ownerToken;
@@ -314,11 +332,60 @@ class ZLinkLocationRuntime {
         catch (error) {
             if (signal?.aborted === true)
                 throw error;
-            this.recordOwnerLeaseRenewFailure();
             this.recordFailure(errorMessage(error), 'owner_lease_claim');
             return false;
         }
     }
+    async claimOwnerLeaseWithConfirmation(timeoutDeadline, signal) {
+        signal?.throwIfAborted();
+        let claim;
+        try {
+            claim = await withTimeout((claimSignal) => this.stores.ownerLeaseStore.claimOwnerLease(this.ownerId, this.options.ownerLeaseTtlMs, claimSignal), remainingTimeoutMs(timeoutDeadline), signal);
+        }
+        catch (error) {
+            if (error instanceof OwnerLeaseRenewTimeoutError)
+                throw error;
+            const confirmed = await this.confirmOwnerLease(error, timeoutDeadline, signal);
+            if (confirmed !== undefined)
+                return confirmed;
+            throw error;
+        }
+        if (claim.kind !== 'conflict')
+            return claim;
+        const conflict = new ZLinkOwnerLeaseClaimError(claim.kind);
+        return await this.confirmOwnerLease(conflict, timeoutDeadline, signal) ?? claim;
+    }
+    async confirmOwnerLease(originalError, timeoutDeadline, signal) {
+        const confirmationTimeoutMs = remainingTimeoutMs(timeoutDeadline);
+        if (confirmationTimeoutMs <= 0)
+            throw originalError;
+        let confirmed;
+        try {
+            confirmed = await withTimeout(readSignal => this.stores.ownerLeaseStore.readOwnerLease(this.ownerId, readSignal), confirmationTimeoutMs);
+        }
+        catch (confirmationError) {
+            this.recordFailure(errorMessage(confirmationError), 'owner_lease_claim_confirmation');
+            throw originalError;
+        }
+        if (confirmed.kind !== 'found' || confirmed.token.ownerId !== this.ownerId) {
+            throw originalError;
+        }
+        if (signal?.aborted === true) {
+            try {
+                await withTimeout(releaseSignal => this.stores.ownerLeaseStore.releaseOwnerLease(confirmed.token, releaseSignal), remainingTimeoutMs(timeoutDeadline));
+            }
+            catch (releaseError) {
+                this.recordFailure(errorMessage(releaseError), 'owner_lease_release');
+            }
+            throw originalError;
+        }
+        return {
+            kind: 'claimed',
+            token: confirmed.token,
+            leaseExpiresAt: confirmed.leaseExpiresAt,
+            storeNow: confirmed.storeNow
+        };
+    }
     setOwnerLeaseDeadline(requestStartedAtMs, requestCompletedAtMs, leaseExpiresAt, storeNow) {
         const roundTripMs = Math.max(0, requestCompletedAtMs - requestStartedAtMs);
         const remainingTtlMs = Math.max(0, leaseExpiresAt.getTime() - storeNow.getTime());
@@ -771,22 +838,43 @@ async function withTimeout(action, timeoutMs, signal) {
         ? timeoutController.signal
         : AbortSignal.any([signal, timeoutController.signal]);
     let timeout;
+    let abort;
     const deadline = new Promise((_resolve, reject) => {
         timeout = setTimeout(() => {
-            const error = new Error(`Owner lease renewal exceeded ${timeoutMs}ms.`);
+            const error = new OwnerLeaseRenewTimeoutError(timeoutMs);
             timeoutController.abort(error);
             reject(error);
         }, timeoutMs);
         timeout.unref();
     });
+    const cancellation = signal === undefined
+        ? undefined
+        : new Promise((_resolve, reject) => {
+            abort = () => reject(signal.reason);
+            signal.addEventListener('abort', abort, { once: true });
+        });
     try {
-        return await Promise.race([action(operationSignal), deadline]);
+        signal?.throwIfAborted();
+        return await Promise.race(cancellation === undefined
+            ? [action(operationSignal), deadline]
+            : [action(operationSignal), deadline, cancellation]);
     }
     finally {
         if (timeout !== undefined)
             clearTimeout(timeout);
+        if (abort !== undefined)
+            signal?.removeEventListener('abort', abort);
+    }
+}
+class OwnerLeaseRenewTimeoutError extends Error {
+    constructor(timeoutMs) {
+        super(`Owner lease renewal exceeded ${timeoutMs}ms.`);
+        this.name = 'OwnerLeaseRenewTimeoutError';
     }
 }
+function remainingTimeoutMs(timeoutDeadline) {
+    return Math.max(0, timeoutDeadline - performance.now());
+}
 function errorMessage(error) {
     return error instanceof Error ? error.message : String(error);
 }