@intx/hub-sessions 0.3.0 → 0.4.0

package/dist/sidecar-allocation/reconciler.js

@@ -2,9 +2,20 @@ import { type } from "arktype";
 import { sha256 } from "@intx/crypto";
 import { getLogger } from "@intx/log";
 import { hexEncode } from "@intx/types";
+import { SidecarIdentityValidationError } from "../ws/sidecar-handler.js";
 import { SessionLaunchError } from "../session-service.js";
+import { DEFAULT_SIDECAR_ALLOCATION_CONCURRENCY } from "../reconciliation-scheduler.js";
 import { DestroySidecarResult, EnsureSidecarResult, } from "./contracts.js";
+import { DEFAULT_SIDECAR_OPERATION_TIMEOUT_MS, runSidecarOperation, SidecarOperationTimeoutError, } from "./operation.js";
 const logger = getLogger(["hub", "sidecar-allocation"]);
+class ReconciliationLeaseLostError extends Error {
+    constructor(allocationId, cause) {
+        super(`Reconciliation lease lost for allocation ${allocationId}`, {
+            cause,
+        });
+        this.name = "ReconciliationLeaseLostError";
+    }
+}
 const DEFAULT_LEASE_DURATION_MS = 60_000;
 const DEFAULT_CONNECT_TIMEOUT_MS = 120_000;
 const MAX_RETRY_BACKOFF_ATTEMPT = 5;
@@ -28,12 +39,38 @@ function parseDestroyResult(value) {
     }
     return result;
 }
-export function createSidecarAllocationReconciler({ allocationStore, plugins, router, hubWebSocketUrl, onReady, enableAutomaticReplacementRecovery = false, leaseDurationMs = DEFAULT_LEASE_DURATION_MS, connectTimeoutMs = DEFAULT_CONNECT_TIMEOUT_MS, retryDelayMs = defaultRetryDelay, now = () => new Date(), createSidecarId = () => `sc_${randomHex(16)}`, createToken = () => `intx_sc_${randomHex(32)}`, createLeaseId = () => `lease_${randomHex(16)}`, }) {
+export function createSidecarAllocationReconciler({ allocationStore, plugins, router, hubWebSocketUrl, onInitializationRecovery, onReady, enableAutomaticReplacementRecovery = false, leaseDurationMs = DEFAULT_LEASE_DURATION_MS, connectTimeoutMs = DEFAULT_CONNECT_TIMEOUT_MS, operationTimeoutMs = DEFAULT_SIDECAR_OPERATION_TIMEOUT_MS, maxConcurrentClaims = DEFAULT_SIDECAR_ALLOCATION_CONCURRENCY, retryDelayMs = defaultRetryDelay, now = () => new Date(), createSidecarId = () => `sc_${randomHex(16)}`, createToken = () => `intx_sc_${randomHex(32)}`, createLeaseId = () => `lease_${randomHex(16)}`, }) {
     if (leaseDurationMs <= 0)
         throw new Error("leaseDurationMs must be positive");
     if (connectTimeoutMs <= 0) {
         throw new Error("connectTimeoutMs must be positive");
     }
+    if (!Number.isSafeInteger(operationTimeoutMs) || operationTimeoutMs <= 0) {
+        throw new Error("operationTimeoutMs must be a positive integer");
+    }
+    if (!Number.isSafeInteger(maxConcurrentClaims) || maxConcurrentClaims <= 0) {
+        throw new Error("maxConcurrentClaims must be a positive integer");
+    }
+    const activeAllocations = new Map();
+    function trackAllocation(allocation) {
+        const active = activeAllocations.get(allocation.id);
+        if (active === undefined)
+            return;
+        if (active.allocation.generation !== allocation.generation)
+            active.pendingConnect = null;
+        active.allocation = allocation;
+    }
+    async function finishReconciliation(allocationId, apply) {
+        const active = activeAllocations.get(allocationId);
+        if (active === undefined)
+            throw new ReconciliationLeaseLostError(allocationId);
+        const target = { allocationId, generation: active.allocation.generation };
+        await queueReconciliationStep(target, async () => {
+            active.controller.signal.throwIfAborted();
+            await apply(active.pendingConnect?.generation === target.generation &&
+                active.allocation.status === "allocated");
+        });
+    }
     function provisionerFor(allocation) {
         const provisioner = plugins.getProvisioner(allocation.provisionerId);
         if (provisioner === null ||
@@ -47,50 +84,122 @@ export function createSidecarAllocationReconciler({ allocationStore, plugins, ro
     function retryAt(attempt) {
         return new Date(now().getTime() + retryDelayMs(attempt));
     }
-    async function withLeaseHeartbeat(allocation, leaseId, operation) {
-        const interval = setInterval(() => {
-            void allocationStore
-                .extendReconciliationLease(allocation.id, leaseId, leaseDurationMs)
-                .catch((error) => {
-                logger.warn `Failed to extend allocation ${allocation.id} lease: ${error instanceof Error ? error.message : String(error)}`;
-            });
-        }, Math.max(1, Math.floor(leaseDurationMs / 3)));
+    async function withReconciliationLease(allocation, leaseId, operationName, operation, timeoutMs) {
+        trackAllocation(allocation);
+        const active = activeAllocations.get(allocation.id);
+        if (active === undefined)
+            throw new ReconciliationLeaseLostError(allocation.id);
+        const controller = active.controller;
         try {
-            return await operation();
+            return await runSidecarOperation(operationName, timeoutMs, async (signal) => {
+                if (!(await runSidecarOperation("Reconciliation lease validation", operationTimeoutMs, async () => {
+                    try {
+                        return await trackAllocationQuery(allocation.id, () => allocationStore.isReconciliationLeaseCurrent(allocation.id, allocation.generation, leaseId));
+                    }
+                    catch (cause) {
+                        throw new ReconciliationLeaseLostError(allocation.id, cause);
+                    }
+                }, signal))) {
+                    throw new ReconciliationLeaseLostError(allocation.id);
+                }
+                if (performance.now() >= active.leaseDeadline) {
+                    controller.abort(new ReconciliationLeaseLostError(allocation.id));
+                }
+                signal.throwIfAborted();
+                return operation({ signal, leaseId });
+            }, controller.signal);
+        }
+        catch (error) {
+            controller.signal.throwIfAborted();
+            throw error;
+        }
+    }
+    // A timed-out lookup still occupies its database connection. Exclude its
+    // allocation until it settles so retries cannot accumulate duplicate reads.
+    const pendingAllocationQueries = new Map();
+    async function trackAllocationQuery(allocationId, query) {
+        const pending = pendingAllocationQueries.get(allocationId) ?? { count: 0 };
+        pending.count += 1;
+        pendingAllocationQueries.set(allocationId, pending);
+        try {
+            return await query();
         }
         finally {
-            clearInterval(interval);
+            pending.count -= 1;
+            if (pending.count === 0)
+                pendingAllocationQueries.delete(allocationId);
         }
     }
-    async function replaceAfterFailure(allocation, leaseId, code, message) {
-        if (allocation.status === "allocated" &&
-            !enableAutomaticReplacementRecovery) {
-            const releasing = await allocationStore.beginUnrecoverableRelease({
-                allocationId: allocation.id,
-                expectedGeneration: allocation.generation,
-                expectedLeaseId: leaseId,
-                failureCode: code,
-                failureMessage: `Automatic recovery is disabled: ${message}`,
-                now: now(),
-            });
-            if (releasing !== null) {
-                router.fenceAllocation(releasing.id, releasing.generation);
+    async function isSidecarReady(allocation) {
+        const active = activeAllocations.get(allocation.id);
+        if (active === undefined)
+            throw new ReconciliationLeaseLostError(allocation.id);
+        return runSidecarOperation("Sidecar readiness", operationTimeoutMs, () => trackAllocationQuery(allocation.id, () => router.isAllocatedSidecarReady({
+            allocationId: allocation.id,
+            generation: allocation.generation,
+        })), active.controller.signal);
+    }
+    async function replaceAfterFailure(allocation, leaseId, code, message, { onlyIfInitializationIncomplete = false, } = {}) {
+        const initializationCheck = onlyIfInitializationIncomplete
+            ? {
+                onlyIfInitializationIncomplete: true,
+                ...(allocation.initializationLeaseId !== undefined
+                    ? {
+                        expectedInitializationLeaseId: allocation.initializationLeaseId,
+                    }
+                    : {}),
             }
-            return;
-        }
-        const replaced = await allocationStore.beginReplacement({
+            : {};
+        let shouldRetryInitialization = false;
+        await queueReconciliationStep({ allocationId: allocation.id, generation: allocation.generation }, async () => {
+            const updated = allocation.status === "allocated" &&
+                !enableAutomaticReplacementRecovery
+                ? await allocationStore.beginUnrecoverableRelease({
+                    ...initializationCheck,
+                    allocationId: allocation.id,
+                    expectedGeneration: allocation.generation,
+                    expectedLeaseId: leaseId,
+                    failureCode: code,
+                    failureMessage: `Automatic recovery is disabled: ${message}`,
+                    now: now(),
+                })
+                : await allocationStore.beginReplacement({
+                    ...initializationCheck,
+                    allocationId: allocation.id,
+                    expectedStatus: allocation.status === "allocated"
+                        ? "allocated"
+                        : "provisioning",
+                    expectedGeneration: allocation.generation,
+                    expectedLeaseId: leaseId,
+                    nextAttemptAt: retryAt(allocation.ensureAttempts + allocation.destroyAttempts),
+                    failureCode: code,
+                    failureMessage: message,
+                    now: now(),
+                });
+            if (updated !== null) {
+                // A late commit still advances the fence before this queued work
+                // settles and its allocation becomes eligible for another claim.
+                router.fenceAllocation(updated.id, updated.generation);
+            }
+            else {
+                shouldRetryInitialization = onlyIfInitializationIncomplete;
+            }
+        });
+        if (shouldRetryInitialization)
+            await retryInitialization(allocation, leaseId);
+    }
+    async function retryInitialization(allocation, leaseId) {
+        // A publication or unsent rollback may have committed since the claim.
+        // Retry the ordinary callback, including initialization and dispatch requeue.
+        // If ownership was lost instead, this lease-guarded update changes nothing.
+        await finishReconciliation(allocation.id, () => allocationStore.scheduleRetry({
             allocationId: allocation.id,
-            expectedStatus: allocation.status === "allocated" ? "allocated" : "provisioning",
+            expectedStatus: "allocated",
             expectedGeneration: allocation.generation,
             expectedLeaseId: leaseId,
-            nextAttemptAt: retryAt(allocation.ensureAttempts + allocation.destroyAttempts),
-            failureCode: code,
-            failureMessage: message,
+            nextAttemptAt: now(),
             now: now(),
-        });
-        if (replaced !== null) {
-            router.fenceAllocation(replaced.id, replaced.generation);
-        }
+        }));
     }
     async function waitUntilReady(allocation, leaseId, connectionAlreadyReady = false) {
         const target = {
@@ -99,26 +208,49 @@ export function createSidecarAllocationReconciler({ allocationStore, plugins, ro
         };
         const deadline = allocation.connectDeadline;
         const remaining = deadline === undefined ? 0 : deadline.getTime() - now().getTime();
+        // A stalled identity query is not evidence that the worker missed its
+        // connection deadline. Let it retry without releasing the generation.
+        const connectionReady = connectionAlreadyReady || (await isSidecarReady(allocation));
         try {
-            if (!connectionAlreadyReady &&
-                !(await router.isAllocatedSidecarReady(target))) {
-                await withLeaseHeartbeat(allocation, leaseId, () => router.waitForAllocatedSidecar(target, Math.max(0, remaining)));
+            if (!connectionReady) {
+                await withReconciliationLease(allocation, leaseId, "Sidecar connection", () => trackAllocationQuery(allocation.id, () => router.waitForAllocatedSidecar(target, Math.max(0, remaining), (validation) => {
+                    // The waiter can expire before a notification lookup settles.
+                    // Retain its exclusion and capacity independently of the wait.
+                    void trackAllocationQuery(allocation.id, () => validation).catch(() => undefined);
+                })), operationTimeoutMs);
             }
         }
         catch (error) {
+            // Let the router report connection expiry. Our outer deadline can expire
+            // during lease or identity validation without establishing worker loss.
+            // A failed identity lookup is likewise inconclusive: the worker may be
+            // healthy behind it, so retry instead of releasing the generation.
+            if (error instanceof ReconciliationLeaseLostError ||
+                error instanceof SidecarOperationTimeoutError ||
+                error instanceof SidecarIdentityValidationError)
+                throw error;
             await replaceAfterFailure(allocation, leaseId, "sidecar_connect_failed", error instanceof Error ? error.message : String(error));
             return;
         }
+        await queueReconciliationStep(target, async () => {
+            const active = activeAllocations.get(allocation.id);
+            if (active?.allocation.generation !== allocation.generation)
+                throw new ReconciliationLeaseLostError(allocation.id);
+            active.controller.signal.throwIfAborted();
+            active.pendingConnect = null;
+        });
         if (onReady !== undefined) {
             try {
-                await withLeaseHeartbeat(allocation, leaseId, () => onReady(allocation));
+                await withReconciliationLease(allocation, leaseId, "Workflow initialization", (context) => trackAllocationQuery(allocation.id, () => onReady(allocation, context)));
             }
             catch (error) {
+                if (error instanceof ReconciliationLeaseLostError)
+                    throw error;
                 if (error instanceof SessionLaunchError && error.leakedAgent) {
-                    await replaceAfterFailure(allocation, leaseId, "sidecar_initialization_uncertain", error.message);
+                    await replaceAfterFailure(allocation, leaseId, "sidecar_initialization_uncertain", error.message, { onlyIfInitializationIncomplete: true });
                     return;
                 }
-                await allocationStore.scheduleRetry({
+                await finishReconciliation(allocation.id, () => allocationStore.scheduleRetry({
                     allocationId: allocation.id,
                     expectedStatus: "allocated",
                     expectedGeneration: allocation.generation,
@@ -131,16 +263,31 @@ export function createSidecarAllocationReconciler({ allocationStore, plugins, ro
                         message: error instanceof Error ? error.message : String(error),
                     },
                     now: now(),
-                });
+                }));
                 return;
             }
         }
-        await allocationStore.markConnectionReady({
-            allocationId: allocation.id,
-            generation: allocation.generation,
-            expectedLeaseId: leaseId,
-            now: now(),
-        });
+        // A connect that lands during initialization schedules an immediate
+        // follow-up even on success: the new socket may be a restarted worker
+        // with an empty inventory (takeover suppresses the disconnect event), in
+        // which case the follow-up redeploys and restores it. When the worker is
+        // unchanged the follow-up is a no-op: deployReadyAllocation returns early
+        // once the workflow is active and its key is recorded.
+        await finishReconciliation(allocation.id, (pendingConnect) => pendingConnect
+            ? allocationStore.scheduleRetry({
+                allocationId: allocation.id,
+                expectedStatus: "allocated",
+                expectedGeneration: allocation.generation,
+                expectedLeaseId: leaseId,
+                nextAttemptAt: now(),
+                now: now(),
+            })
+            : allocationStore.markConnectionReady({
+                allocationId: allocation.id,
+                generation: allocation.generation,
+                expectedLeaseId: leaseId,
+                now: now(),
+            }));
     }
     async function acceptEnsure(allocation, leaseId, provisioner, token) {
         if (allocation.sidecarId === undefined) {
@@ -149,7 +296,8 @@ export function createSidecarAllocationReconciler({ allocationStore, plugins, ro
         const sidecarId = allocation.sidecarId;
         let result;
         try {
-            result = parseEnsureResult(await withLeaseHeartbeat(allocation, leaseId, () => provisioner.ensure({
+            result = parseEnsureResult(await withReconciliationLease(allocation, leaseId, "Sidecar ensure", ({ signal }) => provisioner.ensure({
+                signal,
                 allocationId: allocation.id,
                 generation: allocation.generation,
                 tenantId: allocation.tenantId,
@@ -157,15 +305,17 @@ export function createSidecarAllocationReconciler({ allocationStore, plugins, ro
                 sidecarId,
                 token,
                 hubWebSocketUrl,
-            })));
+            }), operationTimeoutMs));
         }
         catch (error) {
+            if (error instanceof ReconciliationLeaseLostError)
+                throw error;
             await replaceAfterFailure(allocation, leaseId, "ensure_failed", error instanceof Error ? error.message : String(error));
             return;
         }
         if (result.kind === "rejected") {
             if (!result.retryable) {
-                await allocationStore.failWithoutInfrastructure({
+                const failed = await allocationStore.failWithoutInfrastructure({
                     allocationId: allocation.id,
                     expectedStatus: "provisioning",
                     expectedGeneration: allocation.generation,
@@ -174,6 +324,12 @@ export function createSidecarAllocationReconciler({ allocationStore, plugins, ro
                     expectedLeaseId: leaseId,
                     now: now(),
                 });
+                if (failed !== null) {
+                    router.retireAllocation({
+                        allocationId: failed.id,
+                        generation: failed.generation,
+                    });
+                }
                 return;
             }
             await replaceAfterFailure(allocation, leaseId, result.code, result.message);
@@ -189,11 +345,8 @@ export function createSidecarAllocationReconciler({ allocationStore, plugins, ro
             now: now(),
         });
         if (allocated !== null) {
-            const target = {
-                allocationId: allocated.id,
-                generation: allocated.generation,
-            };
-            if (await router.isAllocatedSidecarReady(target)) {
+            trackAllocation(allocated);
+            if (await isSidecarReady(allocated)) {
                 await waitUntilReady(allocated, leaseId, true);
                 return;
             }
@@ -201,10 +354,19 @@ export function createSidecarAllocationReconciler({ allocationStore, plugins, ro
             // transitions. Do not hold the single reconciliation loop for the full
             // connection timeout: park this lease at its persisted deadline and let
             // sidecar.allocated.connected wake it immediately when the worker arrives.
-            await allocationStore.parkReconciliation(allocated.id, leaseId, {
-                kind: "await-connection",
-                fallbackNextAttemptAt: retryAt(MAX_RETRY_BACKOFF_ATTEMPT),
-            });
+            await finishReconciliation(allocated.id, (pendingConnect) => pendingConnect
+                ? allocationStore.scheduleRetry({
+                    allocationId: allocated.id,
+                    expectedStatus: "allocated",
+                    expectedGeneration: allocated.generation,
+                    expectedLeaseId: leaseId,
+                    nextAttemptAt: now(),
+                    now: now(),
+                })
+                : allocationStore.parkReconciliation(allocated.id, leaseId, {
+                    kind: "await-connection",
+                    fallbackNextAttemptAt: retryAt(MAX_RETRY_BACKOFF_ATTEMPT),
+                }));
         }
     }
     async function bindAndEnsure(allocation, leaseId, provisioner, replacement) {
@@ -232,6 +394,7 @@ export function createSidecarAllocationReconciler({ allocationStore, plugins, ro
             });
         if (bound === null)
             return;
+        trackAllocation(bound);
         router.fenceAllocation(bound.id, bound.generation);
         await acceptEnsure(bound, leaseId, provisioner, token);
     }
@@ -240,47 +403,77 @@ export function createSidecarAllocationReconciler({ allocationStore, plugins, ro
             allocation.status !== "releasing") {
             throw new Error(`Cannot retry destroy while allocation ${allocation.id} is ${allocation.status}`);
         }
-        await allocationStore.scheduleRetry({
+        const status = allocation.status;
+        await finishReconciliation(allocation.id, () => allocationStore.scheduleRetry({
             allocationId: allocation.id,
-            expectedStatus: allocation.status,
+            expectedStatus: status,
             expectedGeneration: allocation.generation,
             nextAttemptAt: retryAt(allocation.destroyAttempts),
             expectedLeaseId: leaseId,
             attempt: "destroy",
             now: now(),
-        });
+        }));
     }
     async function destroyCurrent(allocation, leaseId, provisioner) {
         if (allocation.sidecarId === undefined)
             return true;
         const sidecarId = allocation.sidecarId;
+        let result;
         try {
-            const result = parseDestroyResult(await withLeaseHeartbeat(allocation, leaseId, () => provisioner.destroy({
+            result = parseDestroyResult(await withReconciliationLease(allocation, leaseId, "Sidecar destroy", ({ signal }) => provisioner.destroy({
+                signal,
                 allocationId: allocation.id,
                 generation: allocation.generation,
                 sidecarId,
                 ...(allocation.externalRef !== undefined
                     ? { externalRef: allocation.externalRef }
                     : {}),
-            })));
-            if (result.kind === "destroyed")
-                return true;
+            }), operationTimeoutMs));
         }
         catch (error) {
+            if (error instanceof ReconciliationLeaseLostError)
+                throw error;
             logger.warn `Destroy failed for allocation ${allocation.id}: ${error instanceof Error ? error.message : String(error)}`;
+            await retryDestroy(allocation, leaseId);
+            return false;
+        }
+        if (result.kind === "destroyed")
+            return true;
+        if (!result.retryable) {
+            const failed = await allocationStore.markDestroyFailed({
+                allocationId: allocation.id,
+                expectedGeneration: allocation.generation,
+                expectedLeaseId: leaseId,
+                code: result.code,
+                message: result.message,
+                now: now(),
+            });
+            if (failed !== null) {
+                router.retireAllocation({
+                    allocationId: failed.id,
+                    generation: failed.generation,
+                });
+            }
+            return false;
         }
         await retryDestroy(allocation, leaseId);
         return false;
     }
     async function reconcile(allocation, leaseId) {
         router.fenceAllocation(allocation.id, allocation.generation);
-        if (allocation.status === "released" || allocation.status === "failed") {
+        if (allocation.status === "released" ||
+            allocation.status === "failed" ||
+            allocation.status === "destroy_failed") {
             return;
         }
+        if (allocation.status === "allocated" &&
+            onInitializationRecovery !== undefined) {
+            await withReconciliationLease(allocation, leaseId, "Sender deployment recovery", (context) => trackAllocationQuery(allocation.id, () => onInitializationRecovery(allocation, context)), operationTimeoutMs);
+        }
         const provisioner = provisionerFor(allocation);
         if (provisioner === null) {
             if (allocation.status === "pending") {
-                await allocationStore.failWithoutInfrastructure({
+                const failed = await allocationStore.failWithoutInfrastructure({
                     allocationId: allocation.id,
                     expectedStatus: "pending",
                     expectedGeneration: allocation.generation,
@@ -289,16 +482,23 @@ export function createSidecarAllocationReconciler({ allocationStore, plugins, ro
                     expectedLeaseId: leaseId,
                     now: now(),
                 });
+                if (failed !== null) {
+                    router.retireAllocation({
+                        allocationId: failed.id,
+                        generation: failed.generation,
+                    });
+                }
             }
             else {
-                await allocationStore.scheduleRetry({
+                const status = allocation.status;
+                await finishReconciliation(allocation.id, () => allocationStore.scheduleRetry({
                     allocationId: allocation.id,
-                    expectedStatus: allocation.status,
+                    expectedStatus: status,
                     expectedGeneration: allocation.generation,
                     nextAttemptAt: retryAt(allocation.ensureAttempts + allocation.destroyAttempts),
                     expectedLeaseId: leaseId,
                     now: now(),
-                });
+                }));
             }
             return;
         }
@@ -312,6 +512,10 @@ export function createSidecarAllocationReconciler({ allocationStore, plugins, ro
                 await replaceAfterFailure(allocation, leaseId, "ensure_outcome_unknown", "Hub restarted before sidecar provisioning acceptance was recorded");
                 return;
             case "allocated":
+                if (allocation.initializationLeaseId !== undefined) {
+                    await replaceAfterFailure(allocation, leaseId, "sidecar_initialization_uncertain", "A previous initialization attempt did not record completion", { onlyIfInitializationIncomplete: true });
+                    return;
+                }
                 await waitUntilReady(allocation, leaseId);
                 return;
             case "replacing":
@@ -319,16 +523,23 @@ export function createSidecarAllocationReconciler({ allocationStore, plugins, ro
                     return;
                 await bindAndEnsure(allocation, leaseId, provisioner, true);
                 return;
-            case "releasing":
+            case "releasing": {
                 if (!(await destroyCurrent(allocation, leaseId, provisioner)))
                     return;
-                await allocationStore.markReleased({
+                const released = await allocationStore.markReleased({
                     allocationId: allocation.id,
                     generation: allocation.generation,
                     expectedLeaseId: leaseId,
                     now: now(),
                 });
+                if (released !== null) {
+                    router.retireAllocation({
+                        allocationId: released.id,
+                        generation: released.generation,
+                    });
+                }
                 return;
+            }
             default: {
                 const exhaustive = allocation.status;
                 throw new Error(`Allocation ${allocation.id} has unhandled status ${String(exhaustive)}`);
@@ -355,16 +566,72 @@ export function createSidecarAllocationReconciler({ allocationStore, plugins, ro
             }
         }
     }
-    async function handleDisconnect(target) {
-        await allocationStore.markConnectionLost({
-            allocationId: target.allocationId,
-            generation: target.generation,
-            connectDeadline: new Date(now().getTime() + connectTimeoutMs),
-            now: now(),
+    const connectionEvents = new Map();
+    function queueConnectionEvent(target, apply) {
+        const previous = connectionEvents.get(target.allocationId) ?? Promise.resolve();
+        const pending = previous.catch(() => undefined).then(apply);
+        connectionEvents.set(target.allocationId, pending);
+        const settled = () => {
+            if (connectionEvents.get(target.allocationId) === pending)
+                connectionEvents.delete(target.allocationId);
+        };
+        void pending.then(settled, settled);
+        return pending;
+    }
+    async function queueReconciliationStep(target, apply) {
+        const active = activeAllocations.get(target.allocationId);
+        if (active?.allocation.generation !== target.generation)
+            throw new ReconciliationLeaseLostError(target.allocationId);
+        try {
+            await runSidecarOperation("Allocation connection events", operationTimeoutMs, (signal) => queueConnectionEvent(target, async () => {
+                signal.throwIfAborted();
+                await apply();
+            }), active.controller.signal);
+        }
+        catch (error) {
+            if (error instanceof SidecarOperationTimeoutError) {
+                // Stop this lease without queuing another write behind the same stalled
+                // event. Keep the actual operation queued until it settles, preserving
+                // ordering with reconnects and excluding this allocation from new claims.
+                const cancelled = new ReconciliationLeaseLostError(target.allocationId, error);
+                active.controller.abort(cancelled);
+                throw cancelled;
+            }
+            throw error;
+        }
+    }
+    function noteDisconnect(target, leaseInvalidated = false) {
+        const active = activeAllocations.get(target.allocationId);
+        if (active?.allocation.generation !== target.generation)
+            return;
+        active.pendingConnect = null;
+        if (leaseInvalidated || active.allocation.status === "allocated") {
+            active.controller.abort(new ReconciliationLeaseLostError(target.allocationId));
+        }
+    }
+    function noteConnect(target) {
+        const active = activeAllocations.get(target.allocationId);
+        if (active?.allocation.generation === target.generation)
+            active.pendingConnect = target;
+    }
+    function handleDisconnect(target) {
+        noteDisconnect(target);
+        return queueConnectionEvent(target, async () => {
+            const disconnected = await allocationStore.markConnectionLost({
+                allocationId: target.allocationId,
+                generation: target.generation,
+                connectDeadline: new Date(now().getTime() + connectTimeoutMs),
+                now: now(),
+            });
+            noteDisconnect(target, disconnected !== null);
         });
     }
-    async function handleConnected(target) {
-        await allocationStore.wakeReconciliation(target.allocationId, target.generation);
+    function handleConnected(target) {
+        noteConnect(target);
+        return queueConnectionEvent(target, async () => {
+            await allocationStore.wakeReconciliation(target.allocationId, target.generation);
+            noteConnect(target);
+        });
     }
     async function repairUnscheduledConnections() {
         for (const allocation of await allocationStore.listActive()) {
@@ -378,8 +645,17 @@ export function createSidecarAllocationReconciler({ allocationStore, plugins, ro
                 allocationId: allocation.id,
                 generation: allocation.generation,
             };
-            if (await router.isAllocatedSidecarReady(target))
-                continue;
+            try {
+                if (await router.isAllocatedSidecarReady(target))
+                    continue;
+            }
+            catch (error) {
+                // Unknown readiness is not absence. Leave the allocation for the next
+                // repair sweep instead of scheduling a reconnect the worker may hold.
+                if (error instanceof SidecarIdentityValidationError)
+                    continue;
+                throw error;
+            }
             try {
                 await allocationStore.scheduleReconnectIfUnscheduled({
                     ...target,
@@ -393,23 +669,141 @@ export function createSidecarAllocationReconciler({ allocationStore, plugins, ro
             }
         }
     }
+    let admittedClaims = 0;
     async function reconcileNext() {
-        const leaseId = createLeaseId();
-        const allocation = await allocationStore.claimNextReconcilable({
-            leaseId,
-            leaseDurationMs,
-        });
-        if (allocation === null)
+        const retainedAllocations = new Set([
+            ...connectionEvents.keys(),
+            ...pendingAllocationQueries.keys(),
+        ]);
+        // An active claim already owns capacity for its allocation's pending work.
+        for (const allocationId of activeAllocations.keys())
+            retainedAllocations.delete(allocationId);
+        if (admittedClaims + retainedAllocations.size >= maxConcurrentClaims)
             return false;
+        admittedClaims += 1;
+        let claim;
         try {
+            const leaseId = createLeaseId();
+            const claimStartedAt = performance.now();
+            const allocation = await runSidecarOperation("Sidecar allocation claim", operationTimeoutMs, () => {
+                claim = allocationStore.claimNextReconcilable({
+                    leaseId,
+                    leaseDurationMs,
+                    excludedAllocationIds: [
+                        ...new Set([
+                            ...activeAllocations.keys(),
+                            ...connectionEvents.keys(),
+                            ...pendingAllocationQueries.keys(),
+                        ]),
+                    ],
+                });
+                return claim;
+            });
+            if (allocation === null)
+                return false;
+            return await reconcileClaim(allocation, leaseId, claimStartedAt);
+        }
+        finally {
+            const release = () => {
+                admittedClaims -= 1;
+            };
+            // Keep the reservation through the claim-to-reconciliation handoff. A
+            // timed-out claim still owns capacity until its database query settles.
+            if (claim === undefined)
+                release();
+            else
+                void claim.then(release, release);
+        }
+    }
+    async function reconcileClaim(allocation, leaseId, claimStartedAt) {
+        // Local work may have started since this claim's exclusion snapshot. Let
+        // its lease expire without adding another database write behind that work.
+        if (connectionEvents.has(allocation.id) ||
+            activeAllocations.has(allocation.id) ||
+            pendingAllocationQueries.has(allocation.id))
+            return true;
+        const active = {
+            allocation,
+            controller: new AbortController(),
+            leaseDeadline: claimStartedAt + leaseDurationMs,
+            pendingConnect: null,
+        };
+        activeAllocations.set(allocation.id, active);
+        let finished = false;
+        let renewing = false;
+        const renew = async () => {
+            if (finished || renewing || active.controller.signal.aborted)
+                return;
+            renewing = true;
+            const startedAt = performance.now();
+            try {
+                const renewed = await trackAllocationQuery(allocation.id, () => allocationStore.extendReconciliationLease(allocation.id, leaseId, leaseDurationMs));
+                if (finished || active.controller.signal.aborted)
+                    return;
+                if (!renewed || performance.now() >= active.leaseDeadline) {
+                    active.controller.abort(new ReconciliationLeaseLostError(allocation.id));
+                }
+                else {
+                    // The database grants the lease during the request. Counting from its
+                    // start avoids extending ownership by the response's transit time.
+                    active.leaseDeadline = startedAt + leaseDurationMs;
+                }
+            }
+            catch (error) {
+                if (!finished && !active.controller.signal.aborted) {
+                    active.controller.abort(new ReconciliationLeaseLostError(allocation.id, error));
+                }
+            }
+            finally {
+                renewing = false;
+            }
+        };
+        // One heartbeat covers the whole claim, including database transitions
+        // between provider calls. Short stages must not keep postponing renewal.
+        const heartbeat = setInterval(() => {
+            void renew();
+        }, Math.max(1, Math.floor(leaseDurationMs / 3)));
+        let expiryTimer;
+        const checkLeaseExpiry = () => {
+            if (active.controller.signal.aborted)
+                return;
+            const remaining = active.leaseDeadline - performance.now();
+            if (remaining <= 0) {
+                active.controller.abort(new ReconciliationLeaseLostError(allocation.id));
+            }
+            else {
+                expiryTimer = setTimeout(checkLeaseExpiry, Math.ceil(remaining));
+            }
+        };
+        checkLeaseExpiry();
+        try {
+            active.controller.signal.throwIfAborted();
             await reconcile(allocation, leaseId);
         }
         catch (error) {
+            if (error instanceof ReconciliationLeaseLostError) {
+                const cause = error.cause;
+                if (cause === undefined) {
+                    logger.info `Allocation ${allocation.id} reconciliation stopped: lease ${leaseId} is no longer current`;
+                }
+                else {
+                    logger.warn `Allocation ${allocation.id} reconciliation stopped because lease ${leaseId} could not be confirmed: ${cause instanceof Error ? cause.message : String(cause)}`;
+                }
+                // The durable schedule survives the claim. Stop renewing and let the
+                // lease expire; handling a lease failure must not require another write.
+                return true;
+            }
             logger.error `Allocation ${allocation.id} reconciliation failed: ${error instanceof Error ? error.message : String(error)}`;
-            await allocationStore.parkReconciliation(allocation.id, leaseId, {
+            await finishReconciliation(allocation.id, () => allocationStore.parkReconciliation(allocation.id, leaseId, {
                 kind: "retry-after-error",
                 notBefore: retryAt(MAX_RETRY_BACKOFF_ATTEMPT),
-            });
+            }));
+        }
+        finally {
+            finished = true;
+            clearInterval(heartbeat);
+            clearTimeout(expiryTimer);
+            activeAllocations.delete(allocation.id);
         }
         return true;
     }