lorenz 0.1.5 → 0.1.7

package/node_modules/@lorenz/runtime/dist/index.js

@@ -28,6 +28,7 @@ const POISON_WORKER_ERROR_PREFIXES = [
     // `hook failed with status N` (no `workspace ` prefix) and stays healthy.
     "workspace hook failed with status ",
 ];
+const CLAIM_OWNER_HEARTBEAT_INTERVAL_MS = 10_000;
 export function classifyWorkerOutcome(error) {
     const message = error instanceof Error ? error.message : String(error);
     return POISON_WORKER_ERROR_PREFIXES.some((prefix) => message.startsWith(prefix))
@@ -64,7 +65,21 @@ function mergePollOptions(existing, requested) {
         waitForRuns: existingIntent.waitForRuns || requestedIntent.waitForRuns,
     });
 }
+class ClaimStoreRuntimeError extends Error {
+    reason;
+    original;
+    constructor(reason, original) {
+        super(errorMessage(original));
+        this.reason = reason;
+        this.original = original;
+    }
+}
+function throwRuntimeError(error) {
+    throw error;
+}
 class ActiveRunHandle {
+    issueId;
+    slotIndex;
     key;
     runId;
     activeRuns;
@@ -75,7 +90,10 @@ class ActiveRunHandle {
      * generic `agent_run_aborted` (which would otherwise classify as healthy).
      */
     reason = null;
-    constructor(key, runId, activeRuns) {
+    abandonClaimOnSettlement = false;
+    constructor(issueId, slotIndex, key, runId, activeRuns) {
+        this.issueId = issueId;
+        this.slotIndex = slotIndex;
         this.key = key;
         this.runId = runId;
         this.activeRuns = activeRuns;
@@ -89,9 +107,11 @@ class ActiveRunHandle {
     abort() {
         this.controller.abort();
     }
-    finishExternally(reason = null) {
+    finishExternally(reason = null, options = {}) {
         if (reason)
             this.reason = reason;
+        if (options.abandonClaimOnSettlement)
+            this.abandonClaimOnSettlement = true;
         this.abort();
         this.release();
     }
@@ -126,6 +146,8 @@ export class LorenzRuntime {
     activePollOptions = null;
     pendingPollOptions = null;
     workerPoolDrained = false;
+    claimOwnerHeartbeatTimer = null;
+    pendingStoppedClaimSettlements = 0;
     /**
      * Live tracker push subscription (see {@link RuntimeTrackerClient.watch}), opened once in the
      * recurring `start()` loop and closed on `stop()`. `undefined` for pull-only trackers and the
@@ -135,11 +157,10 @@ export class LorenzRuntime {
     changeStream;
     changeStreamOpening = false;
     /**
-     * The reload-surviving coordinator singleton. Built ONCE here: either the
+     * The reload-surviving coordinator singleton. Built here from either the
      * pre-built `input.coordinator` (preferred), or a null-endpoint passthrough
-     * wrapping a bare `input.workerPool` (the low-churn path that keeps every existing
-     * workerPool-injecting site byte-identical at the runtime boundary). `undefined`
-     * when neither is supplied (the static/local path, byte-identical to today).
+     * wrapping a bare `input.workerPool`, which keeps the default runtime boundary unchanged.
+     * `undefined` when neither is supplied, which uses the static/local path.
      */
     coordinator;
     constructor(input) {
@@ -157,7 +178,7 @@ export class LorenzRuntime {
         const coordinator = this.coordinator;
         this.orchestrator =
             input.orchestrator ??
-                new Orchestrator(input.workflow.settings, this.clock, undefined, 
+                new Orchestrator(input.workflow.settings, this.clock, input.claimStore, 
                 // The coordinator IS the orchestrator's capacity authority (it satisfies
                 // the CapacityProbe shape directly). It is installed for the orchestrator's
                 // lifetime whenever it exists, but a reload can disable the underlying pool
@@ -208,6 +229,7 @@ export class LorenzRuntime {
             blocked: orchestration.blocked.map((entry) => ({ ...entry })),
             usageTotals: orchestration.usageTotals,
             rateLimits: orchestration.rateLimits,
+            claimStore: orchestration.claimStore,
             logFile: this.workflow.settings.logging.logFile,
         });
     }
@@ -246,9 +268,14 @@ export class LorenzRuntime {
         void this.closeChangeStream();
         // finishExternally (abort + release) mirrors the other abort sites and clears
         // isActive, so the resulting agent_run_aborted rejection is treated as a clean
-        // shutdown in runClaim rather than recorded as a failed run.
-        for (const handle of [...this.activeRuns.values()])
-            handle.finishExternally();
+        // shutdown in runClaim rather than recorded as a failed run. Durable claims are
+        // abandoned only after the runner settles; until then a shared store must still
+        // show the slot as owned.
+        for (const handle of [...this.activeRuns.values()]) {
+            this.markStoppedClaimSettlementPending(handle);
+            handle.finishExternally(null, { abandonClaimOnSettlement: true });
+        }
+        this.stopClaimOwnerHeartbeatIfIdle();
         this.retryScheduler.stop();
         this.emit();
     }
@@ -390,16 +417,16 @@ export class LorenzRuntime {
         this.appStatus = this.inFlight.size > 0 ? "running" : "polling";
         this.lastPollAt = this.clock.now().toISOString();
         this.lastError = null;
-        this.emit();
         const dispatched = [];
         try {
+            this.emit();
             await this.reloadWorkflowIfConfigured();
             this.validateDispatch(this.workflow.settings);
             await this.cleanupTerminalWorkspacesOnce();
-            this.reconcileStalledRuns();
+            await this.reconcileStalledRuns();
             await this.reconcileTrackedIssues();
             const issues = await this.client.fetchCandidateIssues();
-            const eligibleIssues = this.orchestrator.eligibleIssues(issues);
+            const eligibleIssues = await this.orchestrator.eligibleIssuesAsync(issues);
             if (!options.dryRun)
                 this.syncRetryTimersForIssues(issues);
             this.candidates = issues.length;
@@ -416,7 +443,7 @@ export class LorenzRuntime {
                 await Promise.allSettled(dispatched);
             }
             this.pollStatus = "idle";
-            this.appStatus = this.inFlight.size > 0 ? "running" : "idle";
+            this.updateAppStatusFromInFlight();
             this.nextPollAt = new Date(this.clock.now().getTime() + this.workflow.settings.polling.intervalMs).toISOString();
         }
         catch (error) {
@@ -436,12 +463,11 @@ export class LorenzRuntime {
             this.addEvent("dispatch_skipped", `${issue.identifier} missing_before_dispatch`);
             return [];
         }
-        const claim = this.orchestrator.claim(refreshed);
+        const claim = await this.orchestrator.claimAsync(refreshed);
         if (!claim) {
             this.addEvent("dispatch_skipped", `${refreshed.identifier} stale_before_dispatch`);
             return [];
         }
-        this.syncRetryTimer(refreshed.id);
         const slotIndex = claim.kind === "running" ? claim.entry.slotIndex : claim.reservation.slotIndex;
         const key = slotKey(refreshed.id, slotIndex);
         const runId = `run-${this.nextRunNumber}`;
@@ -449,26 +475,44 @@ export class LorenzRuntime {
         // The handle is registered for the WHOLE run lifecycle - including the reserved
         // path's acquire window - so stop()/reconcile abort an in-acquire run (the
         // signal reaches the pool's FIFO waiter) exactly as they abort a running one.
-        const handle = new ActiveRunHandle(key, runId, this.activeRuns);
+        const handle = new ActiveRunHandle(refreshed.id, slotIndex, key, runId, this.activeRuns);
         this.activeRuns.set(key, handle);
-        // On the static/local path the run starts immediately. On the pool-governed
-        // path run_reserving marks dispatch intent and run_started moves AFTER
-        // bindReservation (inside runReservedClaim): a capacity-refused dispatch
-        // never emits a phantom run_started.
-        if (claim.kind === "running") {
-            this.addEvent("run_started", `${refreshed.identifier} slot=${slotIndex}`);
+        try {
+            this.syncRetryTimer(refreshed.id);
+            await this.startClaimOwnerHeartbeat();
+            // On the static/local path the run starts immediately. On the pool-governed
+            // path run_reserving marks dispatch intent and run_started moves AFTER
+            // bindReservation (inside runReservedClaim): a capacity-refused dispatch
+            // never emits a phantom run_started.
+            if (claim.kind === "running") {
+                this.addEvent("run_started", `${refreshed.identifier} slot=${slotIndex}`);
+            }
+            else {
+                this.addEvent("run_reserving", `${refreshed.identifier} slot=${slotIndex}`);
+            }
+            this.input.onIssueDispatched?.(refreshed);
         }
-        else {
-            this.addEvent("run_reserving", `${refreshed.identifier} slot=${slotIndex}`);
+        catch (error) {
+            try {
+                await this.orchestrator.abandonClaimAsync(refreshed.id, slotIndex);
+            }
+            catch {
+                // Preserve the heartbeat failure; if the backend is unavailable, abandon may fail too.
+            }
+            finally {
+                handle.release();
+                this.stopClaimOwnerHeartbeatIfIdle();
+            }
+            throw error;
         }
-        this.input.onIssueDispatched?.(refreshed);
         const run = claim.kind === "running"
             ? this.runClaim(refreshed, claim.entry.slotIndex, claim.entry.agentKind, runId, claim.entry.workerHost ?? null, handle)
             : this.runReservedClaim(refreshed, claim.reservation, runId, handle);
         this.inFlight.add(run);
         void run.finally(() => {
             this.inFlight.delete(run);
-            this.appStatus = this.inFlight.size > 0 ? "running" : "idle";
+            this.stopClaimOwnerHeartbeatIfIdle();
+            this.updateAppStatusFromInFlight();
             this.emit();
         });
         this.emit();
@@ -484,6 +528,62 @@ export class LorenzRuntime {
             return null;
         }
     }
+    async startClaimOwnerHeartbeat() {
+        await this.orchestrator.heartbeatClaimOwnerAsync();
+        if (this.claimOwnerHeartbeatTimer)
+            return;
+        this.claimOwnerHeartbeatTimer = this.clock.setTimeout(() => {
+            void this.heartbeatClaimOwnerWhileActive();
+        }, CLAIM_OWNER_HEARTBEAT_INTERVAL_MS);
+        this.claimOwnerHeartbeatTimer.unref?.();
+    }
+    async heartbeatClaimOwnerWhileActive() {
+        this.claimOwnerHeartbeatTimer = null;
+        if (this.activeRuns.size === 0 && this.pendingStoppedClaimSettlements === 0)
+            return;
+        try {
+            await this.orchestrator.heartbeatClaimOwnerAsync();
+        }
+        catch (error) {
+            this.handleClaimOwnerHeartbeatFailure(error);
+        }
+        if (this.activeRuns.size === 0 && this.pendingStoppedClaimSettlements === 0)
+            return;
+        this.claimOwnerHeartbeatTimer = this.clock.setTimeout(() => {
+            void this.heartbeatClaimOwnerWhileActive();
+        }, CLAIM_OWNER_HEARTBEAT_INTERVAL_MS);
+        this.claimOwnerHeartbeatTimer.unref?.();
+    }
+    handleClaimOwnerHeartbeatFailure(error) {
+        const message = `claim_owner_heartbeat_failed ${errorMessage(error)}`;
+        this.stopped = true;
+        this.appStatus = "error";
+        this.lastError = message;
+        for (const handle of [...this.activeRuns.values()]) {
+            this.markStoppedClaimSettlementPending(handle);
+            handle.finishExternally(null, { abandonClaimOnSettlement: true });
+        }
+        this.retryScheduler.stop();
+        this.addEvent("poll_error", message);
+    }
+    stopClaimOwnerHeartbeatIfIdle() {
+        if (this.activeRuns.size > 0)
+            return;
+        if (this.pendingStoppedClaimSettlements > 0)
+            return;
+        this.stopClaimOwnerHeartbeat();
+    }
+    stopClaimOwnerHeartbeat() {
+        if (!this.claimOwnerHeartbeatTimer)
+            return;
+        this.clock.clearTimeout(this.claimOwnerHeartbeatTimer);
+        this.claimOwnerHeartbeatTimer = null;
+    }
+    updateAppStatusFromInFlight() {
+        if (this.appStatus === "error")
+            return;
+        this.appStatus = this.inFlight.size > 0 ? "running" : "idle";
+    }
     /**
      * Phase 1 -> negotiation -> phase 2 for a pool-governed (reserved) claim: drive the
      * coordinator's acquire inside this detached per-run promise (a cold provision never blocks
@@ -498,8 +598,9 @@ export class LorenzRuntime {
             // production implies a coordinator. An injected probe without one (test
             // wiring) is treated like an acquire fault: cancel and skip, never strand.
             this.addEvent("dispatch_skipped", `${issue.identifier} worker_pool_acquire_error coordinator_missing`);
-            this.orchestrator.cancelReservation(reservation);
-            handle.release();
+            await this.cancelReservationAfterSkippedAcquire(issue, reservation, handle, {
+                syncRetryTimer: true,
+            });
             return;
         }
         let acquired;
@@ -535,9 +636,9 @@ export class LorenzRuntime {
             // coordinator already settled any just-bound lease healthy before throwing,
             // so there is nothing to settle here.
             this.addEvent("dispatch_skipped", `${issue.identifier} worker_pool_acquire_error ${errorMessage(error)}`);
-            this.orchestrator.cancelReservation(reservation);
-            this.syncRetryTimer(issue.id);
-            handle.release();
+            await this.cancelReservationAfterSkippedAcquire(issue, reservation, handle, {
+                syncRetryTimer: true,
+            });
             return;
         }
         if (acquired.status !== "bound") {
@@ -548,13 +649,51 @@ export class LorenzRuntime {
             // affinity and attempt counter intact; never record history for a run that
             // did not start.
             this.addEvent("dispatch_skipped", `${issue.identifier} worker_host_capacity`);
-            this.orchestrator.cancelReservation(reservation);
-            this.syncRetryTimer(issue.id);
-            handle.release();
+            await this.cancelReservationAfterSkippedAcquire(issue, reservation, handle, {
+                syncRetryTimer: true,
+            });
             return;
         }
         const slot = acquired.slot;
-        const entry = this.orchestrator.bindReservation(reservation, slot.workerHost);
+        let entry;
+        try {
+            entry = await this.orchestrator.bindReservationAsync(reservation, slot.workerHost);
+        }
+        catch (error) {
+            const bindError = errorMessage(error);
+            let releaseErrorMessage = null;
+            try {
+                await slot.release("healthy");
+            }
+            catch (releaseError) {
+                releaseErrorMessage = errorMessage(releaseError);
+            }
+            handle.release();
+            const bindFailureMessage = `claim_bind_failed ${bindError}`;
+            this.markRuntimeError(bindFailureMessage);
+            let retrySyncError = null;
+            if (handle.abandonClaimOnSettlement) {
+                await this.settleStoppedClaim(handle, issue.id, reservation.slotIndex);
+            }
+            else {
+                try {
+                    await this.orchestrator.abandonClaimAsync(issue.id, reservation.slotIndex);
+                    retrySyncError = this.syncRetryTimerSafely(issue.id);
+                }
+                catch {
+                    // Preserve the bind failure event; if the backend is unavailable, abandon may fail too.
+                }
+            }
+            this.stopClaimOwnerHeartbeatIfIdle();
+            this.addEvent("dispatch_skipped", `${issue.identifier} bind_reservation_error ${bindError}`);
+            if (releaseErrorMessage) {
+                this.addEvent("dispatch_skipped", `${issue.identifier} bind_reservation_release_error ${releaseErrorMessage}`);
+            }
+            this.addEvent("poll_error", bindFailureMessage);
+            if (retrySyncError)
+                this.addEvent("poll_error", retrySyncError);
+            return;
+        }
         if (!entry) {
             // The reservation was cancelled/expired during the acquire (cleanup, stop,
             // or the expiry sweep): release the bound worker back to warm inventory (the
@@ -562,16 +701,60 @@ export class LorenzRuntime {
             await slot.release("healthy");
             this.addEvent("dispatch_skipped", `${issue.identifier} reservation_lapsed`);
             handle.release();
+            await this.settleStoppedClaim(handle, issue.id, reservation.slotIndex);
             return;
         }
         this.addEvent("run_started", `${issue.identifier} slot=${reservation.slotIndex}`);
         await this.runClaim(issue, reservation.slotIndex, reservation.agentKind, runId, slot.workerHost, handle, slot);
     }
+    async cancelReservationAfterSkippedAcquire(issue, reservation, handle, options = {}) {
+        let cancelled = false;
+        let cancelError;
+        try {
+            await this.orchestrator.cancelReservationAsync(reservation);
+            cancelled = true;
+        }
+        catch (error) {
+            cancelError = error;
+        }
+        finally {
+            handle.release();
+            await this.settleStoppedClaim(handle, issue.id, reservation.slotIndex);
+            this.stopClaimOwnerHeartbeatIfIdle();
+        }
+        if (cancelled && options.syncRetryTimer) {
+            const retrySyncError = this.syncRetryTimerSafely(issue.id);
+            if (retrySyncError)
+                this.addEvent("poll_error", retrySyncError);
+        }
+        if (!cancelError)
+            return;
+        const message = `claim_cancel_failed ${errorMessage(cancelError)}`;
+        if (this.appStatus !== "error") {
+            this.appStatus = "error";
+            this.lastError = message;
+        }
+        this.addEvent("poll_error", message);
+    }
     async runClaim(issue, slotIndex, agentKind, runId, workerHost, handle, slot = null) {
         const startedAt = this.clock.now().toISOString();
         const effectiveWorkerHost = workerHost;
         let workerOutcome = "healthy";
         const heartbeatSlot = slot;
+        let claimStoreRuntimeError = null;
+        let updateQueue = Promise.resolve();
+        const enqueueUpdate = (update) => {
+            const next = updateQueue.then(async () => {
+                heartbeatSlot?.heartbeat();
+                await this.orchestrator.applyUpdateAsync(issue.id, slotIndex, update);
+                this.addEvent(update.type, agentUpdateRuntimeMessage(issue.identifier, update));
+                this.input.onAgentUpdate?.(issue, update);
+            });
+            updateQueue = next.catch((error) => {
+                claimStoreRuntimeError ??= new ClaimStoreRuntimeError("claim_update_failed", error);
+                handle.abort();
+            });
+        };
         try {
             const result = await this.runner({
                 issue,
@@ -588,26 +771,33 @@ export class LorenzRuntime {
                 // path; force the per-slot suffix whenever co-residence is possible.
                 // Single-tenant (default) keeps the bare path.
                 forceSlotSuffix: (this.workflow.settings.worker.workerPool?.slotsPerMachine ?? 1) > 1,
-                onUpdate: (update) => {
-                    heartbeatSlot?.heartbeat();
-                    this.orchestrator.applyUpdate(issue.id, slotIndex, update);
-                    this.addEvent(update.type, agentUpdateRuntimeMessage(issue.identifier, update));
-                    this.input.onAgentUpdate?.(issue, update);
-                },
+                onUpdate: enqueueUpdate,
                 fetchIssue: async (current) => {
                     const refreshed = await this.client.fetchIssuesByIds([current.id]);
                     return refreshed[0] ?? current;
                 },
                 abortSignal: handle.signal,
             });
+            await updateQueue;
+            if (claimStoreRuntimeError)
+                throwRuntimeError(claimStoreRuntimeError);
             if (!handle.isActive)
                 return;
             const finalIssue = result.finalIssue ?? (await this.fetchIssueOrSelf(issue));
             if (!handle.isActive)
                 return;
-            const entry = this.runningEntry(issue.id, slotIndex);
-            this.orchestrator.finish(issue.id, slotIndex, true, undefined, "continuation");
-            this.syncRetryTimer(issue.id);
+            let finished;
+            try {
+                finished = await this.orchestrator.finishAsync(issue.id, slotIndex, true, undefined, "continuation");
+            }
+            catch (error) {
+                this.recordClaimStoreFailure("claim_finish_failed", error);
+                return;
+            }
+            if (!finished) {
+                this.addEvent("dispatch_skipped", `${issue.identifier} claim_lost_before_finish`);
+                return;
+            }
             this.recordHistory(buildRunHistoryEntry({
                 id: runId,
                 issue,
@@ -616,15 +806,30 @@ export class LorenzRuntime {
                 agentKind,
                 outcome: "success",
                 turnCount: result.turnCount,
-                runningEntry: entry,
+                runningEntry: finished,
                 workspacePath: result.workspace,
                 startedAt,
                 endedAt: this.clock.now().toISOString(),
                 durationMs: durationMs(startedAt, this.clock.now().toISOString()),
             }));
+            const retrySyncError = this.syncRetryTimerSafely(issue.id);
             this.addEvent("run_completed", `${issue.identifier} turns=${result.turnCount}`);
+            if (retrySyncError)
+                this.addEvent("poll_error", retrySyncError);
         }
         catch (error) {
+            await updateQueue;
+            const runtimeError = error instanceof ClaimStoreRuntimeError ? error : claimStoreRuntimeError;
+            if (runtimeError) {
+                this.recordClaimStoreFailure(runtimeError.reason, runtimeError.original);
+                try {
+                    await this.orchestrator.abandonClaimAsync(issue.id, slotIndex);
+                }
+                catch (abandonError) {
+                    this.recordClaimStoreFailure("claim_abandon_failed", abandonError);
+                }
+                return;
+            }
             // Classify the worker outcome BEFORE any early return so a run finished
             // externally (e.g. a stall reconciliation aborted it -> the runner throws
             // `agent_run_aborted`) still poisons the worker: a stall-finished run is
@@ -636,26 +841,36 @@ export class LorenzRuntime {
             // run_failed event the TUI renders as a red error banner on Ctrl+C.
             if (!handle.isActive)
                 return;
-            const entry = this.runningEntry(issue.id, slotIndex);
-            if (!handle.isActive)
+            let finished;
+            try {
+                finished = await this.orchestrator.finishAsync(issue.id, slotIndex, true, errorMessage(error), "failure");
+            }
+            catch (finishError) {
+                this.recordClaimStoreFailure("claim_finish_failed", finishError);
+                return;
+            }
+            if (!finished) {
+                this.addEvent("dispatch_skipped", `${issue.identifier} claim_lost_before_finish`);
                 return;
-            this.orchestrator.finish(issue.id, slotIndex, true, errorMessage(error), "failure");
-            this.syncRetryTimer(issue.id);
+            }
             this.recordHistory(buildRunHistoryEntry({
                 id: runId,
                 issue,
                 slotIndex,
                 agentKind,
                 outcome: "failed",
-                turnCount: entry?.turnCount ?? 0,
-                runningEntry: entry,
+                turnCount: finished.turnCount,
+                runningEntry: finished,
                 startedAt,
                 endedAt: this.clock.now().toISOString(),
                 durationMs: durationMs(startedAt, this.clock.now().toISOString()),
                 error: errorMessage(error),
                 fallbackLastEvent: "turn_failed",
             }));
+            const retrySyncError = this.syncRetryTimerSafely(issue.id);
             this.addEvent("run_failed", `${issue.identifier} ${errorMessage(error)}`);
+            if (retrySyncError)
+                this.addEvent("poll_error", retrySyncError);
         }
         finally {
             handle.release();
@@ -680,6 +895,32 @@ export class LorenzRuntime {
                     await slot.release("healthy");
                 }
             }
+            await this.settleStoppedClaim(handle, issue.id, slotIndex);
+        }
+    }
+    markStoppedClaimSettlementPending(handle) {
+        if (handle.abandonClaimOnSettlement)
+            return;
+        this.pendingStoppedClaimSettlements += 1;
+    }
+    async settleStoppedClaim(handle, issueId, slotIndex) {
+        if (!handle.abandonClaimOnSettlement)
+            return;
+        handle.abandonClaimOnSettlement = false;
+        try {
+            await this.orchestrator.abandonClaimAsync(issueId, slotIndex);
+        }
+        catch (error) {
+            const message = `claim_abandon_failed ${errorMessage(error)}`;
+            if (this.appStatus !== "error") {
+                this.appStatus = "error";
+                this.lastError = message;
+            }
+            this.addEvent("poll_error", message);
+        }
+        finally {
+            this.pendingStoppedClaimSettlements = Math.max(0, this.pendingStoppedClaimSettlements - 1);
+            this.stopClaimOwnerHeartbeatIfIdle();
         }
     }
     async fetchIssueOrSelf(issue) {
@@ -717,12 +958,14 @@ export class LorenzRuntime {
             // applied, so dispatch can never use settings that do not match the live pool.
             //
             // The coordinator (and its pool) is a reload-surviving singleton: diff
-            // prev-vs-next worker-pool settings instead of being reconstructed. When the
-            // reload REMOVES the worker_pool block entirely (next === undefined), reconcile
-            // to a disabled-equivalent of the prior settings so the live pool drains to
-            // zero instead of leaking its (paid) workers unmanaged. A present block (even
-            // one with `enabled: false`) keeps the existing path: reconcile handles the
-            // disable-and-drain itself.
+            // prev-vs-next worker-pool settings instead of being reconstructed. The pool is
+            // the single dispatch path, so a parsed config always carries a worker_pool (an
+            // absent block defaults to the enabled `local` pool). The
+            // `disabledWorkerPoolSettings(prevWorkerPool)` fallback only fires for a settings
+            // object built outside parseConfig that genuinely omits the block; reconcile then
+            // drains the live pool to zero instead of leaking its (paid) workers. A present
+            // block (including an explicit `enabled: false`) reconciles directly: reconcile
+            // handles the disable-and-drain itself.
             if (this.coordinator) {
                 const next = workflow.settings.worker.workerPool ?? disabledWorkerPoolSettings(prevWorkerPool);
                 // Awaited: reconcile is async so the coordinator's injected driverLoader
@@ -747,29 +990,41 @@ export class LorenzRuntime {
         }
     }
     async reconcileTrackedIssues() {
-        const snapshot = this.orchestrator.snapshot();
+        const snapshot = await this.orchestrator.snapshotAsync();
         const tracked = new Map();
         // Reserving (in-acquire) slots are tracked host-less so an issue that goes
         // terminal mid-acquire is still aborted and cleaned up; running/retrying
         // entries below override with their richer metadata when present.
-        for (const entry of snapshot.reserving)
+        for (const entry of snapshot.reserving) {
+            if (!(await this.orchestrator.ownsClaimAsync(entry.issueId, entry.slotIndex)))
+                continue;
             tracked.set(entry.issueId, {
+                kind: "claim",
                 identifier: entry.identifier,
                 workerHost: null,
                 workspacePath: null,
             });
-        for (const entry of snapshot.running)
+        }
+        for (const entry of snapshot.running) {
+            if (!(await this.orchestrator.ownsClaimAsync(entry.issue.id, entry.slotIndex)))
+                continue;
             tracked.set(entry.issue.id, {
+                kind: "claim",
                 identifier: entry.issue.identifier,
                 workerHost: entry.workerHost,
                 workspacePath: entry.workspacePath,
             });
-        for (const entry of snapshot.retrying)
+        }
+        for (const entry of snapshot.retrying) {
+            if (tracked.has(entry.issueId))
+                continue;
             tracked.set(entry.issueId, {
+                kind: "retry",
                 identifier: entry.identifier,
                 workerHost: entry.workerHost,
                 workspacePath: entry.workspacePath,
             });
+        }
         if (tracked.size === 0)
             return;
         let refreshed;
@@ -782,18 +1037,25 @@ export class LorenzRuntime {
         }
         const refreshedIds = new Set(refreshed.map((issue) => issue.id));
         for (const issue of refreshed) {
-            if (issueIsActive(issue, this.workflow.settings) &&
-                routedToThisWorker(issue, this.workflow.settings) &&
-                !issueHasOpenBlockers(issue, this.workflow.settings)) {
-                this.orchestrator.refreshRunningIssue(issue);
+            const meta = tracked.get(issue.id);
+            const active = issueIsActive(issue, this.workflow.settings);
+            const routed = routedToThisWorker(issue, this.workflow.settings);
+            if (active && routed && !issueHasOpenBlockers(issue, this.workflow.settings)) {
+                await this.orchestrator.refreshRunningIssueAsync(issue);
+                continue;
+            }
+            try {
+                await this.orchestrator.cleanupIssueAsync(issue.id);
+            }
+            catch (error) {
+                this.recordClaimStoreFailure("claim_cleanup_failed", error);
                 continue;
             }
             this.abortIssueRuns(issue.id);
-            this.orchestrator.cleanupIssue(issue.id);
             this.clearRetryTimer(issue.id);
             const reason = reconciliationStopReason(issue, this.workflow.settings);
             if (isTerminalState(issue.state, this.workflow.settings.tracker.terminalStates)) {
-                await this.removeIssueWorkspaces(this.workflow.settings, issue.identifier || tracked.get(issue.id)?.identifier, tracked.get(issue.id)?.workerHost, issue);
+                await this.removeIssueWorkspaces(this.workflow.settings, issue.identifier || meta?.identifier, meta?.workerHost, issue);
                 this.addEvent("workspace_cleanup", `${issue.identifier} ${reason}`);
             }
             else {
@@ -803,15 +1065,26 @@ export class LorenzRuntime {
         for (const [issueId, meta] of tracked.entries()) {
             if (refreshedIds.has(issueId))
                 continue;
+            try {
+                await this.orchestrator.cleanupIssueAsync(issueId);
+            }
+            catch (error) {
+                this.recordClaimStoreFailure("claim_cleanup_failed", error);
+                continue;
+            }
             this.abortIssueRuns(issueId);
-            this.orchestrator.cleanupIssue(issueId);
             this.clearRetryTimer(issueId);
             this.addEvent("run_reconciled", `${meta.identifier} missing`);
         }
     }
-    reconcileStalledRuns() {
-        for (const snapshotEntry of this.orchestrator.snapshot().running) {
-            const currentEntry = this.runningEntry(snapshotEntry.issue.id, snapshotEntry.slotIndex);
+    async reconcileStalledRuns() {
+        // Let already-settled runner and run-claim continuations finish before acting on a stale snapshot.
+        await Promise.resolve();
+        await Promise.resolve();
+        for (const snapshotEntry of (await this.orchestrator.snapshotAsync()).running) {
+            if (!(await this.orchestrator.ownsClaimAsync(snapshotEntry.issue.id, snapshotEntry.slotIndex)))
+                continue;
+            const currentEntry = await this.runningEntry(snapshotEntry.issue.id, snapshotEntry.slotIndex);
             if (!currentEntry)
                 continue;
             const effective = settingsForIssueState(this.workflow.settings, currentEntry.issue.state);
@@ -829,11 +1102,22 @@ export class LorenzRuntime {
             const activeHandle = this.activeRuns.get(key);
             const runId = activeHandle?.runId ?? `stalled-${currentEntry.issue.id}-${currentEntry.slotIndex}`;
             const error = `agent_stalled after ${timeoutMs}ms`;
-            const entry = this.runningEntry(snapshotEntry.issue.id, snapshotEntry.slotIndex);
+            const entry = await this.runningEntry(snapshotEntry.issue.id, snapshotEntry.slotIndex);
             if (!entry)
                 continue;
-            this.orchestrator.finish(entry.issue.id, entry.slotIndex, true, error, "failure");
-            this.syncRetryTimer(entry.issue.id);
+            let finished;
+            try {
+                finished = await this.orchestrator.finishAsync(entry.issue.id, entry.slotIndex, true, error, "failure");
+            }
+            catch (finishError) {
+                this.recordClaimStoreFailure("claim_finish_failed", finishError);
+                continue;
+            }
+            if (!finished) {
+                activeHandle?.finishExternally();
+                this.addEvent("dispatch_skipped", `${entry.identifier} claim_lost_before_finish`);
+                continue;
+            }
             activeHandle?.finishExternally("stalled");
             const endedAt = this.clock.now().toISOString();
             this.recordHistory(buildRunHistoryEntry({
@@ -851,13 +1135,14 @@ export class LorenzRuntime {
                 error,
                 fallbackLastEvent: "agent_stalled",
             }));
+            const retrySyncError = this.syncRetryTimerSafely(entry.issue.id);
             this.addEvent("run_stalled", `${entry.identifier} ${error}`);
+            if (retrySyncError)
+                this.addEvent("poll_error", retrySyncError);
         }
     }
-    runningEntry(issueId, slotIndex) {
-        return this.orchestrator
-            .snapshot()
-            .running.find((entry) => entry.issue.id === issueId && entry.slotIndex === slotIndex);
+    async runningEntry(issueId, slotIndex) {
+        return (await this.orchestrator.snapshotAsync()).running.find((entry) => entry.issue.id === issueId && entry.slotIndex === slotIndex);
     }
     // Cleanup is driven by what is actually on disk: list existing per-issue workspace
     // directories and look up just those issues, instead of enumerating every terminal
@@ -905,23 +1190,27 @@ export class LorenzRuntime {
             return;
         }
         this.retryScheduler.sync(runtimeRetryEntry(retry), (scheduled) => {
-            const current = this.orchestrator
-                .snapshot()
-                .retrying.find((entry) => entry.issueId === scheduled.issueId);
-            if (!current ||
-                current.attempt !== scheduled.attempt ||
-                current.dueAtIso !== scheduled.dueAtIso) {
-                return;
+            try {
+                const current = this.orchestrator
+                    .snapshot()
+                    .retrying.find((entry) => entry.issueId === scheduled.issueId);
+                if (!current ||
+                    current.attempt !== scheduled.attempt ||
+                    current.dueAtIso !== scheduled.dueAtIso) {
+                    return;
+                }
+                this.addEvent("retry_timer_due", `${scheduled.issueIdentifier} attempt=${scheduled.attempt}`);
+                if (this.pollInProgress) {
+                    this.queuePendingPoll({}, true);
+                    return;
+                }
+                this.pollOnce().catch((error) => {
+                    this.recordRetryTimerError(error);
+                });
             }
-            this.addEvent("retry_timer_due", `${scheduled.issueIdentifier} attempt=${scheduled.attempt}`);
-            if (this.pollInProgress) {
-                this.queuePendingPoll({}, true);
-                return;
+            catch (error) {
+                this.recordRetryTimerError(error);
             }
-            this.pollOnce().catch((error) => {
-                this.lastError = errorMessage(error);
-                this.addEvent("retry_timer_error", this.lastError);
-            });
         });
     }
     clearRetryTimer(issueId) {
@@ -939,6 +1228,33 @@ export class LorenzRuntime {
     recordHistory(entry) {
         this.projection.recordRunHistory(entry);
     }
+    recordRetryTimerError(error) {
+        const message = errorMessage(error);
+        this.markRuntimeError(message);
+        this.addEvent("retry_timer_error", message);
+    }
+    recordClaimStoreFailure(reason, error) {
+        const message = `${reason} ${errorMessage(error)}`;
+        this.markRuntimeError(message);
+        this.addEvent("poll_error", message);
+    }
+    syncRetryTimerSafely(issueId) {
+        try {
+            this.syncRetryTimer(issueId);
+            return null;
+        }
+        catch (error) {
+            const message = `retry_timer_sync_failed ${errorMessage(error)}`;
+            this.markRuntimeError(message);
+            return message;
+        }
+    }
+    markRuntimeError(message) {
+        if (this.appStatus !== "error") {
+            this.appStatus = "error";
+            this.lastError = message;
+        }
+    }
     addEvent(type, message) {
         const event = { type, message, at: this.clock.now().toISOString() };
         this.projection.recordEvent(event);
@@ -966,7 +1282,15 @@ export class LorenzRuntime {
             return this.input.appendLogEvent(logFile, event);
     }
     emit() {
-        const snapshot = this.snapshot();
+        let snapshot;
+        try {
+            snapshot = this.snapshot();
+        }
+        catch (error) {
+            if (this.appStatus === "error")
+                return;
+            throw error;
+        }
         for (const listener of this.listeners)
             listener(snapshot);
     }
@@ -1109,13 +1433,13 @@ function disabledWorkerPoolSettings(prev) {
 /**
  * Wraps a bare {@link WorkerPool} in a null-endpoint passthrough {@link DispatchCoordinator} so the
  * runtime drives every run through the uniform coordinator surface while a bare-pool injection
- * stays byte-identical at the runtime boundary. STEP 1's null manager mints nothing
- * (`perRunEndpoint=false`, every `RunSlot.mcpEndpoint=null`), so this is a 1:1 passthrough over
+ * stays byte-identical at the runtime boundary. The null manager mints nothing
+ * (`perRunClaimEnforcement=false`, every `RunSlot.mcpEndpoint=null`), so this is a passthrough over
  * the pool: `acquireRunSlot` delegates to `pool.acquire`, settle delegates straight to the
  * `WorkerLease`, and `reconcile`/`drain`/`governs`/`canAcquire` forward verbatim. Returns
  * `undefined` when no pool is supplied (the static/local path).
  *
- * `settings` only needs to satisfy the coordinator's constructor; in STEP 1 the coordinator does
+ * `settings` only needs to satisfy the coordinator's constructor; this wrapper does
  * not read it past construction (the pool owns live settings), so the live `worker.workerPool`
  * settings are passed when present and a disabled placeholder otherwise.
  */