@agent-relay/factory 0.1.74 → 0.1.76

package/dist/orchestrator/factory.js

@@ -2,7 +2,7 @@ import { AsyncLocalStorage } from 'node:async_hooks';
 import { randomUUID } from 'node:crypto';
 import { readFile } from 'node:fs/promises';
 import { dirname, isAbsolute, resolve } from 'node:path';
-import { DEFAULT_READINESS_RECONCILE_TIMEOUT_MS, FactoryConfigSchema } from '../config/schema.js';
+import { DEFAULT_DISCOVERY_SWEEP_BUDGET_MS, DEFAULT_READINESS_RECONCILE_TIMEOUT_MS, FactoryConfigSchema, resolvedSweepBudgetMs, } from '../config/schema.js';
 import { DEFAULT_RELAYFILE_OPERATION_TIMEOUT_MS, RelayfileOperationTimeoutError, relayfileTimeoutWithPhase, withRelayfileCallDeadline, } from '../mount/relayfile-operation-timeout.js';
 import { linearByStatePath, linearByIdPath, linearByUuidPath } from '../constants/linear.js';
 import { stateResolutionFromIds } from '../linear/state-resolver.js';
@@ -11,8 +11,9 @@ import { factoryGithubIssueCommentDraftName, isFactoryGithubIssueCommentDraftNam
 import { VerificationPipeline } from '../environments/verification-pipeline.js';
 import { factoryWorktreeIssueSlug, factoryWorktreePath } from '../git/agent-worktree.js';
 import { InMemoryStateStore } from '../state/in-memory-state-store.js';
+import { DISCOVERY_SWEEP_TEARDOWN_TIMEOUT_MS, DiscoverySweepBudgetExceededError, startDiscoverySweepBudget, withSweepTeardownDeadline, } from './sweep-budget.js';
 import { dispatchHandedOffToBabysitters, dispatchLifecycleOccupiesSlot, dispatchPhaseOccupiesSlot, } from '../state/dispatch-lifecycle-slot.js';
-import { branchImplementsIssue, containsExplicitIssueReference, containsIssueKey, factoryBranchBelongsToIssue, prBodyDisclaimsClosing, prClosureAuthority } from '../issue-key-match.js';
+import { ISSUE_KEY_PARTS, branchImplementsIssue, containsExplicitIssueReference, containsIssueKey, factoryBranchBelongsToIssue, prBodyDisclaimsClosing, prClosureAuthority } from '../issue-key-match.js';
 import { normalizeLogger, normalizeLogValue, setSafeErrorStack, stringifyLogValue } from '../logging.js';
 import { isInFactoryScope } from '../safety/factory-scope.js';
 import { dispatchRelayflowForChangeEvent } from '../dispatch/relayflow-registry.js';
@@ -557,6 +558,31 @@ export class FactoryLoop {
      * returns: a deadline checked between awaits never regains control to check.
      */
     #relayfileOperationTimeoutMs = DEFAULT_RELAYFILE_OPERATION_TIMEOUT_MS;
+    /**
+     * Aggregate budget for ONE sweep (#372).
+     *
+     * The third bound on this path and the only transport-agnostic one.
+     * `#relayfileOperationTimeoutMs` bounds one relayfile call and cannot see the
+     * retry loop around it; `#readinessReconcileTimeoutMs` bounds the *wait* and
+     * leaves `runOnce()` running for the next cycle to coalesce onto. This is
+     * charged against one timer for the whole pass, so however many calls,
+     * retries and transports the sweep is spread across, it cannot outlive this.
+     *
+     * Held at `min(configured, #readinessReconcileTimeoutMs)`: a budget looser
+     * than the wait would let the wait give up first, which is the behaviour it
+     * exists to remove.
+     */
+    #discoverySweepBudgetMs = DEFAULT_DISCOVERY_SWEEP_BUDGET_MS;
+    /**
+     * The budgets of every sweep currently in flight.
+     *
+     * `stop()` drains the sweep it started (see `#readinessReconcileAbandonedWait`
+     * below), so a wedged sweep holds shutdown open for the whole budget — 90
+     * minutes at the default, and unbounded before this budget existed. A set
+     * rather than one field because a live sweep and a mismatched dry-run one can
+     * be in flight at the same time.
+     */
+    #discoverySweepBudgets = new Set();
     // Set for exactly as long as a sweep is running. `state` is derived from
     // this, so an in-flight pass can no longer masquerade as the last settled one.
     #readinessReconcileInFlightSinceMs;
@@ -772,6 +798,7 @@ export class FactoryLoop {
         // Also read here, not only in `#startLiveSubscription`: a standalone
         // `runOnce()` never starts the live subscription and must still be bounded.
         this.#relayfileOperationTimeoutMs = config.liveSubscription.relayfileOperationTimeoutMs;
+        this.#discoverySweepBudgetMs = resolvedSweepBudgetMs(config.liveSubscription.sweepBudgetMs, config.liveSubscription.reconcileTimeoutMs);
         this.#mount = ports.mount;
         // Resolved role<->state mapping. The CLI injects a name-resolved, per-team
         // resolution via ports; fall back to one built from explicit stateIds plus
@@ -1202,31 +1229,50 @@ export class FactoryLoop {
             clearTimeout(this.#heldAgentDeadlineTimer);
         this.#heldAgentDeadlineTimer = undefined;
         this.#heldAgentDeadlineDueAtMs = undefined;
-        await this.#heldAgentDeadlineSweepInFlight;
-        for (const timer of this.#dispatchLifecycleRetryTimers.values())
-            clearTimeout(timer);
-        this.#dispatchLifecycleRetryTimers.clear();
-        this.#abandonedDispatchReasons.clear();
-        this.#dispatchLifecycleCapacityWaits.clear();
-        this.#dispatchLifecycleOwnershipWaitLogged.clear();
-        if (this.#completionSweepTimer)
-            clearTimeout(this.#completionSweepTimer);
-        this.#completionSweepTimer = undefined;
-        if (this.#readinessReconcileTimer)
-            clearTimeout(this.#readinessReconcileTimer);
-        this.#readinessReconcileTimer = undefined;
-        if (this.#previewSweepTimer)
-            clearTimeout(this.#previewSweepTimer);
-        this.#previewSweepTimer = undefined;
-        await this.#readinessReconcileInFlight;
-        // #301 review: the deadline ends the *wait*, so `#readinessReconcileInFlight`
-        // can settle with its `runOnce()` still live. Shutdown releases dispatch
-        // lifecycle leases and disposes ports below, and `#isPassFatalFailure` only
-        // fences a stopping sweep once something in it throws — so a sweep whose
-        // dependency recovers cleanly would otherwise dispatch through torn-down
-        // state. Draining here restores exactly the pre-deadline shutdown contract:
-        // stop() outlives the sweep it started.
-        await this.#readinessReconcileAbandonedWait;
+        // #372: the drain below is unbounded in the one case that matters — a
+        // wedged sweep — so shutdown inherits the sweep budget, 90 minutes at the
+        // default. Spending it after one teardown window bounds shutdown without
+        // discarding a sweep that was about to finish.
+        //
+        // Armed BEFORE the first shutdown await, not just before the sweep drain
+        // (cubic-dev-ai, #374 review). The grace is a timer, so arming it costs
+        // nothing and starts the clock at the moment shutdown starts; arming it
+        // after `#heldAgentDeadlineSweepInFlight` made the lever's start depend on
+        // an UNRELATED in-flight sweep finishing first, so a slow held-agent pass
+        // simply added its own latency to the wedged discovery sweep's reprieve —
+        // in the limit the lever never arms at all, which is the bound this whole
+        // change exists to provide.
+        const releaseSweepBudgetGrace = this.#cutSweepBudgetsShortForStop();
+        try {
+            await this.#heldAgentDeadlineSweepInFlight;
+            for (const timer of this.#dispatchLifecycleRetryTimers.values())
+                clearTimeout(timer);
+            this.#dispatchLifecycleRetryTimers.clear();
+            this.#abandonedDispatchReasons.clear();
+            this.#dispatchLifecycleCapacityWaits.clear();
+            this.#dispatchLifecycleOwnershipWaitLogged.clear();
+            if (this.#completionSweepTimer)
+                clearTimeout(this.#completionSweepTimer);
+            this.#completionSweepTimer = undefined;
+            if (this.#readinessReconcileTimer)
+                clearTimeout(this.#readinessReconcileTimer);
+            this.#readinessReconcileTimer = undefined;
+            if (this.#previewSweepTimer)
+                clearTimeout(this.#previewSweepTimer);
+            this.#previewSweepTimer = undefined;
+            await this.#readinessReconcileInFlight;
+            // #301 review: the deadline ends the *wait*, so `#readinessReconcileInFlight`
+            // can settle with its `runOnce()` still live. Shutdown releases dispatch
+            // lifecycle leases and disposes ports below, and `#isPassFatalFailure` only
+            // fences a stopping sweep once something in it throws — so a sweep whose
+            // dependency recovers cleanly would otherwise dispatch through torn-down
+            // state. Draining here restores exactly the pre-deadline shutdown contract:
+            // stop() outlives the sweep it started.
+            await this.#readinessReconcileAbandonedWait;
+        }
+        finally {
+            releaseSweepBudgetGrace();
+        }
         await this.#previewSweepInFlight;
         this.#stoppingHeartbeatRefreshActive = await this.#stopLiveHeartbeat('stopping');
         try {
@@ -1613,6 +1659,10 @@ export class FactoryLoop {
         // floor here: a deadline under one interval would kill every pass.
         this.#readinessReconcileTimeoutMs = Math.max(options.reconcileTimeoutMs, options.reconcileIntervalMs);
         this.#relayfileOperationTimeoutMs = options.relayfileOperationTimeoutMs;
+        // Same re-application as the deadline above: `start()` overrides bypass the
+        // schema's cross-field check, and a budget looser than the wait is not a
+        // budget.
+        this.#discoverySweepBudgetMs = resolvedSweepBudgetMs(options.sweepBudgetMs, this.#readinessReconcileTimeoutMs);
         this.#liveConnectStartedAtMs = this.#clock.now();
         this.#liveReplaySkewMarginMs = options.replaySkewMarginMs;
         const highWatermark = await this.#currentEventHighWatermark();
@@ -1776,6 +1826,7 @@ export class FactoryLoop {
             reconcileTimeoutMs: overrides.reconcileTimeoutMs ?? this.#config.liveSubscription.reconcileTimeoutMs,
             relayfileOperationTimeoutMs: overrides.relayfileOperationTimeoutMs
                 ?? this.#config.liveSubscription.relayfileOperationTimeoutMs,
+            sweepBudgetMs: overrides.sweepBudgetMs ?? this.#config.liveSubscription.sweepBudgetMs,
         };
     }
     async #currentEventCursor(limit) {
@@ -2638,12 +2689,131 @@ export class FactoryLoop {
             throw contextualError('Factory dispatch paused because the fleet control plane is unavailable', error);
         }
     }
+    /**
+     * One sweep, under one aggregate budget (#372).
+     *
+     * The budget wraps the fence rather than the fence's caller, and that is the
+     * whole point. #296's deadline lives in `#runOnceWithReadinessDeadline`,
+     * outside `runOnce()`: expiry rejects the wait and leaves the sweep running,
+     * so the next cycle coalesces onto the same wedged promise (`runOnce()`, the
+     * `#runOnceInFlight` branch) and the daemon never recovers. Expiring in HERE
+     * unwinds the body below, which releases the discovery lease on its way out
+     * and lets `runOnce()` clear `#runOnceInFlight` — so the next cycle claims a
+     * fresh lease and runs clean.
+     *
+     * See `sweep-budget.ts` for what the mechanism can and cannot interrupt. In
+     * short: it abandons the in-flight await, it does not cancel the call.
+     */
     async #runOnceWithDiscoveryFence(opts) {
+        const budget = startDiscoverySweepBudget(this.#discoverySweepBudgetMs);
+        this.#discoverySweepBudgets.add(budget);
+        const sweepStartedAtMs = this.#clock.now();
+        // A sweep that starts while shutdown is already draining would otherwise
+        // get a full fresh budget to hold `stop()` open with.
+        if (this.#stopping)
+            budget.expire();
+        try {
+            return await this.#runDiscoverySweep(opts, budget);
+        }
+        catch (error) {
+            if (error instanceof DiscoverySweepBudgetExceededError) {
+                this.#increment('discoverySweepBudgetExceeded');
+                this.#logger.error?.('[factory] discovery sweep aborted at its aggregate budget', {
+                    budgetMs: error.budgetMs,
+                    // The await the sweep was abandoned on. The one diagnostic no
+                    // per-call bound can produce once the sweep is already wedged: it
+                    // says WHICH transport this wedge is on without anyone having to
+                    // guess which layer to bound next.
+                    phase: error.phase,
+                    elapsedMs: this.#elapsedSince(sweepStartedAtMs),
+                });
+            }
+            throw error;
+        }
+        finally {
+            this.#discoverySweepBudgets.delete(budget);
+            budget.dispose();
+        }
+    }
+    /**
+     * Cut every in-flight sweep's budget short once shutdown starts draining.
+     *
+     * `stop()` deliberately outlives the sweep it started (#301), so a wedged
+     * sweep makes shutdown as long as the budget. The grace window is what keeps
+     * an ordinary restart from throwing away a sweep that was about to commit;
+     * after it, expiry routes the sweep into exactly the abort path a real expiry
+     * takes — lease released, teardown bounded — instead of holding the process.
+     *
+     * Returns the cancel for the grace timer. Always call it: on a prompt
+     * shutdown there is nothing to cut short.
+     */
+    #cutSweepBudgetsShortForStop() {
+        if (this.#discoverySweepBudgets.size === 0)
+            return () => undefined;
+        const timer = setTimeout(() => {
+            for (const budget of this.#discoverySweepBudgets) {
+                if (budget.expired())
+                    continue;
+                this.#increment('discoverySweepBudgetsCutShortForStop');
+                this.#logger.warn?.('[factory] shutdown is draining a sweep; spending its budget now', {
+                    graceMs: STOP_TEARDOWN_TIMEOUT_MS,
+                    budgetMs: budget.budgetMs,
+                });
+                budget.expire();
+            }
+        }, STOP_TEARDOWN_TIMEOUT_MS);
+        timer.unref?.();
+        return () => clearTimeout(timer);
+    }
+    /**
+     * Claim the sweep lease under the budget, compensating for a claim that
+     * lands after we stopped waiting for it.
+     *
+     * The budget abandons the wait, not the call, so the store can still persist
+     * a lease for a claim this sweep has already given up on — and that lease
+     * would be held by an owner that will never renew, commit or release it, so
+     * every later sweep defers until it expires. It is self-expiring and a later
+     * sweep reclaims it as an orphan, which makes this a latency fix rather than
+     * a correctness one; the latency is one whole lease window with no discovery,
+     * which is the thing this PR exists to stop paying.
+     */
+    async #claimDiscoverySweepUnderBudget(budget) {
+        // Issued INSIDE the budget callback, so a budget that is already spent
+        // rejects the phase without opening a lease it could only hand straight
+        // back. The handle is kept out here because the compensation below needs
+        // the promise the wait was abandoned on.
+        let claim;
+        try {
+            return await budget.run('discovery-lease-claim', () => {
+                claim = this.#state.claimDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, this.#clock.now(), DISCOVERY_SWEEP_LEASE_MS);
+                return claim;
+            });
+        }
+        catch (error) {
+            if (!(error instanceof DiscoverySweepBudgetExceededError) || claim === undefined)
+                throw error;
+            void claim.then(async (late) => {
+                if (!late.acquired || !late.lease)
+                    return;
+                this.#increment('discoverySweepStrandedClaimsReleased');
+                this.#logger.warn?.('[factory] releasing a discovery lease that was claimed after the sweep budget expired', {
+                    epoch: late.lease.epoch,
+                    budgetMs: error.budgetMs,
+                });
+                await this.#sweepTeardownStep('stranded discovery lease release', () => this.#state.releaseDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, late.lease.epoch));
+            }, () => undefined).catch(() => undefined);
+            throw error;
+        }
+    }
+    async #runDiscoverySweep(opts, budget) {
         const sweepStartedAtMs = this.#clock.now();
         if (!(opts.dryRun ?? this.#config.dryRun)) {
-            await this.#assertFleetControlPlaneAvailable();
+            // Under the budget, and first, because this is where the 2026-08-25
+            // 07:52:59Z wedge sat: a pre-claim probe on a transport neither #351 nor
+            // #368 covers. The budget does not care which one it is.
+            await budget.run('fleet-control-plane-probe', () => this.#assertFleetControlPlaneAvailable());
         }
-        let claim = await this.#state.claimDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, this.#clock.now(), DISCOVERY_SWEEP_LEASE_MS);
+        let claim = await this.#claimDiscoverySweepUnderBudget(budget);
         if (!claim.acquired && claim.reason === 'backoff') {
             const delayMs = Math.max(0, claim.state.backoffUntilMs - this.#clock.now());
             this.#increment('discoveryBackoffWaits');
@@ -2652,8 +2822,8 @@ export class FactoryLoop {
                 backoffUntilMs: claim.state.backoffUntilMs,
                 consecutiveOverloads: claim.state.consecutiveOverloads,
             });
-            await this.#clock.sleep(delayMs);
-            claim = await this.#state.claimDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, this.#clock.now(), DISCOVERY_SWEEP_LEASE_MS);
+            await budget.run('discovery-backoff-wait', () => this.#clock.sleep(delayMs));
+            claim = await this.#claimDiscoverySweepUnderBudget(budget);
         }
         if (!claim.acquired || !claim.lease) {
             this.#increment('discoverySweepsSkippedInFlight');
@@ -2698,13 +2868,13 @@ export class FactoryLoop {
         this.#startDiscoverySweepRenewal(claim.lease.epoch);
         let leaseReleased = false;
         try {
-            this.#discoverySession = await this.#prepareDiscoverySession(claim);
+            this.#discoverySession = await budget.run('discovery-session', () => this.#prepareDiscoverySession(claim));
             // #297: a 429 raised anywhere in the sweep used to latch and be rethrown
             // here, discarding a completed pass — every issue read, every dispatch —
             // because of one transient shed operation. The work this sweep did is
             // now kept instead, and the ratchet below records that the dependency is
             // shedding but still serving.
-            const report = await this.#performRunOnce(opts);
+            const report = await budget.run('run-once', () => this.#performRunOnce(opts, budget));
             // The exception, and the reason skipping shed units cannot make a sweep
             // unconditionally green: a sweep that was shed AND got no work unit
             // through accomplished nothing. There is no progress to preserve, and
@@ -2715,17 +2885,20 @@ export class FactoryLoop {
             if (this.#discoveryOverloadError !== undefined && !this.#discoverySweepProgress) {
                 throw this.#discoveryOverloadError;
             }
-            const checkpoint = await this.#finalizeDiscoveryCheckpoint();
+            const checkpoint = await budget.run('discovery-checkpoint', () => this.#finalizeDiscoveryCheckpoint());
             // Do not clear the durable lease while a renewal can still be waiting on
             // the same state-file lock. A late renewal that observes the completed
             // (lease-less) checkpoint is a false lease-loss signal and can poison an
             // otherwise successful reconcile cycle.
-            await this.#stopDiscoverySweepRenewal();
+            await budget.run('discovery-renewal-stop', () => this.#stopDiscoverySweepRenewal());
             if (this.#discoverySweepLeaseLost) {
                 throw new Error('discovery sweep lease was lost before checkpoint commit');
             }
             const residual = this.#discoveryOverloadOutcome(claim.state.consecutiveOverloads, 'committed');
-            const completed = await this.#commitDiscoverySweep(claim.lease.epoch, checkpoint, residual);
+            // Under the budget too. A hung commit is the same class of wedge as a
+            // hung read, and the epoch guard in the store makes a late one a no-op:
+            // the teardown below has already released this epoch's lease.
+            const completed = await budget.run('discovery-commit', () => this.#commitDiscoverySweep(claim.lease.epoch, checkpoint, residual));
             leaseReleased = completed;
             if (!completed)
                 throw new Error('discovery sweep lease was lost before completion');
@@ -2742,11 +2915,14 @@ export class FactoryLoop {
             return report;
         }
         catch (error) {
-            await this.#stopDiscoverySweepRenewal();
+            await this.#sweepTeardownStep('discovery sweep renewal stop', () => this.#stopDiscoverySweepRenewal());
             const overload = relayfileOverload(error);
             if (overload) {
                 const outcome = this.#discoveryOverloadOutcome(claim.state.consecutiveOverloads, 'aborted', error);
-                leaseReleased = await this.#state.deferDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, outcome.backoffUntilMs, outcome.consecutiveOverloads);
+                // Bounded for the same reason the release below is: this is the other
+                // path that hands the lease back, and an unbounded one would hold
+                // `#runOnceInFlight` open past the budget that just expired.
+                leaseReleased = await withSweepTeardownDeadline(DISCOVERY_SWEEP_TEARDOWN_TIMEOUT_MS, () => this.#state.deferDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, outcome.backoffUntilMs, outcome.consecutiveOverloads)) ?? false;
                 this.#logger.warn?.('[factory] Relayfile discovery overloaded; backing off before another sweep', {
                     status: overload.status,
                     reason: overload.reason,
@@ -2767,7 +2943,7 @@ export class FactoryLoop {
             throw error;
         }
         finally {
-            await this.#stopDiscoverySweepRenewal();
+            await this.#sweepTeardownStep('discovery sweep renewal stop', () => this.#stopDiscoverySweepRenewal());
             this.#discoverySession = undefined;
             this.#discoverySweepEpoch = undefined;
             this.#discoverySweepStartedAtMs = undefined;
@@ -2784,10 +2960,39 @@ export class FactoryLoop {
             // happens to run and reset it at the top of this method.
             this.#discoverySweepLeaseLost = false;
             if (!leaseReleased) {
-                await this.#state.releaseDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch);
+                // The half of the budget that makes the NEXT cycle clean, so it gets
+                // its own deadline rather than the spent one: an unbounded release
+                // would re-create the wedge one layer down, which is the pattern this
+                // change exists to end. An abandoned release is survivable — the
+                // durable lease carries its own expiry and a later sweep reclaims it
+                // as an orphan (`claim.reclaimedLease` above).
+                await this.#sweepTeardownStep('discovery sweep lease release', () => this.#state.releaseDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch));
             }
         }
     }
+    /**
+     * One sweep-teardown step, under its own deadline.
+     *
+     * Teardown cannot run under the sweep's aggregate budget: on the path that
+     * matters the budget is already spent, so every step would reject and the
+     * lease would never be released. It gets a short independent deadline
+     * instead. Abandoning it costs an orphaned lease for one expiry window;
+     * NOT bounding it costs the whole invariant, because a hung release holds
+     * `#runOnceInFlight` open and every later cycle coalesces onto it.
+     */
+    async #sweepTeardownStep(label, step) {
+        const outcome = await withSweepTeardownDeadline(DISCOVERY_SWEEP_TEARDOWN_TIMEOUT_MS, async () => {
+            await step();
+            return true;
+        });
+        if (outcome === undefined) {
+            this.#increment('discoverySweepTeardownDeadlineExceeded');
+            this.#logger.warn?.('[factory] discovery sweep teardown step abandoned at its deadline', {
+                step: label,
+                timeoutMs: DISCOVERY_SWEEP_TEARDOWN_TIMEOUT_MS,
+            });
+        }
+    }
     /**
      * Commit the sweep, carrying any residual overload backoff into the store.
      *
@@ -2876,7 +3081,7 @@ export class FactoryLoop {
             ...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }),
         };
     }
-    async #performRunOnce(opts = {}) {
+    async #performRunOnce(opts = {}, budget) {
         const dryRun = opts.dryRun ?? this.#config.dryRun;
         const startedAtMs = this.#clock.now();
         const relayfileWaitWarningsAtStart = this.#counters.relayfileOperationWaitWarnings ?? 0;
@@ -2936,6 +3141,12 @@ export class FactoryLoop {
             let readyIssueReads = 0;
             const issueEntries = [];
             for (const path of paths) {
+                // A between-await check, worth exactly what #368 said such a check is
+                // worth against a call that never returns: nothing. What it buys is
+                // the other half — a pass already abandoned at the budget unwinds at
+                // its next iteration if it ever regains control, instead of running to
+                // completion beside the sweep that replaced it.
+                budget?.assertNotExpired('run-once');
                 let issue;
                 let shed = false;
                 try {
@@ -2986,8 +3197,11 @@ export class FactoryLoop {
                     ? '[factory] GitHub ready issue read progress'
                     : '[factory] Linear ready issue read progress', startedAtMs, lastReadyReadProgressAtMs, { read: readyIssueReads, total: paths.length, path });
                 if (!shed) {
-                    if (issue && issueSource === 'linear') {
-                        await this.#recordCanonicalIssueState(issue);
+                    // Both surfaces. Gating this on Linear is what left GitHub ingestion
+                    // with no canonical state at all, so nothing could ever clear the
+                    // terminal row a completed run left behind (#334).
+                    if (issue) {
+                        await this.#recordCanonicalIssueState(issue, this.#issueLifecycleRole(issue));
                     }
                     issueEntries.push({ path, issue });
                 }
@@ -3009,6 +3223,10 @@ export class FactoryLoop {
                 });
             }
             for (const { issue } of issueEntries) {
+                // The dispatch half of the same fence as the read loop above. Without
+                // it a pass abandoned during enumeration would go on to dispatch after
+                // its lease had been handed back, racing the sweep that replaced it.
+                budget?.assertNotExpired('run-once');
                 await this.#refreshLiveHeartbeatIfDue();
                 if (!issue) {
                     continue;
@@ -3650,7 +3868,7 @@ export class FactoryLoop {
                 const activeAgents = lifecycle.agents.filter((agent) => agent.releasedAtMs === undefined);
                 const hasLiveAgent = activeAgents.some((agent) => onlineAgents.has(agent.name));
                 const exitRecoveryActive = activeAgents.some((agent) => this.#agentExitsInFlight.has(agent.name));
-                const dispatchCallActive = this.#dispatchInFlight.has(issueKey(lifecycle.issue));
+                const dispatchCallActive = this.#hasDispatchCallInFlight(lifecycle.issue);
                 // The provider's `factory:in-progress` transition happens immediately
                 // before the durable lifecycle advances from dispatching to running.
                 // A crash can therefore leave any nonterminal phase behind while the
@@ -4194,10 +4412,35 @@ export class FactoryLoop {
         const paths = session.checkpoint.trees[prefix];
         return paths ? [...paths] : undefined;
     }
+    /**
+     * True when this call is a continuation of a sweep that has already ended.
+     *
+     * #372: the aggregate budget abandons the WAIT, not the call, so a read
+     * issued by an aborted sweep can still resolve — by which time the shared
+     * `#discoverySession` and `#discoverySweepEpoch` may belong to the sweep
+     * that replaced it. `discoveryEnumerationPass` is an `AsyncLocalStorage`, so
+     * its store follows the async continuation and still carries the epoch that
+     * ISSUED the read; comparing the two is what tells a live write from a
+     * late one.
+     *
+     * Only when a store exists: a caller outside a discovery pass legitimately
+     * has none, and treating that as stale would silence the live event drain.
+     */
+    #isStaleDiscoveryContinuation() {
+        const issuingPass = discoveryEnumerationPass.getStore();
+        return issuingPass !== undefined && issuingPass.epoch !== this.#discoverySweepEpoch;
+    }
     async #rememberDiscoveryTree(prefix, paths) {
         const session = this.#discoverySession;
         if (!session)
             return;
+        // Committing this listing would put a tree from an abandoned pass into the
+        // replacement sweep's checkpoint, under a watermark that claims to describe
+        // the replacement. That is checkpoint corruption, and it outlives the sweep.
+        if (this.#isStaleDiscoveryContinuation()) {
+            this.#increment('discoveryStaleTreeWritesDropped');
+            return;
+        }
         const uniquePaths = new Set();
         for (let index = 0; index < paths.length; index += 1) {
             uniquePaths.add(paths[index]);
@@ -4272,7 +4515,10 @@ export class FactoryLoop {
         }
         catch (error) {
             const overload = relayfileOverload(error);
-            if (overload && this.#discoverySweepEpoch !== undefined) {
+            // The stale check for the same reason as the tree write above: a 429 that
+            // arrives after its sweep was abandoned is not this sweep's evidence, and
+            // attributing it here would drive the replacement's overload ratchet.
+            if (overload && this.#discoverySweepEpoch !== undefined && !this.#isStaleDiscoveryContinuation()) {
                 this.#discoveryOverloadError ??= error;
                 this.#discoverySweepOverloads += 1;
                 if (overload.retryAfterSeconds !== undefined) {
@@ -4459,6 +4705,44 @@ export class FactoryLoop {
             }
         }
     }
+    /**
+     * Is a `dispatch()` call that could own this work unit's durable lifecycle
+     * running in this process right now?
+     *
+     * `#dispatchInFlight` is keyed by work unit *plus* the dry-run flag and phase
+     * the call was made under, so this must be built from the same three parts
+     * `dispatch()` writes rather than from the bare identity.
+     *
+     * Live only, but BOTH phases — the same `` `${key}:live:` `` prefix `stop()`
+     * uses at `:1531`, and for the same reason.
+     *
+     * The dry-run half of the key is decided once and is stable across both
+     * functions, and `durableDispatch` is `!dryRun && …`, so a `:dry-run:*` call
+     * provably never claims a lifecycle and matching it would preserve a
+     * genuinely orphaned claim (#369 review, codex).
+     *
+     * The phase half is NOT stable. `dispatch()` derives it from the incoming
+     * decision, while `#dispatchUnlocked` re-derives the escalation reason from
+     * the post-routing decision — and `authoritativeRoutedDecision` upgrades a
+     * routeless `confidence: 'low'` triage to `'high'` when the live labels
+     * resolve a repository. A call keyed `:live:escalation` therefore does reach
+     * lifecycle creation, so excluding that phase would reopen exactly the defect
+     * below for it (#369 review, cubic).
+     *
+     * #367: this was `#dispatchInFlight.has(issueKey(issue))`, which no key in
+     * the map can ever equal. `issueKey` is `<key>:<uuid>:<path>`, while every
+     * entry is `<dispatchLifecycleKey>:<dry-run|live>:<phase>` — so the guard was
+     * unsatisfiable by construction and orphan recovery could class a lifecycle
+     * as abandoned while its own dispatch was still mid-flight.
+     */
+    #hasDispatchCallInFlight(issue) {
+        const livePrefix = `${dispatchLifecycleKey(issue)}:live:`;
+        for (const key of this.#dispatchInFlight.keys()) {
+            if (key.startsWith(livePrefix))
+                return true;
+        }
+        return false;
+    }
     async #dispatchUnlocked(decision, opts = {}) {
         const dryRun = opts.dryRun ?? this.#config.dryRun;
         const batch = await this.#batch();
@@ -7041,6 +7325,10 @@ export class FactoryLoop {
         }
         if (!await this.#saveDispatchLifecycle(record, 'complete'))
             return false;
+        // The terminal rows now exist. If a reopen was observed while they were
+        // still being written, it found nothing to clear and consumed the edge; do
+        // the clear it could not do (#334, #375 review).
+        await this.#reconcileReopenObservedDuringCompletion(record);
         this.#increment(releaseReason === 'issue-human-review' ? 'humanReview' : 'done');
         this.#emit('issue-done', { issue: record.issue });
         await this.#writeInFlightRegistry();
@@ -7084,8 +7372,8 @@ export class FactoryLoop {
         }
         try {
             const issue = await this.#readIssue(path);
-            if (issue && !isGithubIssue(issue)) {
-                await this.#recordCanonicalIssueState(issue);
+            if (issue) {
+                await this.#recordCanonicalIssueState(issue, this.#issueLifecycleRole(issue));
             }
             if (issue && this.#dependencyIssueIsTerminal(issue)) {
                 await this.#markDependencyTerminalAndReconcile(issue);
@@ -8195,38 +8483,152 @@ export class FactoryLoop {
         state.backoffUntilMs = 0;
         await this.#state.recordDispatchAttempt(this.#workspaceId, key, state);
     }
-    async #recordCanonicalIssueState(issue) {
-        const key = issueStateKey(issue);
-        const previousStateId = await this.#state.getCanonicalState(this.#workspaceId, key);
-        const previousRole = this.#states.roleOf(previousStateId);
+    /**
+     * The lifecycle role a work unit is in, from whichever surface described it.
+     *
+     * A GitHub issue carries no Linear workflow state — `githubIssueAsFactoryIssue`
+     * sets `stateId: ''` — so its role lives in the `factory:*` labels and the
+     * open/closed flag instead, exactly as `#isIssueReady` and
+     * `#isIssueExternallyTerminal` already read it. Canonical state is recorded as
+     * this role rather than as a raw state id so one provider-neutral value means
+     * the same thing on both surfaces (#334).
+     */
+    #issueLifecycleRole(issue) {
+        if (!isGithubIssue(issue))
+            return this.#states.roleOf(issue.stateId);
+        if (githubFactoryIssueIsClosed(issue))
+            return 'done';
+        const labels = new Set(issue.labels.map((label) => label.trim().toLowerCase()));
+        if (labels.has('factory:human-review'))
+            return 'humanReview';
+        if (this.#isIssueReady(issue))
+            return 'readyForAgent';
+        if (labels.has('factory:in-progress'))
+            return 'agentImplementing';
+        return undefined;
+    }
+    /**
+     * Remember the role this work unit was last seen in, and clear the durable
+     * refusals a completed run left behind when it comes back ready.
+     *
+     * Called for BOTH surfaces. It used to be gated to Linear, so on a
+     * GitHub-sourced Factory the reopen cleanup below was never reached at all
+     * and a terminal row refused the reopened work forever — seven work units
+     * across five repositories on 2026-08-25 (#334).
+     */
+    async #recordCanonicalIssueState(issue, role) {
+        const ref = issueRefForState(issue);
+        const canonicalKey = canonicalStateKey(ref);
+        const previousRole = await this.#state.getCanonicalState(this.#workspaceId, canonicalKey);
         const reopenedFromTerminal = previousRole === 'done' || previousRole === 'humanReview';
-        if (reopenedFromTerminal && this.#states.isRole(issue.stateId, 'readyForAgent')) {
-            const dispatchState = await this.#state.getDispatchAttempts(this.#workspaceId, key);
-            if (dispatchState?.terminal) {
-                dispatchState.attempts = 0;
-                dispatchState.inFlight = false;
-                dispatchState.terminal = false;
-                dispatchState.backoffUntilMs = 0;
-                await this.#state.recordDispatchAttempt(this.#workspaceId, key, dispatchState);
-                this.#increment('dispatchTerminalReopened');
-            }
-            // Match on the work unit, not the surface key: a completed Linear mirror
-            // persists `AR-448` while the GitHub-native arrival of the same issue is
-            // `448`, and comparing surface keys would leave that terminal row in
-            // place to refuse the reopened work forever.
-            const reopenedIdentity = safeDispatchLifecycleKey(issueRefForState(issue));
-            for (const [lifecycleKey, lifecycle] of await this.#state.listDispatchLifecycles(this.#workspaceId)) {
-                if (!isTerminalDispatchLifecycle(lifecycle))
-                    continue;
-                const matches = reopenedIdentity !== undefined &&
-                    safeDispatchLifecycleKey(lifecycle.issue) === reopenedIdentity;
-                if (!matches && lifecycle.issue.key !== issue.key)
-                    continue;
-                await this.#state.clearDispatchLifecycle(this.#workspaceId, lifecycleKey);
-                this.#dispatchLifecycleEpochs.delete(lifecycleKey);
-            }
+        if (reopenedFromTerminal && role === 'readyForAgent') {
+            await this.#clearTerminalRefusals(ref);
+        }
+        // Never erase a remembered terminal role on an intermediate observation.
+        // A reopened issue is routinely seen in a non-ready shape first — reopened
+        // while `factory:in-progress` is still hanging on it, or reopened one event
+        // before the readiness label is reapplied — and writing `role ?? ''` there
+        // overwrote the remembered `done`, disarming the reopen edge so the later
+        // ready observation refused the work exactly as #334 did. Only the three
+        // roles the comparison above acts on are recorded, and that comparison is
+        // this row's only consumer: `getCanonicalState` has no other caller
+        // (cubic-dev-ai, #375 review).
+        if (role === 'done' || role === 'humanReview' || role === 'readyForAgent') {
+            await this.#state.recordCanonicalState(this.#workspaceId, canonicalKey, role);
+        }
+    }
+    /**
+     * Clear the durable refusals a completed run left behind for this work unit,
+     * and report whether anything was actually cleared.
+     *
+     * One increment per reopened work unit, not per cleared row: the attempt
+     * counters and the durable lifecycle are two records of the same refusal and
+     * a unit that clears both reopened once. Counted here rather than at the
+     * attempt clear alone so a reopen that only had a durable row — the shape a
+     * restart leaves, and the one #334 saw in production — is still visible to an
+     * operator.
+     */
+    async #clearTerminalRefusals(ref) {
+        // Dispatch attempts stay on the surface key: that is the key every other
+        // reader and writer of the attempt counters uses, and rekeying only this
+        // one site is the writer/reader mismatch #367 was filed for.
+        const attemptKey = issueStateKey(ref);
+        let reopened = false;
+        const dispatchState = await this.#state.getDispatchAttempts(this.#workspaceId, attemptKey);
+        if (dispatchState?.terminal) {
+            dispatchState.attempts = 0;
+            dispatchState.inFlight = false;
+            dispatchState.terminal = false;
+            dispatchState.backoffUntilMs = 0;
+            await this.#state.recordDispatchAttempt(this.#workspaceId, attemptKey, dispatchState);
+            reopened = true;
+        }
+        // Match on the work unit, not the surface key: a completed Linear mirror
+        // persists `AR-448` while the GitHub-native arrival of the same issue is
+        // `448`, and comparing surface keys would leave that terminal row in
+        // place to refuse the reopened work forever.
+        const reopenedIdentity = safeDispatchLifecycleKey(ref);
+        for (const [lifecycleKey, lifecycle] of await this.#state.listDispatchLifecycles(this.#workspaceId)) {
+            if (!isTerminalDispatchLifecycle(lifecycle))
+                continue;
+            const lifecycleIdentity = safeDispatchLifecycleKey(lifecycle.issue);
+            const matches = reopenedIdentity !== undefined && lifecycleIdentity === reopenedIdentity;
+            // The surface-key fallback covers only a row so old it cannot produce a
+            // work-unit identity at all, and only for a Linear-shaped key. A GitHub
+            // issue key is a bare number that repeats in every repository, so
+            // `key === key` there would let a reopened `factory#364` clear
+            // `cloud#364`'s completed row — the #334 evidence spans five
+            // repositories with overlapping numbering.
+            const legacySurfaceMatch = lifecycleIdentity === undefined &&
+                ISSUE_KEY_PARTS.test(ref.key) &&
+                lifecycle.issue.key === ref.key;
+            if (!matches && !legacySurfaceMatch)
+                continue;
+            await this.#state.clearDispatchLifecycle(this.#workspaceId, lifecycleKey);
+            this.#dispatchLifecycleEpochs.delete(lifecycleKey);
+            reopened = true;
+        }
+        if (reopened)
+            this.#increment('dispatchTerminalReopened');
+        return reopened;
+    }
+    /**
+     * Consume a reopen that was observed *while* completion was still writing its
+     * terminal rows.
+     *
+     * Completion records the terminal role at the provider write, but the durable
+     * rows that refuse a redispatch land much later — `#recordDispatchTerminal`
+     * and the `complete` lifecycle save happen after the completion comment, the
+     * Slack thread and every agent release. A reopen observed inside that window
+     * reads terminal → ready, finds nothing terminal to clear, and *consumes the
+     * edge anyway* by recording `readyForAgent`. The terminal rows then land on a
+     * work unit whose canonical role will never again read terminal → ready, so
+     * the refusal is permanent: #334's exact failure reached through the
+     * completion window instead of through a missing canonical row
+     * (chatgpt-codex-connector P1, #375 review).
+     *
+     * So recheck at the terminal save, which is the first instant the rows the
+     * reopen would have cleared actually exist. This clears nothing the reopen
+     * path would not have cleared itself had it run a moment later — it only
+     * repairs the ordering — and it is armed solely for a completion that
+     * recorded a terminal role of its own.
+     */
+    async #reconcileReopenObservedDuringCompletion(record) {
+        if (!record.canonicalTerminalRoleRecorded)
+            return;
+        const ref = issueRefForState(record.issue);
+        const role = await this.#state.getCanonicalState(this.#workspaceId, canonicalStateKey(ref));
+        if (role !== 'readyForAgent')
+            return;
+        // Counted from the repair, not from the observation. A reopen that landed
+        // after the terminal rows existed was already cleared by the observation
+        // path itself and leaves the same `readyForAgent` row behind, so counting
+        // the read would put a reopen completion never raced into the
+        // completion-window bucket (cubic-dev-ai P3, #375 review). Only a clear
+        // that actually found a refusal to remove is one this recheck repaired.
+        if (await this.#clearTerminalRefusals(ref)) {
+            this.#increment('dispatchTerminalReopenedDuringCompletion');
         }
-        await this.#state.recordCanonicalState(this.#workspaceId, key, issue.stateId);
     }
     async #writeLoopHeartbeat(path, registryPath, status, iteration, maxIterations) {
         const updatedAtMs = this.#clock.now();
@@ -13455,14 +13857,34 @@ export class FactoryLoop {
         this.#postMergeDoneAdvances.add(advanceKey);
         try {
             const githubIssue = isGithubIssue(issue);
+            let terminalStateObserved = true;
             if (githubIssue) {
-                await this.#githubWriteback.closeIssue(issue, `Factory observed pull request #${snapshot.number} merge and completed this issue.\n\nClosing authority: ${authority.evidence}.`);
+                const closeWrite = await this.#githubWriteback.closeIssue(issue, `Factory observed pull request #${snapshot.number} merge and completed this issue.\n\nClosing authority: ${authority.evidence}.`);
+                if (closeWrite === undefined)
+                    this.#recordMissingGithubWritebackReceipt('closeIssue');
+                terminalStateObserved = closeWrite !== undefined;
             }
             else {
                 const doneStateId = this.#states.idFor(issue.team, 'done');
                 await this.#linear.setState(issue, doneStateId);
-                await this.#recordCanonicalIssueState({ ...issueRef(issue), stateId: doneStateId });
             }
+            // Recorded from the write, not left to the next sweep's read: a reopen
+            // landing in that gap would otherwise be read as the FIRST terminal
+            // observation of this unit and the reopen would never be seen (#334).
+            //
+            // Any defined close receipt records it, because canonical state answers
+            // "what state is this unit in", which is a different question from the
+            // "who created this transition" that `applied` alone answers and that
+            // ownership gates elsewhere require. Every receipt in
+            // `GithubIssueCloseWriteResult` is a state observation: `already-matched`
+            // is a provider read of the closed state, `acknowledged` is by contract
+            // an unattributed *visible* transition, and the shipped `gh` adapter
+            // throws unless it reads the issue back as closed. Only a legacy void
+            // adapter (`undefined`) carries no state evidence at all; there the next
+            // sweep records whatever the provider does say (cubic-dev-ai, #375
+            // review).
+            if (terminalStateObserved)
+                await this.#recordCanonicalIssueState(issueRef(issue), 'done');
             this.#emit('writeback-verified', { issue: issueRef(issue), path: issue.path });
             await this.#markDependencyTerminalAndReconcile(issue);
             this.#increment('mergedPrAdvancedDone');
@@ -14200,6 +14622,16 @@ export class FactoryLoop {
                 // park during it must still abort immediately rather than waiting on a
                 // possibly long completion path.
                 this.#issueWritebackInFlight.set(completionKey, issueWritebackSettled);
+                // Whether the PROVIDER's visible state is now terminal. Deliberately a
+                // different question from `issueWritebackConfirmedAtMs`, which asks
+                // whether THIS dispatch owns that transition and stays `applied`-only:
+                // canonical state exists to answer "what state is this unit in", and
+                // every defined receipt answers that (`already-matched` is a provider
+                // read of the target state, `acknowledged` is by contract an
+                // unattributed *visible* transition). Only a legacy void adapter
+                // carries no state evidence, and there the next sweep still records
+                // whatever the provider says (cubic-dev-ai, #375 review).
+                let terminalStateObserved = false;
                 if (githubIssue) {
                     if (humanReview) {
                         const statusWrite = await this.#githubWriteback.setStatus(issue, 'human-review');
@@ -14211,6 +14643,7 @@ export class FactoryLoop {
                         if (statusWrite === 'applied') {
                             record.issueWritebackConfirmedAtMs ??= this.#clock.now();
                         }
+                        terminalStateObserved = statusWrite !== undefined;
                         // The lifecycle-state outcome is now known. Unblock the concurrent
                         // post-spawn read before the separate completion comment write.
                         settleIssueWritebackOnce();
@@ -14227,6 +14660,7 @@ export class FactoryLoop {
                         if (closeWrite === 'applied') {
                             record.issueWritebackConfirmedAtMs ??= this.#clock.now();
                         }
+                        terminalStateObserved = closeWrite !== undefined;
                     }
                 }
                 else {
@@ -14235,7 +14669,17 @@ export class FactoryLoop {
                         : this.#states.idFor(issueTeam, 'done');
                     await this.#linear.setState(issue, targetState);
                     record.issueWritebackConfirmedAtMs ??= this.#clock.now();
-                    await this.#recordCanonicalIssueState({ ...record.issue, stateId: targetState });
+                    terminalStateObserved = true;
+                }
+                if (terminalStateObserved) {
+                    // Both surfaces now, and recorded from the write rather than left to
+                    // the next sweep so a reopen cannot slip into the gap (#334). The
+                    // marker is what lets the terminal save — which happens after the
+                    // completion comment, the Slack thread and every agent release —
+                    // detect a reopen that landed inside that window and consumed the
+                    // reopen edge before there was anything to clear.
+                    await this.#recordCanonicalIssueState(record.issue, humanReview ? 'humanReview' : 'done');
+                    record.canonicalTerminalRoleRecorded = true;
                 }
                 if (record.issueWritebackConfirmedAtMs !== undefined) {
                     this.#emit('writeback-verified', { issue: record.issue, path: issue.path });
@@ -17386,6 +17830,21 @@ const dispatchLifecycleKey = (issue) => dispatchIssueIdentity(issue);
 // Preserve the historical Linear state namespace while keeping GitHub-native
 // issue numbers independent across repositories in the same workspace.
 const issueStateKey = (issue) => githubIssuePathParts(issue.path) ? issueKey(issue) : issue.key;
+/**
+ * The canonical-state key: the work unit, the same identity the dispatch
+ * lifecycle it gates is keyed under (#211, #329, #367).
+ *
+ * Canonical state exists for exactly one decision — did this work unit reach a
+ * terminal role and then come back ready — and that decision clears a durable
+ * row keyed on the work unit. Keying the question per surface and the answer
+ * per work unit is what made #334 reachable: a GitHub-native ref could not see
+ * the terminal role its own Linear mirror had recorded.
+ *
+ * Falls back to the surface key for a ref with no derivable provider identity,
+ * so a malformed row degrades to today's behaviour instead of throwing out of
+ * ingestion.
+ */
+const canonicalStateKey = (issue) => safeDispatchLifecycleKey(issue) ?? issueStateKey(issue);
 const pidsFromSpawnResult = (result) => {
     const pids = new Set();
     for (const pid of result?.pids ?? []) {