@agent-relay/factory 0.1.70 → 0.1.71

package/dist/orchestrator/factory.js

@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto';
 import { readFile } from 'node:fs/promises';
 import { dirname, isAbsolute, resolve } from 'node:path';
 import { DEFAULT_READINESS_RECONCILE_TIMEOUT_MS, FactoryConfigSchema } from '../config/schema.js';
+import { DEFAULT_RELAYFILE_OPERATION_TIMEOUT_MS, RelayfileOperationTimeoutError, relayfileTimeoutWithPhase, withRelayfileCallDeadline, } from '../mount/relayfile-operation-timeout.js';
 import { linearByStatePath, linearByIdPath, linearByUuidPath } from '../constants/linear.js';
 import { stateResolutionFromIds } from '../linear/state-resolver.js';
 import { GithubMergeGate, closeProbePr } from '../github/index.js';
@@ -204,9 +205,25 @@ const DISPATCH_FAILURE_HANDOFF_UNRESOLVED_TTL_MS = 5 * 60_000;
 const DEFAULT_LIVE_HEARTBEAT_INTERVAL_MS = 15_000;
 const REMOTE_OPERATION_PROGRESS_INTERVAL_MS = 15_000;
 const REMOTE_OPERATION_SLOW_WARN_MS = 30_000;
+/** How far the cycle-level relayfile backstop sits above the transport deadline. */
+const RELAYFILE_OPERATION_BACKSTOP_RATIO = 1.25;
 const DISCOVERY_SWEEP_LEASE_MS = 5 * 60_000;
 const DISCOVERY_SWEEP_RENEW_MS = 30_000;
 const READINESS_RECONCILE_FAILURE_THRESHOLD = 3;
+/**
+ * A relayfile fault a swallowing catch must not turn into "no result".
+ *
+ * A 429 already escaped every one of these (#297): it is a fact about the
+ * dependency, not about the one item being read, so folding it into an empty
+ * result reports a clean pass over work that was never served.
+ *
+ * A per-call timeout (#351) is the same fact in a worse costume. The read that
+ * hung will hang for the next item too, and `#githubIssuePaths` swallowing it
+ * turns a wedged dependency into a *successful* sweep that discovered zero
+ * issues — `consecutiveFailures: 0`, no `lastError`, nothing dispatched. That
+ * is the exact silence this bound exists to remove, so it escapes here too.
+ */
+const isPassWideRelayfileFault = (error) => Boolean(relayfileOverload(error)) || error instanceof RelayfileOperationTimeoutError;
 const DISCOVERY_CHANGE_EVENT_LIMIT = 1_000;
 const DISCOVERY_OVERLOAD_BACKOFF_MAX_MS = 5 * 60_000;
 /** First rung of the ladder when the 429 advertises no `Retry-After`. */
@@ -492,6 +509,14 @@ export class FactoryLoop {
     #readinessReconcileInFlight;
     #readinessReconcileIntervalMs = 60_000;
     #readinessReconcileTimeoutMs = DEFAULT_READINESS_RECONCILE_TIMEOUT_MS;
+    /**
+     * Deadline for ONE relayfile call (#351).
+     *
+     * `#readinessReconcileTimeoutMs` bounds the sweep; this bounds the calls
+     * inside it. Only the second one can reach a dependency call that never
+     * returns: a deadline checked between awaits never regains control to check.
+     */
+    #relayfileOperationTimeoutMs = DEFAULT_RELAYFILE_OPERATION_TIMEOUT_MS;
     // Set for exactly as long as a sweep is running. `state` is derived from
     // this, so an in-flight pass can no longer masquerade as the last settled one.
     #readinessReconcileInFlightSinceMs;
@@ -652,6 +677,9 @@ export class FactoryLoop {
     constructor(config, ports) {
         this.#readOnly = ports.readOnly ?? false;
         this.#config = config;
+        // Also read here, not only in `#startLiveSubscription`: a standalone
+        // `runOnce()` never starts the live subscription and must still be bounded.
+        this.#relayfileOperationTimeoutMs = config.liveSubscription.relayfileOperationTimeoutMs;
         this.#mount = ports.mount;
         // Resolved role<->state mapping. The CLI injects a name-resolved, per-team
         // resolution via ports; fall back to one built from explicit stateIds plus
@@ -1234,6 +1262,7 @@ export class FactoryLoop {
         // `start()` overrides skip the schema's cross-field check, so re-apply its
         // floor here: a deadline under one interval would kill every pass.
         this.#readinessReconcileTimeoutMs = Math.max(options.reconcileTimeoutMs, options.reconcileIntervalMs);
+        this.#relayfileOperationTimeoutMs = options.relayfileOperationTimeoutMs;
         this.#liveConnectStartedAtMs = this.#clock.now();
         this.#liveReplaySkewMarginMs = options.replaySkewMarginMs;
         const highWatermark = await this.#currentEventHighWatermark();
@@ -1389,6 +1418,8 @@ export class FactoryLoop {
             replaySkewMarginMs: overrides.replaySkewMarginMs ?? this.#config.liveSubscription.replaySkewMarginMs,
             reconcileIntervalMs: overrides.reconcileIntervalMs ?? this.#config.liveSubscription.reconcileIntervalMs,
             reconcileTimeoutMs: overrides.reconcileTimeoutMs ?? this.#config.liveSubscription.reconcileTimeoutMs,
+            relayfileOperationTimeoutMs: overrides.relayfileOperationTimeoutMs
+                ?? this.#config.liveSubscription.relayfileOperationTimeoutMs,
         };
     }
     async #currentEventCursor(limit) {
@@ -2491,7 +2522,13 @@ export class FactoryLoop {
             }
             const paths = await this.#readyIssuePaths();
             const orphanRecovery = issueSource === 'github'
-                ? await this.#githubOrphanRecoveryContext()
+                ? await this.#githubOrphanRecoveryContext(dryRun)
+                : undefined;
+            // Preserved-not-reconciled is a fact the caller needs either way, but the
+            // two causes are not the same event: a dry run skips the context by
+            // design, a live sweep that has none failed to build one.
+            const orphanRecoveryDegraded = issueSource === 'github' && orphanRecovery === undefined
+                ? (dryRun ? 'dry-run' : 'context-unavailable')
                 : undefined;
             const pulled = [];
             const triaged = [];
@@ -2748,7 +2785,15 @@ export class FactoryLoop {
                         this.#reconciledGithubInProgress.delete(recoveredIdentity);
                 }
             }
-            report = { pulled, triaged, dispatched, skipped, dryRun, slackDegraded: this.#slackDegraded };
+            report = {
+                pulled,
+                triaged,
+                dispatched,
+                skipped,
+                dryRun,
+                slackDegraded: this.#slackDegraded,
+                ...(orphanRecoveryDegraded ? { orphanRecoveryDegraded } : {}),
+            };
             return report;
         }
         catch (error) {
@@ -2769,6 +2814,7 @@ export class FactoryLoop {
                     dispatched: report.dispatched.length,
                     skipped: report.skipped.length,
                     slackDegraded: report.slackDegraded ?? false,
+                    orphanRecoveryDegraded: report.orphanRecoveryDegraded,
                     relayfileWaitWarnings: (this.#counters.relayfileOperationWaitWarnings ?? 0) - relayfileWaitWarningsAtStart,
                     relayfileSlowOperations: (this.#counters.relayfileSlowOperations ?? 0) - relayfileSlowOperationsAtStart,
                     relayfileOperationFailures: (this.#counters.relayfileOperationFailures ?? 0) - relayfileOperationFailuresAtStart,
@@ -3039,7 +3085,7 @@ export class FactoryLoop {
                 : { available: true, highWatermark };
         }
         catch (error) {
-            if (relayfileOverload(error))
+            if (isPassWideRelayfileFault(error))
                 throw error;
             this.#increment('discoveryHighWatermarkFailures');
             this.#logger.warn?.('[factory] discovery high-watermark unavailable; using a full sweep without caching', {
@@ -3062,7 +3108,7 @@ export class FactoryLoop {
             events = [...page.events].sort(compareDiscoveryEvents);
         }
         catch (error) {
-            if (relayfileOverload(error))
+            if (isPassWideRelayfileFault(error))
                 throw error;
             this.#logger.warn?.('[factory] discovery change feed unavailable; refreshing tree prefixes once', {
                 error: describeError(error).errorMessage,
@@ -3124,7 +3170,34 @@ export class FactoryLoop {
             }
         }
     }
-    async #githubOrphanRecoveryContext() {
+    /**
+     * The safety context orphan recovery needs before it may release a leaked
+     * `factory:in-progress` claim: who is online, which issues are actively
+     * owned, and which durable lifecycle rows look abandoned.
+     *
+     * A dry run never builds it. `#reconcileOrphanedGithubInProgress` refuses to
+     * release a claim under `dryRun` before it so much as looks at the context,
+     * and `mayRecoverGithubOrphan` in `#performRunOnce` is itself `!dryRun` —
+     * so the context was already provably unused on that path. Gathering it
+     * anyway was not free: `fleet.roster()` mints this process's workspace
+     * identity on demand, which is exactly what a read-only client refuses
+     * (#343) and exactly what a dry run must not need. Deciding what a sweep
+     * WOULD do must not require an identity to do it with.
+     *
+     * Returning `undefined` is therefore the honest answer for a dry run, not a
+     * failure: the caller reports the sweep as `orphanRecoveryDegraded:
+     * 'dry-run'` and preserves in-progress issues, which is what a dry run does
+     * regardless. The failure counter and the warn below stay for the live path,
+     * where an absent context IS a degradation.
+     */
+    async #githubOrphanRecoveryContext(dryRun) {
+        if (dryRun) {
+            this.#increment('githubOrphanRecoveryContextSkippedDryRun');
+            this.#logger.info?.('[factory] dry run skipped the orphan-recovery safety context; in-progress issues are preserved', {
+                reason: 'a dry run never releases an in-progress claim, so it needs no workspace identity to build one',
+            });
+            return undefined;
+        }
         try {
             const [registry, roster, lifecycles, waitingClarifications] = await Promise.all([
                 readFactoryInFlightRegistry(this.#config.loop.registryPath),
@@ -3701,7 +3774,17 @@ export class FactoryLoop {
             progressTimer.unref?.();
         }
         try {
-            const result = await fn();
+            // The bound the 2026-08-23 wedge needed (#351). `#readinessReconcileTimeoutMs`
+            // is 90 minutes and rejects only the WAIT — the sweep keeps its discovery
+            // lease and the next cycle coalesces onto it — so a call that never
+            // returns stopped dispatch for as long as the process lived. Bounding the
+            // call instead makes the pass unwind, which releases the lease and lets
+            // the next cycle actually start.
+            //
+            // Slightly above the transport's own deadline so the mount's cancellation
+            // wins the race and reports the more precise error; this is the backstop
+            // for mounts that cannot honour a signal.
+            const result = await withRelayfileCallDeadline(operation, details.phase, this.#relayfileOperationBackstopMs(), fn);
             const elapsedMs = this.#elapsedSince(startedAtMs);
             const count = opts.count?.(result);
             if (opts.logComplete) {
@@ -3758,7 +3841,11 @@ export class FactoryLoop {
                     error: describeError(error).errorMessage,
                 });
             }
-            throw error;
+            // The transport deadline is meant to win the race above, and the mount
+            // does not know which phase it was serving. Without this, `lastError`
+            // would name the call but not the context — one of many list/read sites
+            // (codex on #354).
+            throw relayfileTimeoutWithPhase(error, details.phase);
         }
         finally {
             if (progressTimer) {
@@ -3766,6 +3853,19 @@ export class FactoryLoop {
             }
         }
     }
+    /**
+     * The cycle-level backstop budget: the transport deadline plus a margin.
+     *
+     * Proportional rather than a flat grace so it holds at both ends of the
+     * range — a five-minute production budget and a millisecond test budget both
+     * leave the transport room to fire first.
+     */
+    #relayfileOperationBackstopMs() {
+        const timeoutMs = this.#relayfileOperationTimeoutMs;
+        if (!Number.isFinite(timeoutMs) || timeoutMs <= 0)
+            return undefined;
+        return Math.ceil(timeoutMs * RELAYFILE_OPERATION_BACKSTOP_RATIO);
+    }
     #elapsedSince(startedAtMs) {
         return Math.max(0, this.#clock.now() - startedAtMs);
     }
@@ -6571,7 +6671,7 @@ export class FactoryLoop {
             return [...issuePaths.values()].sort();
         }
         catch (error) {
-            if (relayfileOverload(error))
+            if (isPassWideRelayfileFault(error))
                 throw error;
             this.#githubIssuePathIndexReady = false;
             this.#increment('githubIssueListFailures');
@@ -6587,7 +6687,7 @@ export class FactoryLoop {
             parsed = parseJsonContent(content);
         }
         catch (error) {
-            if (relayfileOverload(error))
+            if (isPassWideRelayfileFault(error))
                 throw error;
             return undefined;
         }
@@ -6684,7 +6784,7 @@ export class FactoryLoop {
             this.#increment('githubIssueMirrorsCreated');
         }
         catch (error) {
-            if (relayfileOverload(error))
+            if (isPassWideRelayfileFault(error))
                 throw error;
             this.#logger.error?.('[factory] failed to ingest GitHub issue', error);
         }
@@ -7488,7 +7588,7 @@ export class FactoryLoop {
             return resolvedIssue;
         }
         catch (error) {
-            if (relayfileOverload(error))
+            if (isPassWideRelayfileFault(error))
                 throw error;
             if (isMissingIssueFileError(error) && isIssuePathUnderRoot(path)) {
                 this.#increment('phantomSkipped');
@@ -8584,7 +8684,7 @@ export class FactoryLoop {
                 paths = await this.#listRelayfileTree(root, 'exact-head PR confirmation');
             }
             catch (error) {
-                if (relayfileOverload(error))
+                if (isPassWideRelayfileFault(error))
                     throw error;
                 continue;
             }
@@ -8854,7 +8954,7 @@ export class FactoryLoop {
                     return await this.#listRelayfileTree(root, 'published PR confirmation');
                 }
                 catch (error) {
-                    if (relayfileOverload(error))
+                    if (isPassWideRelayfileFault(error))
                         throw error;
                     return [];
                 }
@@ -10436,7 +10536,7 @@ export class FactoryLoop {
                 }
             }
             catch (error) {
-                if (relayfileOverload(error))
+                if (isPassWideRelayfileFault(error))
                     throw error;
                 this.#logger.warn?.('[factory] unable to list GitHub issue comments for replay', { prefix, error });
             }
@@ -12982,7 +13082,7 @@ export class FactoryLoop {
                 found.push(...tree.filter((path) => path.endsWith('.json') && numberSegment.test(path)));
             }
             catch (error) {
-                if (relayfileOverload(error))
+                if (isPassWideRelayfileFault(error))
                     throw error;
                 // try the next root
             }
@@ -14161,7 +14261,7 @@ export class FactoryLoop {
                 return await this.#listRelayfileTree(prefix, 'Slack identity lookup');
             }
             catch (error) {
-                if (relayfileOverload(error))
+                if (isPassWideRelayfileFault(error))
                     throw error;
                 return [];
             }
@@ -17088,7 +17188,7 @@ const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, listTree
                     paths.add(path);
             }
             catch (error) {
-                if (relayfileOverload(error))
+                if (isPassWideRelayfileFault(error))
                     throw error;
                 listErrors.push(error);
             }