@agent-relay/factory 0.1.69 → 0.1.71

package/dist/orchestrator/factory.js

@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto';
 import { readFile } from 'node:fs/promises';
 import { dirname, isAbsolute, resolve } from 'node:path';
 import { DEFAULT_READINESS_RECONCILE_TIMEOUT_MS, FactoryConfigSchema } from '../config/schema.js';
+import { DEFAULT_RELAYFILE_OPERATION_TIMEOUT_MS, RelayfileOperationTimeoutError, relayfileTimeoutWithPhase, withRelayfileCallDeadline, } from '../mount/relayfile-operation-timeout.js';
 import { linearByStatePath, linearByIdPath, linearByUuidPath } from '../constants/linear.js';
 import { stateResolutionFromIds } from '../linear/state-resolver.js';
 import { GithubMergeGate, closeProbePr } from '../github/index.js';
@@ -204,9 +205,25 @@ const DISPATCH_FAILURE_HANDOFF_UNRESOLVED_TTL_MS = 5 * 60_000;
 const DEFAULT_LIVE_HEARTBEAT_INTERVAL_MS = 15_000;
 const REMOTE_OPERATION_PROGRESS_INTERVAL_MS = 15_000;
 const REMOTE_OPERATION_SLOW_WARN_MS = 30_000;
+/** How far the cycle-level relayfile backstop sits above the transport deadline. */
+const RELAYFILE_OPERATION_BACKSTOP_RATIO = 1.25;
 const DISCOVERY_SWEEP_LEASE_MS = 5 * 60_000;
 const DISCOVERY_SWEEP_RENEW_MS = 30_000;
 const READINESS_RECONCILE_FAILURE_THRESHOLD = 3;
+/**
+ * A relayfile fault a swallowing catch must not turn into "no result".
+ *
+ * A 429 already escaped every one of these (#297): it is a fact about the
+ * dependency, not about the one item being read, so folding it into an empty
+ * result reports a clean pass over work that was never served.
+ *
+ * A per-call timeout (#351) is the same fact in a worse costume. The read that
+ * hung will hang for the next item too, and `#githubIssuePaths` swallowing it
+ * turns a wedged dependency into a *successful* sweep that discovered zero
+ * issues — `consecutiveFailures: 0`, no `lastError`, nothing dispatched. That
+ * is the exact silence this bound exists to remove, so it escapes here too.
+ */
+const isPassWideRelayfileFault = (error) => Boolean(relayfileOverload(error)) || error instanceof RelayfileOperationTimeoutError;
 const DISCOVERY_CHANGE_EVENT_LIMIT = 1_000;
 const DISCOVERY_OVERLOAD_BACKOFF_MAX_MS = 5 * 60_000;
 /** First rung of the ladder when the 429 advertises no `Retry-After`. */
@@ -484,11 +501,22 @@ export class FactoryLoop {
     #liveHeartbeatInFlight = false;
     #liveHeartbeatRefresh;
     #liveHeartbeatLastWriteMs = 0;
+    #heartbeatStartedAtMs;
+    #progressSequence = 0;
+    #lastProgressAtMs;
     #stoppingHeartbeatRefreshActive = false;
     #readinessReconcileTimer;
     #readinessReconcileInFlight;
     #readinessReconcileIntervalMs = 60_000;
     #readinessReconcileTimeoutMs = DEFAULT_READINESS_RECONCILE_TIMEOUT_MS;
+    /**
+     * Deadline for ONE relayfile call (#351).
+     *
+     * `#readinessReconcileTimeoutMs` bounds the sweep; this bounds the calls
+     * inside it. Only the second one can reach a dependency call that never
+     * returns: a deadline checked between awaits never regains control to check.
+     */
+    #relayfileOperationTimeoutMs = DEFAULT_RELAYFILE_OPERATION_TIMEOUT_MS;
     // Set for exactly as long as a sweep is running. `state` is derived from
     // this, so an in-flight pass can no longer masquerade as the last settled one.
     #readinessReconcileInFlightSinceMs;
@@ -645,8 +673,13 @@ export class FactoryLoop {
     #started = false;
     #startMode;
     #stopping = false;
+    #readOnly;
     constructor(config, ports) {
+        this.#readOnly = ports.readOnly ?? false;
         this.#config = config;
+        // Also read here, not only in `#startLiveSubscription`: a standalone
+        // `runOnce()` never starts the live subscription and must still be bounded.
+        this.#relayfileOperationTimeoutMs = config.liveSubscription.relayfileOperationTimeoutMs;
         this.#mount = ports.mount;
         // Resolved role<->state mapping. The CLI injects a name-resolved, per-team
         // resolution via ports; fall back to one built from explicit stateIds plus
@@ -731,7 +764,14 @@ export class FactoryLoop {
                 }
             },
         });
-        this.#wireFleetEvents();
+        // Read-only commands consume no fleet events, and subscribing is not free:
+        // the relay client mints this process's workspace identity to open the
+        // socket. Wiring it here unconditionally is what made `factory status`
+        // register an agent it then abandoned before presence (factory-cloud#55).
+        // The live paths wire in `#start()` instead, where the subscription is
+        // actually used and the identity is meant to persist.
+        if (!this.#readOnly)
+            this.#wireFleetEvents();
     }
     async #resolveIntegrationInstructions() {
         if (this.#integrationInstructionsRefresh) {
@@ -895,6 +935,13 @@ export class FactoryLoop {
         return observedPhase;
     }
     async start(opts = {}) {
+        // A read-only instance was built with no fleet event wiring and, in the
+        // CLI, with a fleet client that cannot register. Starting one would be a
+        // half-live daemon; refuse rather than let a future caller reintroduce the
+        // side effect this mode exists to remove.
+        if (this.#readOnly) {
+            throw new Error('Factory was constructed read-only and cannot be started');
+        }
         if (this.#started) {
             return;
         }
@@ -1215,6 +1262,7 @@ export class FactoryLoop {
         // `start()` overrides skip the schema's cross-field check, so re-apply its
         // floor here: a deadline under one interval would kill every pass.
         this.#readinessReconcileTimeoutMs = Math.max(options.reconcileTimeoutMs, options.reconcileIntervalMs);
+        this.#relayfileOperationTimeoutMs = options.relayfileOperationTimeoutMs;
         this.#liveConnectStartedAtMs = this.#clock.now();
         this.#liveReplaySkewMarginMs = options.replaySkewMarginMs;
         const highWatermark = await this.#currentEventHighWatermark();
@@ -1292,6 +1340,7 @@ export class FactoryLoop {
         }
     }
     async #startLiveHeartbeat() {
+        this.#beginHeartbeatLifetime();
         this.#liveHeartbeatActive = true;
         await this.#writeLiveHeartbeat('running');
         this.#scheduleLiveHeartbeatRefresh();
@@ -1356,6 +1405,11 @@ export class FactoryLoop {
         await this.#writeLoopHeartbeat(this.#config.loop.heartbeatPath, this.#config.loop.registryPath, status, 0, 0);
         this.#liveHeartbeatLastWriteMs = this.#clock.now();
     }
+    #beginHeartbeatLifetime() {
+        this.#heartbeatStartedAtMs = this.#clock.now();
+        this.#progressSequence = 0;
+        this.#lastProgressAtMs = undefined;
+    }
     #liveOptions(overrides) {
         return {
             transport: overrides.transport ?? this.#config.liveSubscription.transport,
@@ -1364,6 +1418,8 @@ export class FactoryLoop {
             replaySkewMarginMs: overrides.replaySkewMarginMs ?? this.#config.liveSubscription.replaySkewMarginMs,
             reconcileIntervalMs: overrides.reconcileIntervalMs ?? this.#config.liveSubscription.reconcileIntervalMs,
             reconcileTimeoutMs: overrides.reconcileTimeoutMs ?? this.#config.liveSubscription.reconcileTimeoutMs,
+            relayfileOperationTimeoutMs: overrides.relayfileOperationTimeoutMs
+                ?? this.#config.liveSubscription.relayfileOperationTimeoutMs,
         };
     }
     async #currentEventCursor(limit) {
@@ -2296,6 +2352,10 @@ export class FactoryLoop {
             leaseReleased = completed;
             if (!completed)
                 throw new Error('discovery sweep lease was lost before completion');
+            // This is the progress boundary consumed by deployment health. A timer
+            // firing, a sweep starting, or a failed/deferred sweep must not move it.
+            this.#progressSequence += 1;
+            this.#lastProgressAtMs = this.#clock.now();
             this.#logger.info?.('[factory] discovery sweep checkpoint committed', {
                 owner: claim.lease.owner,
                 epoch: claim.lease.epoch,
@@ -2462,7 +2522,13 @@ export class FactoryLoop {
             }
             const paths = await this.#readyIssuePaths();
             const orphanRecovery = issueSource === 'github'
-                ? await this.#githubOrphanRecoveryContext()
+                ? await this.#githubOrphanRecoveryContext(dryRun)
+                : undefined;
+            // Preserved-not-reconciled is a fact the caller needs either way, but the
+            // two causes are not the same event: a dry run skips the context by
+            // design, a live sweep that has none failed to build one.
+            const orphanRecoveryDegraded = issueSource === 'github' && orphanRecovery === undefined
+                ? (dryRun ? 'dry-run' : 'context-unavailable')
                 : undefined;
             const pulled = [];
             const triaged = [];
@@ -2719,7 +2785,15 @@ export class FactoryLoop {
                         this.#reconciledGithubInProgress.delete(recoveredIdentity);
                 }
             }
-            report = { pulled, triaged, dispatched, skipped, dryRun, slackDegraded: this.#slackDegraded };
+            report = {
+                pulled,
+                triaged,
+                dispatched,
+                skipped,
+                dryRun,
+                slackDegraded: this.#slackDegraded,
+                ...(orphanRecoveryDegraded ? { orphanRecoveryDegraded } : {}),
+            };
             return report;
         }
         catch (error) {
@@ -2740,6 +2814,7 @@ export class FactoryLoop {
                     dispatched: report.dispatched.length,
                     skipped: report.skipped.length,
                     slackDegraded: report.slackDegraded ?? false,
+                    orphanRecoveryDegraded: report.orphanRecoveryDegraded,
                     relayfileWaitWarnings: (this.#counters.relayfileOperationWaitWarnings ?? 0) - relayfileWaitWarningsAtStart,
                     relayfileSlowOperations: (this.#counters.relayfileSlowOperations ?? 0) - relayfileSlowOperationsAtStart,
                     relayfileOperationFailures: (this.#counters.relayfileOperationFailures ?? 0) - relayfileOperationFailuresAtStart,
@@ -3010,7 +3085,7 @@ export class FactoryLoop {
                 : { available: true, highWatermark };
         }
         catch (error) {
-            if (relayfileOverload(error))
+            if (isPassWideRelayfileFault(error))
                 throw error;
             this.#increment('discoveryHighWatermarkFailures');
             this.#logger.warn?.('[factory] discovery high-watermark unavailable; using a full sweep without caching', {
@@ -3033,7 +3108,7 @@ export class FactoryLoop {
             events = [...page.events].sort(compareDiscoveryEvents);
         }
         catch (error) {
-            if (relayfileOverload(error))
+            if (isPassWideRelayfileFault(error))
                 throw error;
             this.#logger.warn?.('[factory] discovery change feed unavailable; refreshing tree prefixes once', {
                 error: describeError(error).errorMessage,
@@ -3095,7 +3170,34 @@ export class FactoryLoop {
             }
         }
     }
-    async #githubOrphanRecoveryContext() {
+    /**
+     * The safety context orphan recovery needs before it may release a leaked
+     * `factory:in-progress` claim: who is online, which issues are actively
+     * owned, and which durable lifecycle rows look abandoned.
+     *
+     * A dry run never builds it. `#reconcileOrphanedGithubInProgress` refuses to
+     * release a claim under `dryRun` before it so much as looks at the context,
+     * and `mayRecoverGithubOrphan` in `#performRunOnce` is itself `!dryRun` —
+     * so the context was already provably unused on that path. Gathering it
+     * anyway was not free: `fleet.roster()` mints this process's workspace
+     * identity on demand, which is exactly what a read-only client refuses
+     * (#343) and exactly what a dry run must not need. Deciding what a sweep
+     * WOULD do must not require an identity to do it with.
+     *
+     * Returning `undefined` is therefore the honest answer for a dry run, not a
+     * failure: the caller reports the sweep as `orphanRecoveryDegraded:
+     * 'dry-run'` and preserves in-progress issues, which is what a dry run does
+     * regardless. The failure counter and the warn below stay for the live path,
+     * where an absent context IS a degradation.
+     */
+    async #githubOrphanRecoveryContext(dryRun) {
+        if (dryRun) {
+            this.#increment('githubOrphanRecoveryContextSkippedDryRun');
+            this.#logger.info?.('[factory] dry run skipped the orphan-recovery safety context; in-progress issues are preserved', {
+                reason: 'a dry run never releases an in-progress claim, so it needs no workspace identity to build one',
+            });
+            return undefined;
+        }
         try {
             const [registry, roster, lifecycles, waitingClarifications] = await Promise.all([
                 readFactoryInFlightRegistry(this.#config.loop.registryPath),
@@ -3672,7 +3774,17 @@ export class FactoryLoop {
             progressTimer.unref?.();
         }
         try {
-            const result = await fn();
+            // The bound the 2026-08-23 wedge needed (#351). `#readinessReconcileTimeoutMs`
+            // is 90 minutes and rejects only the WAIT — the sweep keeps its discovery
+            // lease and the next cycle coalesces onto it — so a call that never
+            // returns stopped dispatch for as long as the process lived. Bounding the
+            // call instead makes the pass unwind, which releases the lease and lets
+            // the next cycle actually start.
+            //
+            // Slightly above the transport's own deadline so the mount's cancellation
+            // wins the race and reports the more precise error; this is the backstop
+            // for mounts that cannot honour a signal.
+            const result = await withRelayfileCallDeadline(operation, details.phase, this.#relayfileOperationBackstopMs(), fn);
             const elapsedMs = this.#elapsedSince(startedAtMs);
             const count = opts.count?.(result);
             if (opts.logComplete) {
@@ -3729,7 +3841,11 @@ export class FactoryLoop {
                     error: describeError(error).errorMessage,
                 });
             }
-            throw error;
+            // The transport deadline is meant to win the race above, and the mount
+            // does not know which phase it was serving. Without this, `lastError`
+            // would name the call but not the context — one of many list/read sites
+            // (codex on #354).
+            throw relayfileTimeoutWithPhase(error, details.phase);
         }
         finally {
             if (progressTimer) {
@@ -3737,6 +3853,19 @@ export class FactoryLoop {
             }
         }
     }
+    /**
+     * The cycle-level backstop budget: the transport deadline plus a margin.
+     *
+     * Proportional rather than a flat grace so it holds at both ends of the
+     * range — a five-minute production budget and a millisecond test budget both
+     * leave the transport room to fire first.
+     */
+    #relayfileOperationBackstopMs() {
+        const timeoutMs = this.#relayfileOperationTimeoutMs;
+        if (!Number.isFinite(timeoutMs) || timeoutMs <= 0)
+            return undefined;
+        return Math.ceil(timeoutMs * RELAYFILE_OPERATION_BACKSTOP_RATIO);
+    }
     #elapsedSince(startedAtMs) {
         return Math.max(0, this.#clock.now() - startedAtMs);
     }
@@ -3756,6 +3885,7 @@ export class FactoryLoop {
         const maxConsecutiveFailures = Math.min(5, Math.max(1, Math.trunc(opts.maxConsecutiveFailures ?? this.#config.loop.maxConsecutiveFailures)));
         const heartbeatPath = opts.heartbeatPath ?? this.#config.loop.heartbeatPath;
         const registryPath = opts.registryPath ?? this.#config.loop.registryPath;
+        this.#beginHeartbeatLifetime();
         this.#loopReapPaths = { heartbeatPath, registryPath };
         const reports = [];
         let consecutiveFailures = 0;
@@ -6541,7 +6671,7 @@ export class FactoryLoop {
             return [...issuePaths.values()].sort();
         }
         catch (error) {
-            if (relayfileOverload(error))
+            if (isPassWideRelayfileFault(error))
                 throw error;
             this.#githubIssuePathIndexReady = false;
             this.#increment('githubIssueListFailures');
@@ -6557,7 +6687,7 @@ export class FactoryLoop {
             parsed = parseJsonContent(content);
         }
         catch (error) {
-            if (relayfileOverload(error))
+            if (isPassWideRelayfileFault(error))
                 throw error;
             return undefined;
         }
@@ -6654,7 +6784,7 @@ export class FactoryLoop {
             this.#increment('githubIssueMirrorsCreated');
         }
         catch (error) {
-            if (relayfileOverload(error))
+            if (isPassWideRelayfileFault(error))
                 throw error;
             this.#logger.error?.('[factory] failed to ingest GitHub issue', error);
         }
@@ -7205,13 +7335,28 @@ export class FactoryLoop {
     }
     async #writeLoopHeartbeat(path, registryPath, status, iteration, maxIterations) {
         const updatedAtMs = this.#clock.now();
+        this.#heartbeatStartedAtMs ??= updatedAtMs;
         const heartbeat = {
             pid: process.pid,
             status,
+            source: maxIterations === 0 ? 'live-timer' : 'bounded-loop',
             iteration,
             maxIterations,
+            startedAt: new Date(this.#heartbeatStartedAtMs).toISOString(),
+            startedAtMs: this.#heartbeatStartedAtMs,
+            progressContract: 'discovery-sweep-v1',
             updatedAt: new Date(updatedAtMs).toISOString(),
             updatedAtMs,
+            ...(this.#lastProgressAtMs !== undefined
+                ? {
+                    progress: {
+                        sequence: this.#progressSequence,
+                        operation: 'discovery-sweep',
+                        updatedAt: new Date(this.#lastProgressAtMs).toISOString(),
+                        updatedAtMs: this.#lastProgressAtMs,
+                    },
+                }
+                : {}),
             registryPath,
             eventListener: this.#eventListenerStatus(),
             readinessReconcile: this.#readinessReconcileStatus(),
@@ -7443,7 +7588,7 @@ export class FactoryLoop {
             return resolvedIssue;
         }
         catch (error) {
-            if (relayfileOverload(error))
+            if (isPassWideRelayfileFault(error))
                 throw error;
             if (isMissingIssueFileError(error) && isIssuePathUnderRoot(path)) {
                 this.#increment('phantomSkipped');
@@ -8539,7 +8684,7 @@ export class FactoryLoop {
                 paths = await this.#listRelayfileTree(root, 'exact-head PR confirmation');
             }
             catch (error) {
-                if (relayfileOverload(error))
+                if (isPassWideRelayfileFault(error))
                     throw error;
                 continue;
             }
@@ -8809,7 +8954,7 @@ export class FactoryLoop {
                     return await this.#listRelayfileTree(root, 'published PR confirmation');
                 }
                 catch (error) {
-                    if (relayfileOverload(error))
+                    if (isPassWideRelayfileFault(error))
                         throw error;
                     return [];
                 }
@@ -10391,7 +10536,7 @@ export class FactoryLoop {
                 }
             }
             catch (error) {
-                if (relayfileOverload(error))
+                if (isPassWideRelayfileFault(error))
                     throw error;
                 this.#logger.warn?.('[factory] unable to list GitHub issue comments for replay', { prefix, error });
             }
@@ -12937,7 +13082,7 @@ export class FactoryLoop {
                 found.push(...tree.filter((path) => path.endsWith('.json') && numberSegment.test(path)));
             }
             catch (error) {
-                if (relayfileOverload(error))
+                if (isPassWideRelayfileFault(error))
                     throw error;
                 // try the next root
             }
@@ -14116,7 +14261,7 @@ export class FactoryLoop {
                 return await this.#listRelayfileTree(prefix, 'Slack identity lookup');
             }
             catch (error) {
-                if (relayfileOverload(error))
+                if (isPassWideRelayfileFault(error))
                     throw error;
                 return [];
             }
@@ -17043,7 +17188,7 @@ const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, listTree
                     paths.add(path);
             }
             catch (error) {
-                if (relayfileOverload(error))
+                if (isPassWideRelayfileFault(error))
                     throw error;
                 listErrors.push(error);
             }