@agent-relay/factory 0.1.70 → 0.1.72

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/dist/cli/diagnose.d.ts +12 -1
  2. package/dist/cli/diagnose.d.ts.map +1 -1
  3. package/dist/cli/diagnose.js +37 -1
  4. package/dist/cli/diagnose.js.map +1 -1
  5. package/dist/cli/fleet.d.ts.map +1 -1
  6. package/dist/cli/fleet.js +3 -0
  7. package/dist/cli/fleet.js.map +1 -1
  8. package/dist/config/schema.d.ts +84 -40
  9. package/dist/config/schema.d.ts.map +1 -1
  10. package/dist/config/schema.js +14 -0
  11. package/dist/config/schema.js.map +1 -1
  12. package/dist/environments/connection-registry.d.ts +20 -20
  13. package/dist/fleet/relay-fleet-client.d.ts.map +1 -1
  14. package/dist/fleet/relay-fleet-client.js +57 -19
  15. package/dist/fleet/relay-fleet-client.js.map +1 -1
  16. package/dist/index.d.ts +2 -0
  17. package/dist/index.d.ts.map +1 -1
  18. package/dist/index.js +1 -0
  19. package/dist/index.js.map +1 -1
  20. package/dist/mount/relayfile-cloud-mount-client.d.ts +10 -2
  21. package/dist/mount/relayfile-cloud-mount-client.d.ts.map +1 -1
  22. package/dist/mount/relayfile-cloud-mount-client.js +90 -14
  23. package/dist/mount/relayfile-cloud-mount-client.js.map +1 -1
  24. package/dist/mount/relayfile-operation-timeout.d.ts +97 -0
  25. package/dist/mount/relayfile-operation-timeout.d.ts.map +1 -0
  26. package/dist/mount/relayfile-operation-timeout.js +149 -0
  27. package/dist/mount/relayfile-operation-timeout.js.map +1 -0
  28. package/dist/orchestrator/factory.d.ts.map +1 -1
  29. package/dist/orchestrator/factory.js +223 -32
  30. package/dist/orchestrator/factory.js.map +1 -1
  31. package/dist/orchestrator/public-health.d.ts.map +1 -1
  32. package/dist/orchestrator/public-health.js +75 -0
  33. package/dist/orchestrator/public-health.js.map +1 -1
  34. package/dist/orchestrator/sweep-skip-reason.d.ts +45 -0
  35. package/dist/orchestrator/sweep-skip-reason.d.ts.map +1 -0
  36. package/dist/orchestrator/sweep-skip-reason.js +82 -0
  37. package/dist/orchestrator/sweep-skip-reason.js.map +1 -0
  38. package/dist/types.d.ts +69 -0
  39. package/dist/types.d.ts.map +1 -1
  40. package/package.json +1 -1
@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto';
2
2
  import { readFile } from 'node:fs/promises';
3
3
  import { dirname, isAbsolute, resolve } from 'node:path';
4
4
  import { DEFAULT_READINESS_RECONCILE_TIMEOUT_MS, FactoryConfigSchema } from '../config/schema.js';
5
+ import { DEFAULT_RELAYFILE_OPERATION_TIMEOUT_MS, RelayfileOperationTimeoutError, relayfileTimeoutWithPhase, withRelayfileCallDeadline, } from '../mount/relayfile-operation-timeout.js';
5
6
  import { linearByStatePath, linearByIdPath, linearByUuidPath } from '../constants/linear.js';
6
7
  import { stateResolutionFromIds } from '../linear/state-resolver.js';
7
8
  import { GithubMergeGate, closeProbePr } from '../github/index.js';
@@ -34,6 +35,7 @@ import { readFactoryInFlightRegistry, terminatePids } from './reaper.js';
34
35
  import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../observability/events.js';
35
36
  import { telemetryErrorClass } from '../observability/error-class.js';
36
37
  import { derivedReadinessReconcileState, publicHealthFromHeartbeat, readinessReconcileInFlightMs, } from './public-health.js';
38
+ import { factorySweepSkipReasonCounts } from './sweep-skip-reason.js';
37
39
  import { boundedRunCostTotal, CostLedger } from '../cost/ledger.js';
38
40
  import { createTicketDispatchDelivery } from '../delivery/ticket-dispatch.js';
39
41
  import { canonicalTrajectorySessionRef, renderTrajectoryPointer, stripTrajectoryPointers, } from '../trajectory.js';
@@ -204,9 +206,25 @@ const DISPATCH_FAILURE_HANDOFF_UNRESOLVED_TTL_MS = 5 * 60_000;
204
206
  const DEFAULT_LIVE_HEARTBEAT_INTERVAL_MS = 15_000;
205
207
  const REMOTE_OPERATION_PROGRESS_INTERVAL_MS = 15_000;
206
208
  const REMOTE_OPERATION_SLOW_WARN_MS = 30_000;
209
+ /** How far the cycle-level relayfile backstop sits above the transport deadline. */
210
+ const RELAYFILE_OPERATION_BACKSTOP_RATIO = 1.25;
207
211
  const DISCOVERY_SWEEP_LEASE_MS = 5 * 60_000;
208
212
  const DISCOVERY_SWEEP_RENEW_MS = 30_000;
209
213
  const READINESS_RECONCILE_FAILURE_THRESHOLD = 3;
214
+ /**
215
+ * A relayfile fault a swallowing catch must not turn into "no result".
216
+ *
217
+ * A 429 already escaped every one of these (#297): it is a fact about the
218
+ * dependency, not about the one item being read, so folding it into an empty
219
+ * result reports a clean pass over work that was never served.
220
+ *
221
+ * A per-call timeout (#351) is the same fact in a worse costume. The read that
222
+ * hung will hang for the next item too, and `#githubIssuePaths` swallowing it
223
+ * turns a wedged dependency into a *successful* sweep that discovered zero
224
+ * issues — `consecutiveFailures: 0`, no `lastError`, nothing dispatched. That
225
+ * is the exact silence this bound exists to remove, so it escapes here too.
226
+ */
227
+ const isPassWideRelayfileFault = (error) => Boolean(relayfileOverload(error)) || error instanceof RelayfileOperationTimeoutError;
210
228
  const DISCOVERY_CHANGE_EVENT_LIMIT = 1_000;
211
229
  const DISCOVERY_OVERLOAD_BACKOFF_MAX_MS = 5 * 60_000;
212
230
  /** First rung of the ladder when the 429 advertises no `Retry-After`. */
@@ -492,6 +510,14 @@ export class FactoryLoop {
492
510
  #readinessReconcileInFlight;
493
511
  #readinessReconcileIntervalMs = 60_000;
494
512
  #readinessReconcileTimeoutMs = DEFAULT_READINESS_RECONCILE_TIMEOUT_MS;
513
+ /**
514
+ * Deadline for ONE relayfile call (#351).
515
+ *
516
+ * `#readinessReconcileTimeoutMs` bounds the sweep; this bounds the calls
517
+ * inside it. Only the second one can reach a dependency call that never
518
+ * returns: a deadline checked between awaits never regains control to check.
519
+ */
520
+ #relayfileOperationTimeoutMs = DEFAULT_RELAYFILE_OPERATION_TIMEOUT_MS;
495
521
  // Set for exactly as long as a sweep is running. `state` is derived from
496
522
  // this, so an in-flight pass can no longer masquerade as the last settled one.
497
523
  #readinessReconcileInFlightSinceMs;
@@ -526,6 +552,19 @@ export class FactoryLoop {
526
552
  #readinessReconcileLastFailureAtMs;
527
553
  #readinessReconcileLastError;
528
554
  #readinessReconcileLastErrorClass;
555
+ /**
556
+ * The last *completed* sweep's arithmetic (#355).
557
+ *
558
+ * Held as one record rather than four fields so it can only ever be replaced
559
+ * whole: publishing a `dispatched` from one pass beside a `candidates` from
560
+ * another would be worse than publishing neither, since the whole use of
561
+ * these numbers is comparing them to each other.
562
+ *
563
+ * `undefined` until a sweep completes, and never initialised to zeroes —
564
+ * "this daemon has not finished a sweep" and "a sweep finished and found
565
+ * nothing" are the two readings #355 has to tell apart.
566
+ */
567
+ #readinessReconcileLastSweep;
529
568
  #liveEventQueue = [];
530
569
  #liveEventDrainScheduled = false;
531
570
  #liveEventDrainActive = false;
@@ -652,6 +691,9 @@ export class FactoryLoop {
652
691
  constructor(config, ports) {
653
692
  this.#readOnly = ports.readOnly ?? false;
654
693
  this.#config = config;
694
+ // Also read here, not only in `#startLiveSubscription`: a standalone
695
+ // `runOnce()` never starts the live subscription and must still be bounded.
696
+ this.#relayfileOperationTimeoutMs = config.liveSubscription.relayfileOperationTimeoutMs;
655
697
  this.#mount = ports.mount;
656
698
  // Resolved role<->state mapping. The CLI injects a name-resolved, per-team
657
699
  // resolution via ports; fall back to one built from explicit stateIds plus
@@ -1234,6 +1276,7 @@ export class FactoryLoop {
1234
1276
  // `start()` overrides skip the schema's cross-field check, so re-apply its
1235
1277
  // floor here: a deadline under one interval would kill every pass.
1236
1278
  this.#readinessReconcileTimeoutMs = Math.max(options.reconcileTimeoutMs, options.reconcileIntervalMs);
1279
+ this.#relayfileOperationTimeoutMs = options.relayfileOperationTimeoutMs;
1237
1280
  this.#liveConnectStartedAtMs = this.#clock.now();
1238
1281
  this.#liveReplaySkewMarginMs = options.replaySkewMarginMs;
1239
1282
  const highWatermark = await this.#currentEventHighWatermark();
@@ -1284,7 +1327,11 @@ export class FactoryLoop {
1284
1327
  const backfillStartedAtMs = this.#clock.now();
1285
1328
  this.#readinessReconcileLastStartedAtMs = backfillStartedAtMs;
1286
1329
  try {
1287
- await this.runOnce();
1330
+ // The startup backfill is a discovery pass like any other, and on a
1331
+ // cold container it is the first — and for the next interval, only —
1332
+ // sweep whose counts exist. Leaving it unrecorded would make a daemon
1333
+ // that has completed a full pass still read as "never ran" (#355).
1334
+ this.#recordReadinessSweepOutcome(await this.runOnce());
1288
1335
  this.#readinessReconcileLastDurationMs = this.#elapsedSince(backfillStartedAtMs);
1289
1336
  this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
1290
1337
  }
@@ -1389,6 +1436,8 @@ export class FactoryLoop {
1389
1436
  replaySkewMarginMs: overrides.replaySkewMarginMs ?? this.#config.liveSubscription.replaySkewMarginMs,
1390
1437
  reconcileIntervalMs: overrides.reconcileIntervalMs ?? this.#config.liveSubscription.reconcileIntervalMs,
1391
1438
  reconcileTimeoutMs: overrides.reconcileTimeoutMs ?? this.#config.liveSubscription.reconcileTimeoutMs,
1439
+ relayfileOperationTimeoutMs: overrides.relayfileOperationTimeoutMs
1440
+ ?? this.#config.liveSubscription.relayfileOperationTimeoutMs,
1392
1441
  };
1393
1442
  }
1394
1443
  async #currentEventCursor(limit) {
@@ -1513,11 +1562,18 @@ export class FactoryLoop {
1513
1562
  this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
1514
1563
  this.#readinessReconcileLastError = undefined;
1515
1564
  this.#readinessReconcileLastErrorClass = undefined;
1565
+ // The three integers below have gone to stdout since this loop existed,
1566
+ // and stdout does not reach the deployed container's operator (#355).
1567
+ // Publishing them is what lets a reader tell a sweep that saw eligible
1568
+ // work and rejected it from one that never pulled it at all.
1569
+ this.#recordReadinessSweepOutcome(report);
1516
1570
  this.#logger.info?.('[factory] periodic readiness reconciliation completed', {
1517
1571
  durationMs: this.#readinessReconcileLastDurationMs,
1518
1572
  candidates: report.pulled.length,
1519
1573
  dispatched: report.dispatched.length,
1520
1574
  skipped: report.skipped.length,
1575
+ skipReasons: this.#readinessReconcileLastSweep?.skipReasons,
1576
+ discoveryDeferred: report.discoveryDeferred,
1521
1577
  });
1522
1578
  }
1523
1579
  catch (error) {
@@ -2491,7 +2547,13 @@ export class FactoryLoop {
2491
2547
  }
2492
2548
  const paths = await this.#readyIssuePaths();
2493
2549
  const orphanRecovery = issueSource === 'github'
2494
- ? await this.#githubOrphanRecoveryContext()
2550
+ ? await this.#githubOrphanRecoveryContext(dryRun)
2551
+ : undefined;
2552
+ // Preserved-not-reconciled is a fact the caller needs either way, but the
2553
+ // two causes are not the same event: a dry run skips the context by
2554
+ // design, a live sweep that has none failed to build one.
2555
+ const orphanRecoveryDegraded = issueSource === 'github' && orphanRecovery === undefined
2556
+ ? (dryRun ? 'dry-run' : 'context-unavailable')
2495
2557
  : undefined;
2496
2558
  const pulled = [];
2497
2559
  const triaged = [];
@@ -2503,6 +2565,7 @@ export class FactoryLoop {
2503
2565
  issue: entry.issue.key,
2504
2566
  path: entry.issue.path,
2505
2567
  reason: entry.reason,
2568
+ code: entry.code,
2506
2569
  });
2507
2570
  };
2508
2571
  // Backstop for the skip-by-default catch below: see #292. Reset only by
@@ -2542,7 +2605,11 @@ export class FactoryLoop {
2542
2605
  retryAfterSeconds: overload.retryAfterSeconds,
2543
2606
  sweepOverloads: this.#discoverySweepOverloads,
2544
2607
  });
2545
- recordSkip({ issue: issueRefFromPath(path), reason: perItemDispatchSkipReason(error) });
2608
+ recordSkip({
2609
+ issue: issueRefFromPath(path),
2610
+ reason: perItemDispatchSkipReason(error),
2611
+ code: 'read-failed',
2612
+ });
2546
2613
  }
2547
2614
  readyIssueReads += 1;
2548
2615
  // Relayfile served this work unit's read: the dependency is shedding
@@ -2603,7 +2670,7 @@ export class FactoryLoop {
2603
2670
  if (!mayRecoverGithubOrphan) {
2604
2671
  const dispatchBlock = await this.#dispatchBlockReason(issue);
2605
2672
  if (dispatchBlock) {
2606
- recordSkip({ issue: issueRef(issue), reason: dispatchBlock });
2673
+ recordSkip({ issue: issueRef(issue), ...dispatchBlock });
2607
2674
  continue;
2608
2675
  }
2609
2676
  }
@@ -2613,20 +2680,25 @@ export class FactoryLoop {
2613
2680
  const recoveredOrphan = orphanResult.recovered;
2614
2681
  const batch = await this.#batch();
2615
2682
  if (batch.isInFlight(issue) || batch.isQueued(issue)) {
2616
- recordSkip({ issue: issueRef(issue), reason: orphanResult.reason ?? 'already tracked' });
2683
+ recordSkip({
2684
+ issue: issueRef(issue),
2685
+ reason: orphanResult.reason ?? 'already tracked',
2686
+ code: 'already-tracked',
2687
+ });
2617
2688
  continue;
2618
2689
  }
2619
2690
  if (!wasReady && !recoveredOrphan) {
2620
2691
  if (mayRecoverGithubOrphan) {
2621
2692
  const dispatchBlock = await this.#dispatchBlockReason(issue);
2622
2693
  if (dispatchBlock) {
2623
- recordSkip({ issue: issueRef(issue), reason: dispatchBlock });
2694
+ recordSkip({ issue: issueRef(issue), ...dispatchBlock });
2624
2695
  continue;
2625
2696
  }
2626
2697
  }
2627
2698
  recordSkip({
2628
2699
  issue: issueRef(issue),
2629
2700
  reason: orphanResult.reason ?? 'live state is not ready-for-agent',
2701
+ code: 'not-ready',
2630
2702
  });
2631
2703
  continue;
2632
2704
  }
@@ -2635,16 +2707,20 @@ export class FactoryLoop {
2635
2707
  if (recoveredOrphan) {
2636
2708
  const dispatchBlock = await this.#dispatchBlockReason(issue);
2637
2709
  if (dispatchBlock) {
2638
- recordSkip({ issue: issueRef(issue), reason: dispatchBlock });
2710
+ recordSkip({ issue: issueRef(issue), ...dispatchBlock });
2639
2711
  continue;
2640
2712
  }
2641
2713
  }
2642
2714
  if (!isInFactoryScope(issue, this.#config.safety)) {
2643
- recordSkip({ issue: issueRef(issue), reason: 'not factory-e2e scope' });
2715
+ recordSkip({ issue: issueRef(issue), reason: 'not factory-e2e scope', code: 'out-of-scope' });
2644
2716
  continue;
2645
2717
  }
2646
2718
  if (!isDispatchableIssue(issue)) {
2647
- recordSkip({ issue: issueRef(issue), reason: 'not reconciled real Linear issue' });
2719
+ recordSkip({
2720
+ issue: issueRef(issue),
2721
+ reason: 'not reconciled real Linear issue',
2722
+ code: 'not-dispatchable',
2723
+ });
2648
2724
  continue;
2649
2725
  }
2650
2726
  const decision = await this.triageIssue(issue);
@@ -2657,12 +2733,17 @@ export class FactoryLoop {
2657
2733
  // decays the durable overload ratchet (#297).
2658
2734
  this.#discoverySweepProgress = true;
2659
2735
  if (result.agents.length === 0 && !dryRun) {
2736
+ const code = result.hold?.kind === 'dependency-cycle'
2737
+ ? 'dependency-cycle'
2738
+ : result.hold?.kind === 'dependency'
2739
+ ? 'parked-dependency'
2740
+ : 'queued-or-escalated';
2660
2741
  const reason = result.hold?.kind === 'dependency-cycle'
2661
2742
  ? `dependency cycle detected: ${result.hold.cycle?.join(' -> ') ?? 'unknown cycle'}`
2662
2743
  : result.hold?.kind === 'dependency'
2663
2744
  ? `parked on dependencies: ${result.hold.blockers?.join(', ') ?? 'unresolved dependency'}`
2664
2745
  : 'queued or escalated';
2665
- recordSkip({ issue: decision.issue, reason });
2746
+ recordSkip({ issue: decision.issue, reason, code });
2666
2747
  }
2667
2748
  else {
2668
2749
  dispatched.push(result);
@@ -2740,7 +2821,11 @@ export class FactoryLoop {
2740
2821
  error: describeError(error).errorMessage,
2741
2822
  });
2742
2823
  }
2743
- recordSkip({ issue: issueRef(issue), reason: perItemDispatchSkipReason(error) });
2824
+ recordSkip({
2825
+ issue: issueRef(issue),
2826
+ reason: perItemDispatchSkipReason(error),
2827
+ code: 'dispatch-failed',
2828
+ });
2744
2829
  continue;
2745
2830
  }
2746
2831
  finally {
@@ -2748,7 +2833,15 @@ export class FactoryLoop {
2748
2833
  this.#reconciledGithubInProgress.delete(recoveredIdentity);
2749
2834
  }
2750
2835
  }
2751
- report = { pulled, triaged, dispatched, skipped, dryRun, slackDegraded: this.#slackDegraded };
2836
+ report = {
2837
+ pulled,
2838
+ triaged,
2839
+ dispatched,
2840
+ skipped,
2841
+ dryRun,
2842
+ slackDegraded: this.#slackDegraded,
2843
+ ...(orphanRecoveryDegraded ? { orphanRecoveryDegraded } : {}),
2844
+ };
2752
2845
  return report;
2753
2846
  }
2754
2847
  catch (error) {
@@ -2769,6 +2862,7 @@ export class FactoryLoop {
2769
2862
  dispatched: report.dispatched.length,
2770
2863
  skipped: report.skipped.length,
2771
2864
  slackDegraded: report.slackDegraded ?? false,
2865
+ orphanRecoveryDegraded: report.orphanRecoveryDegraded,
2772
2866
  relayfileWaitWarnings: (this.#counters.relayfileOperationWaitWarnings ?? 0) - relayfileWaitWarningsAtStart,
2773
2867
  relayfileSlowOperations: (this.#counters.relayfileSlowOperations ?? 0) - relayfileSlowOperationsAtStart,
2774
2868
  relayfileOperationFailures: (this.#counters.relayfileOperationFailures ?? 0) - relayfileOperationFailuresAtStart,
@@ -3039,7 +3133,7 @@ export class FactoryLoop {
3039
3133
  : { available: true, highWatermark };
3040
3134
  }
3041
3135
  catch (error) {
3042
- if (relayfileOverload(error))
3136
+ if (isPassWideRelayfileFault(error))
3043
3137
  throw error;
3044
3138
  this.#increment('discoveryHighWatermarkFailures');
3045
3139
  this.#logger.warn?.('[factory] discovery high-watermark unavailable; using a full sweep without caching', {
@@ -3062,7 +3156,7 @@ export class FactoryLoop {
3062
3156
  events = [...page.events].sort(compareDiscoveryEvents);
3063
3157
  }
3064
3158
  catch (error) {
3065
- if (relayfileOverload(error))
3159
+ if (isPassWideRelayfileFault(error))
3066
3160
  throw error;
3067
3161
  this.#logger.warn?.('[factory] discovery change feed unavailable; refreshing tree prefixes once', {
3068
3162
  error: describeError(error).errorMessage,
@@ -3124,7 +3218,34 @@ export class FactoryLoop {
3124
3218
  }
3125
3219
  }
3126
3220
  }
3127
- async #githubOrphanRecoveryContext() {
3221
+ /**
3222
+ * The safety context orphan recovery needs before it may release a leaked
3223
+ * `factory:in-progress` claim: who is online, which issues are actively
3224
+ * owned, and which durable lifecycle rows look abandoned.
3225
+ *
3226
+ * A dry run never builds it. `#reconcileOrphanedGithubInProgress` refuses to
3227
+ * release a claim under `dryRun` before it so much as looks at the context,
3228
+ * and `mayRecoverGithubOrphan` in `#performRunOnce` is itself `!dryRun` —
3229
+ * so the context was already provably unused on that path. Gathering it
3230
+ * anyway was not free: `fleet.roster()` mints this process's workspace
3231
+ * identity on demand, which is exactly what a read-only client refuses
3232
+ * (#343) and exactly what a dry run must not need. Deciding what a sweep
3233
+ * WOULD do must not require an identity to do it with.
3234
+ *
3235
+ * Returning `undefined` is therefore the honest answer for a dry run, not a
3236
+ * failure: the caller reports the sweep as `orphanRecoveryDegraded:
3237
+ * 'dry-run'` and preserves in-progress issues, which is what a dry run does
3238
+ * regardless. The failure counter and the warn below stay for the live path,
3239
+ * where an absent context IS a degradation.
3240
+ */
3241
+ async #githubOrphanRecoveryContext(dryRun) {
3242
+ if (dryRun) {
3243
+ this.#increment('githubOrphanRecoveryContextSkippedDryRun');
3244
+ this.#logger.info?.('[factory] dry run skipped the orphan-recovery safety context; in-progress issues are preserved', {
3245
+ reason: 'a dry run never releases an in-progress claim, so it needs no workspace identity to build one',
3246
+ });
3247
+ return undefined;
3248
+ }
3128
3249
  try {
3129
3250
  const [registry, roster, lifecycles, waitingClarifications] = await Promise.all([
3130
3251
  readFactoryInFlightRegistry(this.#config.loop.registryPath),
@@ -3701,7 +3822,17 @@ export class FactoryLoop {
3701
3822
  progressTimer.unref?.();
3702
3823
  }
3703
3824
  try {
3704
- const result = await fn();
3825
+ // The bound the 2026-08-23 wedge needed (#351). `#readinessReconcileTimeoutMs`
3826
+ // is 90 minutes and rejects only the WAIT — the sweep keeps its discovery
3827
+ // lease and the next cycle coalesces onto it — so a call that never
3828
+ // returns stopped dispatch for as long as the process lived. Bounding the
3829
+ // call instead makes the pass unwind, which releases the lease and lets
3830
+ // the next cycle actually start.
3831
+ //
3832
+ // Slightly above the transport's own deadline so the mount's cancellation
3833
+ // wins the race and reports the more precise error; this is the backstop
3834
+ // for mounts that cannot honour a signal.
3835
+ const result = await withRelayfileCallDeadline(operation, details.phase, this.#relayfileOperationBackstopMs(), fn);
3705
3836
  const elapsedMs = this.#elapsedSince(startedAtMs);
3706
3837
  const count = opts.count?.(result);
3707
3838
  if (opts.logComplete) {
@@ -3758,7 +3889,11 @@ export class FactoryLoop {
3758
3889
  error: describeError(error).errorMessage,
3759
3890
  });
3760
3891
  }
3761
- throw error;
3892
+ // The transport deadline is meant to win the race above, and the mount
3893
+ // does not know which phase it was serving. Without this, `lastError`
3894
+ // would name the call but not the context — one of many list/read sites
3895
+ // (codex on #354).
3896
+ throw relayfileTimeoutWithPhase(error, details.phase);
3762
3897
  }
3763
3898
  finally {
3764
3899
  if (progressTimer) {
@@ -3766,6 +3901,19 @@ export class FactoryLoop {
3766
3901
  }
3767
3902
  }
3768
3903
  }
3904
+ /**
3905
+ * The cycle-level backstop budget: the transport deadline plus a margin.
3906
+ *
3907
+ * Proportional rather than a flat grace so it holds at both ends of the
3908
+ * range — a five-minute production budget and a millisecond test budget both
3909
+ * leave the transport room to fire first.
3910
+ */
3911
+ #relayfileOperationBackstopMs() {
3912
+ const timeoutMs = this.#relayfileOperationTimeoutMs;
3913
+ if (!Number.isFinite(timeoutMs) || timeoutMs <= 0)
3914
+ return undefined;
3915
+ return Math.ceil(timeoutMs * RELAYFILE_OPERATION_BACKSTOP_RATIO);
3916
+ }
3769
3917
  #elapsedSince(startedAtMs) {
3770
3918
  return Math.max(0, this.#clock.now() - startedAtMs);
3771
3919
  }
@@ -3914,7 +4062,7 @@ export class FactoryLoop {
3914
4062
  }
3915
4063
  const blockReason = await this.#dispatchBlockReason(decision.issue);
3916
4064
  if (blockReason) {
3917
- const error = new Error(`Refusing to dispatch ${decision.issue.key}: ${blockReason}`);
4065
+ const error = new Error(`Refusing to dispatch ${decision.issue.key}: ${blockReason.reason}`);
3918
4066
  this.#error(error, decision.issue);
3919
4067
  throw error;
3920
4068
  }
@@ -4366,6 +4514,23 @@ export class FactoryLoop {
4366
4514
  }
4367
4515
  return { state: 'starting' };
4368
4516
  }
4517
+ /**
4518
+ * Snapshot a settled sweep's counts onto the readiness surface (#355).
4519
+ *
4520
+ * Only successful passes reach here: a pass that threw has no report, and
4521
+ * inventing zeroes for it would publish "found nothing" for a sweep that
4522
+ * never got to look. The previous pass's numbers stay put instead, dated by
4523
+ * `lastCompletedAtMs`, which is the honest reading.
4524
+ */
4525
+ #recordReadinessSweepOutcome(report) {
4526
+ this.#readinessReconcileLastSweep = {
4527
+ candidates: report.pulled.length,
4528
+ dispatched: report.dispatched.length,
4529
+ skipped: report.skipped.length,
4530
+ skipReasons: factorySweepSkipReasonCounts(report.skipped),
4531
+ ...(report.discoveryDeferred ? { discoveryDeferred: report.discoveryDeferred } : {}),
4532
+ };
4533
+ }
4369
4534
  #readinessReconcileStatus() {
4370
4535
  const consecutiveFailures = this.#readinessReconcileConsecutiveFailures;
4371
4536
  // #296 owns the numerator here, #295/#300 own the derivation. The earliest
@@ -4440,6 +4605,22 @@ export class FactoryLoop {
4440
4605
  ...(this.#readinessReconcileLastFailureAtMs !== undefined
4441
4606
  ? { lastFailureAtMs: this.#readinessReconcileLastFailureAtMs }
4442
4607
  : {}),
4608
+ // Spread whole or not at all: see `#readinessReconcileLastSweep`. Zeroes
4609
+ // are published, which is the entire point — an absent `candidates` and
4610
+ // a zero `candidates` are different diagnoses (#355).
4611
+ ...(this.#readinessReconcileLastSweep
4612
+ ? {
4613
+ candidates: this.#readinessReconcileLastSweep.candidates,
4614
+ dispatched: this.#readinessReconcileLastSweep.dispatched,
4615
+ skipped: this.#readinessReconcileLastSweep.skipped,
4616
+ ...(Object.keys(this.#readinessReconcileLastSweep.skipReasons).length > 0
4617
+ ? { skipReasons: { ...this.#readinessReconcileLastSweep.skipReasons } }
4618
+ : {}),
4619
+ ...(this.#readinessReconcileLastSweep.discoveryDeferred
4620
+ ? { discoveryDeferred: this.#readinessReconcileLastSweep.discoveryDeferred }
4621
+ : {}),
4622
+ }
4623
+ : {}),
4443
4624
  ...(this.#readinessReconcileLastError ? { lastError: this.#readinessReconcileLastError } : {}),
4444
4625
  ...(this.#readinessReconcileLastErrorClass
4445
4626
  ? { lastErrorClass: this.#readinessReconcileLastErrorClass }
@@ -6571,7 +6752,7 @@ export class FactoryLoop {
6571
6752
  return [...issuePaths.values()].sort();
6572
6753
  }
6573
6754
  catch (error) {
6574
- if (relayfileOverload(error))
6755
+ if (isPassWideRelayfileFault(error))
6575
6756
  throw error;
6576
6757
  this.#githubIssuePathIndexReady = false;
6577
6758
  this.#increment('githubIssueListFailures');
@@ -6587,7 +6768,7 @@ export class FactoryLoop {
6587
6768
  parsed = parseJsonContent(content);
6588
6769
  }
6589
6770
  catch (error) {
6590
- if (relayfileOverload(error))
6771
+ if (isPassWideRelayfileFault(error))
6591
6772
  throw error;
6592
6773
  return undefined;
6593
6774
  }
@@ -6684,7 +6865,7 @@ export class FactoryLoop {
6684
6865
  this.#increment('githubIssueMirrorsCreated');
6685
6866
  }
6686
6867
  catch (error) {
6687
- if (relayfileOverload(error))
6868
+ if (isPassWideRelayfileFault(error))
6688
6869
  throw error;
6689
6870
  this.#logger.error?.('[factory] failed to ingest GitHub issue', error);
6690
6871
  }
@@ -7134,23 +7315,33 @@ export class FactoryLoop {
7134
7315
  }
7135
7316
  }
7136
7317
  }
7318
+ /**
7319
+ * Why durable dispatch state refuses this issue, or `undefined` to proceed.
7320
+ *
7321
+ * Returns the operator text *and* its #355 code together rather than letting
7322
+ * the sweep re-derive one from the other: these four conditions are the ones
7323
+ * most likely to explain a sweep that saw eligible issues and dispatched
7324
+ * nothing, and two of them (`terminal`, `retry-limit`) never clear on their
7325
+ * own. Deriving the code by matching the message would put that distinction
7326
+ * one rename away from collapsing into `other`.
7327
+ */
7137
7328
  async #dispatchBlockReason(issue) {
7138
7329
  const key = issueStateKey(issue);
7139
7330
  const state = await this.#state.getDispatchAttempts(this.#workspaceId, key);
7140
7331
  if (!state)
7141
7332
  return undefined;
7142
7333
  if (state.terminal)
7143
- return 'dispatch already terminal';
7334
+ return { reason: 'dispatch already terminal', code: 'dispatch-terminal' };
7144
7335
  if (state.inFlight)
7145
- return 'dispatch already in-flight';
7336
+ return { reason: 'dispatch already in-flight', code: 'dispatch-in-flight' };
7146
7337
  const now = this.#clock.now();
7147
7338
  if (state.backoffUntilMs > now) {
7148
- return 'dispatch backoff active';
7339
+ return { reason: 'dispatch backoff active', code: 'dispatch-backoff' };
7149
7340
  }
7150
7341
  if (state.attempts >= this.#config.dispatch.maxAttempts) {
7151
7342
  state.terminal = true;
7152
7343
  await this.#state.recordDispatchAttempt(this.#workspaceId, key, state);
7153
- return 'dispatch retry limit reached';
7344
+ return { reason: 'dispatch retry limit reached', code: 'dispatch-retry-limit' };
7154
7345
  }
7155
7346
  return undefined;
7156
7347
  }
@@ -7488,7 +7679,7 @@ export class FactoryLoop {
7488
7679
  return resolvedIssue;
7489
7680
  }
7490
7681
  catch (error) {
7491
- if (relayfileOverload(error))
7682
+ if (isPassWideRelayfileFault(error))
7492
7683
  throw error;
7493
7684
  if (isMissingIssueFileError(error) && isIssuePathUnderRoot(path)) {
7494
7685
  this.#increment('phantomSkipped');
@@ -8584,7 +8775,7 @@ export class FactoryLoop {
8584
8775
  paths = await this.#listRelayfileTree(root, 'exact-head PR confirmation');
8585
8776
  }
8586
8777
  catch (error) {
8587
- if (relayfileOverload(error))
8778
+ if (isPassWideRelayfileFault(error))
8588
8779
  throw error;
8589
8780
  continue;
8590
8781
  }
@@ -8854,7 +9045,7 @@ export class FactoryLoop {
8854
9045
  return await this.#listRelayfileTree(root, 'published PR confirmation');
8855
9046
  }
8856
9047
  catch (error) {
8857
- if (relayfileOverload(error))
9048
+ if (isPassWideRelayfileFault(error))
8858
9049
  throw error;
8859
9050
  return [];
8860
9051
  }
@@ -10436,7 +10627,7 @@ export class FactoryLoop {
10436
10627
  }
10437
10628
  }
10438
10629
  catch (error) {
10439
- if (relayfileOverload(error))
10630
+ if (isPassWideRelayfileFault(error))
10440
10631
  throw error;
10441
10632
  this.#logger.warn?.('[factory] unable to list GitHub issue comments for replay', { prefix, error });
10442
10633
  }
@@ -12982,7 +13173,7 @@ export class FactoryLoop {
12982
13173
  found.push(...tree.filter((path) => path.endsWith('.json') && numberSegment.test(path)));
12983
13174
  }
12984
13175
  catch (error) {
12985
- if (relayfileOverload(error))
13176
+ if (isPassWideRelayfileFault(error))
12986
13177
  throw error;
12987
13178
  // try the next root
12988
13179
  }
@@ -14161,7 +14352,7 @@ export class FactoryLoop {
14161
14352
  return await this.#listRelayfileTree(prefix, 'Slack identity lookup');
14162
14353
  }
14163
14354
  catch (error) {
14164
- if (relayfileOverload(error))
14355
+ if (isPassWideRelayfileFault(error))
14165
14356
  throw error;
14166
14357
  return [];
14167
14358
  }
@@ -17088,7 +17279,7 @@ const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, listTree
17088
17279
  paths.add(path);
17089
17280
  }
17090
17281
  catch (error) {
17091
- if (relayfileOverload(error))
17282
+ if (isPassWideRelayfileFault(error))
17092
17283
  throw error;
17093
17284
  listErrors.push(error);
17094
17285
  }