@agent-relay/factory 0.1.64 → 0.1.66

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/README.md +9 -0
  2. package/dist/cli/diagnose.d.ts +92 -0
  3. package/dist/cli/diagnose.d.ts.map +1 -0
  4. package/dist/cli/diagnose.js +381 -0
  5. package/dist/cli/diagnose.js.map +1 -0
  6. package/dist/cli/fleet.d.ts +9 -1
  7. package/dist/cli/fleet.d.ts.map +1 -1
  8. package/dist/cli/fleet.js +97 -4
  9. package/dist/cli/fleet.js.map +1 -1
  10. package/dist/config/schema.d.ts +134 -2
  11. package/dist/config/schema.d.ts.map +1 -1
  12. package/dist/config/schema.js +60 -0
  13. package/dist/config/schema.js.map +1 -1
  14. package/dist/fleet/internal-fleet-client.d.ts +16 -0
  15. package/dist/fleet/internal-fleet-client.d.ts.map +1 -1
  16. package/dist/fleet/internal-fleet-client.js +228 -13
  17. package/dist/fleet/internal-fleet-client.js.map +1 -1
  18. package/dist/hosted/orchestrator.d.ts.map +1 -1
  19. package/dist/hosted/orchestrator.js +1 -4
  20. package/dist/hosted/orchestrator.js.map +1 -1
  21. package/dist/index.d.ts +1 -1
  22. package/dist/index.d.ts.map +1 -1
  23. package/dist/index.js +1 -1
  24. package/dist/index.js.map +1 -1
  25. package/dist/observability/error-class.d.ts +29 -0
  26. package/dist/observability/error-class.d.ts.map +1 -0
  27. package/dist/observability/error-class.js +36 -0
  28. package/dist/observability/error-class.js.map +1 -0
  29. package/dist/observability/index.d.ts +1 -0
  30. package/dist/observability/index.d.ts.map +1 -1
  31. package/dist/observability/index.js +1 -0
  32. package/dist/observability/index.js.map +1 -1
  33. package/dist/orchestrator/batch-tracker.d.ts +7 -0
  34. package/dist/orchestrator/batch-tracker.d.ts.map +1 -1
  35. package/dist/orchestrator/batch-tracker.js +1 -0
  36. package/dist/orchestrator/batch-tracker.js.map +1 -1
  37. package/dist/orchestrator/factory.d.ts +18 -0
  38. package/dist/orchestrator/factory.d.ts.map +1 -1
  39. package/dist/orchestrator/factory.js +1059 -82
  40. package/dist/orchestrator/factory.js.map +1 -1
  41. package/dist/orchestrator/index.d.ts +1 -0
  42. package/dist/orchestrator/index.d.ts.map +1 -1
  43. package/dist/orchestrator/index.js +1 -0
  44. package/dist/orchestrator/index.js.map +1 -1
  45. package/dist/orchestrator/public-health.d.ts +82 -0
  46. package/dist/orchestrator/public-health.d.ts.map +1 -0
  47. package/dist/orchestrator/public-health.js +491 -0
  48. package/dist/orchestrator/public-health.js.map +1 -0
  49. package/dist/ports/state.d.ts +25 -0
  50. package/dist/ports/state.d.ts.map +1 -1
  51. package/dist/state/dispatch-lifecycle-slot.d.ts +45 -0
  52. package/dist/state/dispatch-lifecycle-slot.d.ts.map +1 -0
  53. package/dist/state/dispatch-lifecycle-slot.js +67 -0
  54. package/dist/state/dispatch-lifecycle-slot.js.map +1 -0
  55. package/dist/state/file-state-store.d.ts +8 -0
  56. package/dist/state/file-state-store.d.ts.map +1 -1
  57. package/dist/state/file-state-store.js +19 -19
  58. package/dist/state/file-state-store.js.map +1 -1
  59. package/dist/state/in-memory-state-store.d.ts +8 -0
  60. package/dist/state/in-memory-state-store.d.ts.map +1 -1
  61. package/dist/state/in-memory-state-store.js +18 -19
  62. package/dist/state/in-memory-state-store.js.map +1 -1
  63. package/dist/state/watch-state-document.js +1 -0
  64. package/dist/state/watch-state-document.js.map +1 -1
  65. package/dist/triage/schema.d.ts +14 -14
  66. package/dist/types.d.ts +180 -1
  67. package/dist/types.d.ts.map +1 -1
  68. package/package.json +1 -1
@@ -1,7 +1,7 @@
1
1
  import { randomUUID } from 'node:crypto';
2
2
  import { mkdir, readFile, writeFile } from 'node:fs/promises';
3
3
  import { dirname, isAbsolute, resolve } from 'node:path';
4
- import { FactoryConfigSchema } from '../config/schema.js';
4
+ import { DEFAULT_READINESS_RECONCILE_TIMEOUT_MS, FactoryConfigSchema } from '../config/schema.js';
5
5
  import { linearByStatePath, linearByIdPath, linearByUuidPath } from '../constants/linear.js';
6
6
  import { stateResolutionFromIds } from '../linear/state-resolver.js';
7
7
  import { GithubMergeGate, closeProbePr } from '../github/index.js';
@@ -9,6 +9,7 @@ import { factoryGithubIssueCommentDraftName, isFactoryGithubIssueCommentDraftNam
9
9
  import { VerificationPipeline } from '../environments/verification-pipeline.js';
10
10
  import { factoryWorktreeIssueSlug, factoryWorktreePath } from '../git/agent-worktree.js';
11
11
  import { InMemoryStateStore } from '../state/in-memory-state-store.js';
12
+ import { dispatchHandedOffToBabysitters, dispatchLifecycleOccupiesSlot, dispatchPhaseOccupiesSlot, } from '../state/dispatch-lifecycle-slot.js';
12
13
  import { containsExplicitIssueReference, containsIssueKey, factoryBranchBelongsToIssue } from '../issue-key-match.js';
13
14
  import { normalizeLogger, normalizeLogValue, setSafeErrorStack, stringifyLogValue } from '../logging.js';
14
15
  import { isInFactoryScope } from '../safety/factory-scope.js';
@@ -29,6 +30,8 @@ import { CoalescedTaskQueue } from './coalesced-task-queue.js';
29
30
  import { findAgentProcessByName, readProcessIdentity } from './process-identity.js';
30
31
  import { readFactoryInFlightRegistry, terminatePids } from './reaper.js';
31
32
  import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../observability/events.js';
33
+ import { telemetryErrorClass } from '../observability/error-class.js';
34
+ import { derivedReadinessReconcileState, publicHealthFromHeartbeat, readinessReconcileInFlightMs, } from './public-health.js';
32
35
  import { boundedRunCostTotal, CostLedger } from '../cost/ledger.js';
33
36
  import { createTicketDispatchDelivery } from '../delivery/ticket-dispatch.js';
34
37
  import { canonicalTrajectorySessionRef, renderTrajectoryPointer, stripTrajectoryPointers, } from '../trajectory.js';
@@ -139,9 +142,28 @@ const STOP_TEARDOWN_TIMEOUT_MS = 2_500;
139
142
  const DISPATCH_LIFECYCLE_LEASE_MS = 5 * 60_000;
140
143
  const DISPATCH_LIFECYCLE_RENEW_MS = 60_000;
141
144
  const DISPATCH_LIFECYCLE_RETRY_MS = 1_000;
145
+ /**
146
+ * Ceiling on the durable capacity-wait re-arm (#303).
147
+ *
148
+ * The retry was a flat 1 Hz with no bound at all. When the batch was wedged,
149
+ * every queued issue re-read the shared state document once a second forever —
150
+ * production measured 1477 state GETs in 111 s — and the timers could not even
151
+ * keep up, so they coalesced into a continuous spin against the serialized
152
+ * store. Waiting for capacity is legitimate and must not be abandoned (a real
153
+ * multi-hour run holds the slot honestly), so what is bounded is the *rate*.
154
+ */
155
+ const DISPATCH_LIFECYCLE_RETRY_MAX_MS = 30_000;
156
+ /** Rate limit for the capacity-wait warning once the backoff has capped. */
157
+ const DISPATCH_LIFECYCLE_CAPACITY_WAIT_LOG_MS = 60_000;
142
158
  const DISPATCH_WRITEBACK_MAX_ATTEMPTS = 3;
143
159
  const DISPATCH_WRITEBACK_RETRY_MS = 250;
144
160
  const HELD_PAST_DEADLINE_RELEASE_REASON = 'held-past-deadline';
161
+ /**
162
+ * Release reason for a lifecycle that took a batch slot and never placed an
163
+ * agent (#303). Deliberately distinct from `held-past-deadline`: that one
164
+ * means a team ran and never finished, this one means no team ever existed.
165
+ */
166
+ const AGENTLESS_SLOT_PAST_DEADLINE_RELEASE_REASON = 'agentless-slot-past-deadline';
145
167
  const HELD_DEADLINE_OVERDUE_RETRY_MS = 1_000;
146
168
  const STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS = 30_000;
147
169
  const RECONCILED_AGENT_EXIT_CONCURRENCY = 4;
@@ -185,6 +207,38 @@ const DISCOVERY_SWEEP_RENEW_MS = 30_000;
185
207
  const READINESS_RECONCILE_FAILURE_THRESHOLD = 3;
186
208
  const DISCOVERY_CHANGE_EVENT_LIMIT = 1_000;
187
209
  const DISCOVERY_OVERLOAD_BACKOFF_MAX_MS = 5 * 60_000;
210
+ /** First rung of the ladder when the 429 advertises no `Retry-After`. */
211
+ const DISCOVERY_OVERLOAD_BACKOFF_BASE_MS = 5_000;
212
+ /**
213
+ * Floor for the advertised delay. `Retry-After: 0` would otherwise pin the
214
+ * whole ladder at zero (`0 * 2 ** n` is still zero) and turn respecting the
215
+ * dependency into hammering it.
216
+ */
217
+ const DISCOVERY_OVERLOAD_BACKOFF_MIN_MS = 1_000;
218
+ /**
219
+ * Ceiling for the ladder once the dependency has told us how long to wait.
220
+ *
221
+ * relayfile sheds an overloaded workspace DO with a 429 in milliseconds
222
+ * carrying `Retry-After: 5`, and #297 is what happened when the ladder ignored
223
+ * that and climbed to `DISCOVERY_OVERLOAD_BACKOFF_MAX_MS` anyway: the
224
+ * dependency asked for five seconds, Factory slept for five minutes, probed
225
+ * for recovery once per cap-length window, and presented a transient upstream
226
+ * blip as a sustained outage. The ratchet still escalates — it is just bounded
227
+ * by roughly what was actually asked for, and never *below* it, so this is a
228
+ * ceiling and not a licence to retry sooner than the dependency allows.
229
+ */
230
+ const DISCOVERY_OVERLOAD_ADVERTISED_BACKOFF_MAX_MS = 30_000;
231
+ /**
232
+ * How many relayfile operations one sweep may have shed before the sweep is
233
+ * abandoned rather than continued.
234
+ *
235
+ * Per-item overload skips that item and keeps going (#297, the same principle
236
+ * as #292), but skipping must not degenerate into grinding a shedding
237
+ * dependency through an entire backlog one 429 at a time. Past this many, the
238
+ * dependency is not serving this sweep at all: abort, back off, and let the
239
+ * ratchet do its job.
240
+ */
241
+ const DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT = 5;
188
242
  const GITHUB_FACTORY_LABEL = 'factory';
189
243
  const GITHUB_LIFECYCLE_LABELS = new Set(['factory:in-progress', 'factory:human-review']);
190
244
  const GITHUB_MIRROR_TITLE_PREFIX = '[factory]';
@@ -259,6 +313,20 @@ class DispatchLifecycleClaimRefusedError extends Error {
259
313
  this.name = 'DispatchLifecycleClaimRefusedError';
260
314
  }
261
315
  }
316
+ /**
317
+ * The deadline that makes a hung sweep reachable by the existing recovery path.
318
+ * Its message is fully internal (one integer), so it is safe to persist into
319
+ * the operator-facing `readinessReconcile.lastError`.
320
+ */
321
+ class ReadinessReconcileTimeoutError extends Error {
322
+ timeoutMs;
323
+ code = 'FACTORY_READINESS_RECONCILE_TIMEOUT';
324
+ constructor(timeoutMs) {
325
+ super(`readiness reconcile sweep exceeded its ${timeoutMs}ms deadline`);
326
+ this.timeoutMs = timeoutMs;
327
+ this.name = 'ReadinessReconcileTimeoutError';
328
+ }
329
+ }
262
330
  const realClock = {
263
331
  now: () => Date.now(),
264
332
  sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
@@ -369,7 +437,17 @@ export class FactoryLoop {
369
437
  #dispatchLifecycleRetryTimers = new Map();
370
438
  #dispatchLifecycleDrives = new Set();
371
439
  #abandonedDispatchReasons = new Map();
372
- #dispatchLifecycleCapacityWaitLogged = new Set();
440
+ /**
441
+ * Live batch-capacity waits, keyed by issue (#303).
442
+ *
443
+ * Replaces a `Set` of "already logged" keys. That set made the wait a
444
+ * one-shot log and nothing else: after the first line, an outage in which
445
+ * every issue was stuck behind a wedged slot was indistinguishable from an
446
+ * idle Factory on every operator surface. The wait now carries its own start
447
+ * instant and attempt count, which is what both the escalating warning and
448
+ * `status().dispatchCapacity` are derived from.
449
+ */
450
+ #dispatchLifecycleCapacityWaits = new Map();
373
451
  #dispatchLifecycleOwnershipWaitLogged = new Set();
374
452
  #dispatchClaimStatuses = new Map();
375
453
  #localReleaseCheckpoints = new Map();
@@ -408,12 +486,41 @@ export class FactoryLoop {
408
486
  #readinessReconcileTimer;
409
487
  #readinessReconcileInFlight;
410
488
  #readinessReconcileIntervalMs = 60_000;
489
+ #readinessReconcileTimeoutMs = DEFAULT_READINESS_RECONCILE_TIMEOUT_MS;
490
+ // Set for exactly as long as a sweep is running. `state` is derived from
491
+ // this, so an in-flight pass can no longer masquerade as the last settled one.
492
+ #readinessReconcileInFlightSinceMs;
493
+ /**
494
+ * The work a deadline gave up waiting on. The deadline bounds the wait, not
495
+ * the sweep, so this is still live: shutdown has to drain it, and `state` has
496
+ * to keep counting from when it actually started.
497
+ *
498
+ * Two fields rather than a collection, because every live abandoned wait
499
+ * converges on the same sweep. They are all `runOnce()` calls with the same
500
+ * `dryRun`, so whatever they are queued behind, the first one out starts the
501
+ * sweep and the rest coalesce onto it — they settle together. So the newest
502
+ * wait is a sufficient drain target, and the earliest start is the honest
503
+ * age. Both matter (#301 review): keeping only the newest start advanced the
504
+ * age by two intervals every two intervals, so at `reconcileTimeoutMs ===
505
+ * reconcileIntervalMs` it never reached three and `stalled` was never
506
+ * reported; keeping one record per wait grew without bound in exactly the
507
+ * never-settling case this change exists for.
508
+ *
509
+ * The wait, deliberately, and never `#runOnceInFlight`: a mismatched-`dryRun`
510
+ * sweep is waited BEHIND rather than coalesced onto, so that handle can name
511
+ * an unrelated sweep, and the readiness pass would then start its own work
512
+ * after shutdown believed it had drained everything. The wait covers the
513
+ * queueing and the sweep it eventually runs, in every branch.
514
+ */
515
+ #readinessReconcileAbandonedWait;
516
+ #readinessReconcileAbandonedSinceMs;
411
517
  #readinessReconcileConsecutiveFailures = 0;
412
518
  #readinessReconcileLastDurationMs;
413
519
  #readinessReconcileLastStartedAtMs;
414
520
  #readinessReconcileLastCompletedAtMs;
415
521
  #readinessReconcileLastFailureAtMs;
416
522
  #readinessReconcileLastError;
523
+ #readinessReconcileLastErrorClass;
417
524
  #liveEventQueue = [];
418
525
  #liveEventDrainScheduled = false;
419
526
  #liveEventDrainActive = false;
@@ -501,7 +608,34 @@ export class FactoryLoop {
501
608
  // so the loop's catch no longer runs the failure-handoff reaper for it; the
502
609
  // pass reaps inline and must write to the same paths runLoop would.
503
610
  #loopReapPaths;
611
+ /**
612
+ * The first 429 relayfile raised during this sweep, kept for its
613
+ * `Retry-After` and reason when the sweep decides how long to back off.
614
+ *
615
+ * Before #297 this doubled as a sweep-wide kill switch: any 429 from any
616
+ * relayfile call latched here and `#runOnceWithDiscoveryFence` then threw it
617
+ * away along with everything the sweep had already accomplished. It is now
618
+ * only evidence, never a verdict — see `#discoverySweepOverloads` for the
619
+ * fuse that still ends a sweep the dependency is genuinely refusing to serve.
620
+ */
504
621
  #discoveryOverloadError;
622
+ /** Relayfile operations this sweep has been shed on. */
623
+ #discoverySweepOverloads = 0;
624
+ /**
625
+ * The longest `Retry-After` any operation in this sweep advertised.
626
+ *
627
+ * `#discoveryOverloadError` latches the FIRST 429, so deriving the backoff
628
+ * from it alone would let the durable window expire before a later, longer
629
+ * advertised delay permits — breaking the very guarantee #297 is about. The
630
+ * backoff takes the maximum instead.
631
+ */
632
+ #discoverySweepRetryAfterSeconds;
633
+ /**
634
+ * Whether relayfile served at least one ready work unit end to end during
635
+ * this sweep — its issue read, or a dispatch built on it. This is what
636
+ * decays the durable overload ratchet; see `#discoveryOverloadOutcome`.
637
+ */
638
+ #discoverySweepProgress = false;
505
639
  #resolvedIssueSource;
506
640
  #integrationInstructions;
507
641
  #integrationInstructionsRefresh;
@@ -906,6 +1040,7 @@ export class FactoryLoop {
906
1040
  clearTimeout(timer);
907
1041
  this.#dispatchLifecycleRetryTimers.clear();
908
1042
  this.#abandonedDispatchReasons.clear();
1043
+ this.#dispatchLifecycleCapacityWaits.clear();
909
1044
  this.#dispatchLifecycleOwnershipWaitLogged.clear();
910
1045
  if (this.#completionSweepTimer)
911
1046
  clearTimeout(this.#completionSweepTimer);
@@ -917,6 +1052,14 @@ export class FactoryLoop {
917
1052
  clearTimeout(this.#previewSweepTimer);
918
1053
  this.#previewSweepTimer = undefined;
919
1054
  await this.#readinessReconcileInFlight;
1055
+ // #301 review: the deadline ends the *wait*, so `#readinessReconcileInFlight`
1056
+ // can settle with its `runOnce()` still live. Shutdown releases dispatch
1057
+ // lifecycle leases and disposes ports below, and `#isPassFatalFailure` only
1058
+ // fences a stopping sweep once something in it throws — so a sweep whose
1059
+ // dependency recovers cleanly would otherwise dispatch through torn-down
1060
+ // state. Draining here restores exactly the pre-deadline shutdown contract:
1061
+ // stop() outlives the sweep it started.
1062
+ await this.#readinessReconcileAbandonedWait;
920
1063
  await this.#previewSweepInFlight;
921
1064
  this.#stoppingHeartbeatRefreshActive = await this.#stopLiveHeartbeat('stopping');
922
1065
  try {
@@ -1067,6 +1210,9 @@ export class FactoryLoop {
1067
1210
  const options = this.#liveOptions(overrides);
1068
1211
  this.#liveTransport = options.transport;
1069
1212
  this.#readinessReconcileIntervalMs = options.reconcileIntervalMs;
1213
+ // `start()` overrides skip the schema's cross-field check, so re-apply its
1214
+ // floor here: a deadline under one interval would kill every pass.
1215
+ this.#readinessReconcileTimeoutMs = Math.max(options.reconcileTimeoutMs, options.reconcileIntervalMs);
1070
1216
  this.#liveConnectStartedAtMs = this.#clock.now();
1071
1217
  this.#liveReplaySkewMarginMs = options.replaySkewMarginMs;
1072
1218
  const highWatermark = await this.#currentEventHighWatermark();
@@ -1103,10 +1249,27 @@ export class FactoryLoop {
1103
1249
  this.#logger.info?.('[factory] running startup ready-issue backfill before draining buffered events', {
1104
1250
  highWatermarkRouteUnavailable: highWatermark.routeUnavailable,
1105
1251
  });
1252
+ // Review follow-up on #300 (P1, cubic). The startup backfill is a
1253
+ // discovery pass like any other, and it is the one most likely to hang:
1254
+ // #36 measured 61 minutes here while the Relayfile mirror hydrated on a
1255
+ // cold container. Stamping it means a wedged FIRST pass is visible as
1256
+ // in-flight, instead of leaving the timestamps empty and the derived
1257
+ // state reading `healthy` forever.
1258
+ //
1259
+ // Only the timestamps. `consecutiveFailures` and `lastError` belong to
1260
+ // the reconcile loop's own failure accounting, which owns the degraded
1261
+ // threshold and the #297 reason allowlist; a startup failure is already
1262
+ // counted by `liveStartupBackfillErrors` and reported through `#error`.
1263
+ const backfillStartedAtMs = this.#clock.now();
1264
+ this.#readinessReconcileLastStartedAtMs = backfillStartedAtMs;
1106
1265
  try {
1107
1266
  await this.runOnce();
1267
+ this.#readinessReconcileLastDurationMs = this.#elapsedSince(backfillStartedAtMs);
1268
+ this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
1108
1269
  }
1109
1270
  catch (error) {
1271
+ this.#readinessReconcileLastDurationMs = this.#elapsedSince(backfillStartedAtMs);
1272
+ this.#readinessReconcileLastFailureAtMs = this.#clock.now();
1110
1273
  // A startup backfill failure must not abort the daemon: log it and fall
1111
1274
  // back to the live event stream (plus any buffered events) instead of
1112
1275
  // leaving the factory down.
@@ -1198,6 +1361,7 @@ export class FactoryLoop {
1198
1361
  eventLimit: overrides.eventLimit ?? this.#config.liveSubscription.eventLimit,
1199
1362
  replaySkewMarginMs: overrides.replaySkewMarginMs ?? this.#config.liveSubscription.replaySkewMarginMs,
1200
1363
  reconcileIntervalMs: overrides.reconcileIntervalMs ?? this.#config.liveSubscription.reconcileIntervalMs,
1364
+ reconcileTimeoutMs: overrides.reconcileTimeoutMs ?? this.#config.liveSubscription.reconcileTimeoutMs,
1201
1365
  };
1202
1366
  }
1203
1367
  async #currentEventCursor(limit) {
@@ -1246,19 +1410,82 @@ export class FactoryLoop {
1246
1410
  }, delayMs);
1247
1411
  this.#readinessReconcileTimer.unref?.();
1248
1412
  }
1413
+ /**
1414
+ * Runs one sweep under a deadline (#296).
1415
+ *
1416
+ * The sweep itself cannot be cancelled — `runOnce()` owns a durable discovery
1417
+ * lease and abandoning it mid-flight is not safe — so expiry rejects *this*
1418
+ * wait and leaves the underlying pass to finish on its own. That is enough:
1419
+ * the rejection is what reaches the failure path, which re-arms the timer.
1420
+ * A later reconcile pass coalesces onto the still-running `runOnce()` and
1421
+ * fails on its own deadline too, so a persistent hang keeps counting up to
1422
+ * `degraded` instead of going quiet.
1423
+ */
1424
+ async #runOnceWithReadinessDeadline() {
1425
+ const timeoutMs = this.#readinessReconcileTimeoutMs;
1426
+ const startedAtMs = this.#clock.now();
1427
+ const sweep = this.runOnce();
1428
+ let timer;
1429
+ try {
1430
+ return await new Promise((resolve, reject) => {
1431
+ timer = setTimeout(() => {
1432
+ this.#increment('readinessReconcileDeadlineExceeded');
1433
+ if (this.#readinessReconcileAbandonedSinceMs === undefined) {
1434
+ // `state` ages from the FIRST wait that gave up on this work, not
1435
+ // from whenever the latest one began.
1436
+ this.#readinessReconcileAbandonedSinceMs = startedAtMs;
1437
+ // The abandoned pass is still running against the live control
1438
+ // plane. Report where it lands, so an operator can tell a
1439
+ // dependency that recovered late from one that never answered.
1440
+ // Attached once, so a wedge is reported once and not per expiry.
1441
+ void sweep.then((report) => this.#logger.warn?.('[factory] abandoned readiness sweep completed after its deadline', {
1442
+ timeoutMs,
1443
+ overrunMs: this.#elapsedSince(startedAtMs) - timeoutMs,
1444
+ dispatched: report.dispatched.length,
1445
+ }), (error) => this.#logger.warn?.('[factory] abandoned readiness sweep failed after its deadline', {
1446
+ timeoutMs,
1447
+ overrunMs: this.#elapsedSince(startedAtMs) - timeoutMs,
1448
+ error: describeError(error).errorMessage,
1449
+ })).catch(() => undefined);
1450
+ }
1451
+ // Newest wait wins as the drain target: it settles no earlier than
1452
+ // the ones before it, and clearing on it clears them all.
1453
+ const wait = sweep.catch(() => undefined).then(() => {
1454
+ if (this.#readinessReconcileAbandonedWait !== wait)
1455
+ return;
1456
+ this.#readinessReconcileAbandonedWait = undefined;
1457
+ this.#readinessReconcileAbandonedSinceMs = undefined;
1458
+ });
1459
+ this.#readinessReconcileAbandonedWait = wait;
1460
+ reject(new ReadinessReconcileTimeoutError(timeoutMs));
1461
+ }, timeoutMs);
1462
+ timer.unref?.();
1463
+ // Attaching handlers here is also what keeps a late rejection from the
1464
+ // abandoned pass from surfacing as an unhandled rejection.
1465
+ sweep.then(resolve, reject);
1466
+ });
1467
+ }
1468
+ finally {
1469
+ if (timer)
1470
+ clearTimeout(timer);
1471
+ }
1472
+ }
1249
1473
  async #reconcileReadyIssues() {
1250
1474
  const startedAtMs = this.#clock.now();
1251
1475
  this.#readinessReconcileLastStartedAtMs = startedAtMs;
1476
+ this.#readinessReconcileInFlightSinceMs = startedAtMs;
1252
1477
  this.#increment('readinessReconcileSweeps');
1253
1478
  this.#logger.info?.('[factory] periodic readiness reconciliation started', {
1254
1479
  intervalMs: this.#readinessReconcileIntervalMs,
1480
+ timeoutMs: this.#readinessReconcileTimeoutMs,
1255
1481
  });
1256
1482
  try {
1257
- const report = await this.runOnce();
1483
+ const report = await this.#runOnceWithReadinessDeadline();
1258
1484
  this.#readinessReconcileConsecutiveFailures = 0;
1259
1485
  this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
1260
1486
  this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
1261
1487
  this.#readinessReconcileLastError = undefined;
1488
+ this.#readinessReconcileLastErrorClass = undefined;
1262
1489
  this.#logger.info?.('[factory] periodic readiness reconciliation completed', {
1263
1490
  durationMs: this.#readinessReconcileLastDurationMs,
1264
1491
  candidates: report.pulled.length,
@@ -1267,11 +1494,28 @@ export class FactoryLoop {
1267
1494
  });
1268
1495
  }
1269
1496
  catch (error) {
1270
- const errorMessage = describeError(error).errorMessage;
1497
+ // #297: all four relayfile overload reason codes share one message, and
1498
+ // `lastError` is what an operator reads from /evidence. Without the
1499
+ // reason, "workspace durable object is busy" cannot be told apart from
1500
+ // three other conditions with three different remedies.
1501
+ //
1502
+ // Allowlisted, because this is a persisted operator-facing surface and
1503
+ // not just a log line: `lastError` is returned from `status()` and
1504
+ // written into the loop heartbeat file, so an unbounded
1505
+ // dependency-controlled string would land on disk.
1506
+ const overload = relayfileOverload(error);
1507
+ const errorMessage = overload
1508
+ ? `${describeError(error).errorMessage} ` +
1509
+ `[relayfile ${overload.status} ${relayfileOverloadReasonLabel(overload.reason)}` +
1510
+ `${overload.retryAfterSeconds === undefined ? '' : `; retry-after=${overload.retryAfterSeconds}s`}]`
1511
+ : describeError(error).errorMessage;
1271
1512
  this.#readinessReconcileConsecutiveFailures += 1;
1272
1513
  this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
1273
1514
  this.#readinessReconcileLastFailureAtMs = this.#clock.now();
1274
1515
  this.#readinessReconcileLastError = errorMessage;
1516
+ // The class, unlike the message, is publishable: #295 puts it on the
1517
+ // unauthenticated health surface through the same allowlist.
1518
+ this.#readinessReconcileLastErrorClass = telemetryErrorClass(error);
1275
1519
  this.#increment('readinessReconcileErrors');
1276
1520
  this.#logger.warn?.('[factory] periodic readiness reconciliation failed; retry remains scheduled', {
1277
1521
  error: errorMessage,
@@ -1280,6 +1524,11 @@ export class FactoryLoop {
1280
1524
  degraded: this.#readinessReconcileConsecutiveFailures >= READINESS_RECONCILE_FAILURE_THRESHOLD,
1281
1525
  });
1282
1526
  }
1527
+ finally {
1528
+ // Cleared before the heartbeat write below, so a slow-but-successful pass
1529
+ // does not stamp its own tail as `stalled`.
1530
+ this.#readinessReconcileInFlightSinceMs = undefined;
1531
+ }
1283
1532
  await this.#refreshLiveHeartbeat();
1284
1533
  }
1285
1534
  #scheduleLivePoll(delayMs, options) {
@@ -2008,13 +2257,29 @@ export class FactoryLoop {
2008
2257
  this.#discoverySweepStartedAtMs = sweepStartedAtMs;
2009
2258
  this.#discoverySweepLeaseLost = false;
2010
2259
  this.#discoveryOverloadError = undefined;
2260
+ this.#discoverySweepOverloads = 0;
2261
+ this.#discoverySweepRetryAfterSeconds = undefined;
2262
+ this.#discoverySweepProgress = false;
2011
2263
  this.#startDiscoverySweepRenewal(claim.lease.epoch);
2012
2264
  let leaseReleased = false;
2013
2265
  try {
2014
2266
  this.#discoverySession = await this.#prepareDiscoverySession(claim);
2267
+ // #297: a 429 raised anywhere in the sweep used to latch and be rethrown
2268
+ // here, discarding a completed pass — every issue read, every dispatch —
2269
+ // because of one transient shed operation. The work this sweep did is
2270
+ // now kept instead, and the ratchet below records that the dependency is
2271
+ // shedding but still serving.
2015
2272
  const report = await this.#performRunOnce(opts);
2016
- if (this.#discoveryOverloadError)
2273
+ // The exception, and the reason skipping shed units cannot make a sweep
2274
+ // unconditionally green: a sweep that was shed AND got no work unit
2275
+ // through accomplished nothing. There is no progress to preserve, and
2276
+ // committing it would report a clean sweep over a dependency that served
2277
+ // none of it — leaving `readinessReconcile` healthy while Factory
2278
+ // dispatches nothing, which is the #292 wedge wearing the other costume.
2279
+ // Fail it so the ratchet escalates and readiness reflects reality.
2280
+ if (this.#discoveryOverloadError !== undefined && !this.#discoverySweepProgress) {
2017
2281
  throw this.#discoveryOverloadError;
2282
+ }
2018
2283
  const checkpoint = await this.#finalizeDiscoveryCheckpoint();
2019
2284
  // Do not clear the durable lease while a renewal can still be waiting on
2020
2285
  // the same state-file lock. A late renewal that observes the completed
@@ -2024,7 +2289,8 @@ export class FactoryLoop {
2024
2289
  if (this.#discoverySweepLeaseLost) {
2025
2290
  throw new Error('discovery sweep lease was lost before checkpoint commit');
2026
2291
  }
2027
- const completed = await this.#state.completeDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, checkpoint);
2292
+ const residual = this.#discoveryOverloadOutcome(claim.state.consecutiveOverloads, 'committed');
2293
+ const completed = await this.#commitDiscoverySweep(claim.lease.epoch, checkpoint, residual);
2028
2294
  leaseReleased = completed;
2029
2295
  if (!completed)
2030
2296
  throw new Error('discovery sweep lease was lost before completion');
@@ -2040,18 +2306,18 @@ export class FactoryLoop {
2040
2306
  await this.#stopDiscoverySweepRenewal();
2041
2307
  const overload = relayfileOverload(error);
2042
2308
  if (overload) {
2043
- const consecutiveOverloads = claim.state.consecutiveOverloads + 1;
2044
- const delayMs = discoveryOverloadBackoffMs(overload.retryAfterSeconds, consecutiveOverloads);
2045
- const backoffUntilMs = this.#clock.now() + delayMs;
2046
- leaseReleased = await this.#state.deferDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, backoffUntilMs, consecutiveOverloads);
2047
- this.#increment('discoveryOverloadBackoffs');
2309
+ const outcome = this.#discoveryOverloadOutcome(claim.state.consecutiveOverloads, 'aborted', error);
2310
+ leaseReleased = await this.#state.deferDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, outcome.backoffUntilMs, outcome.consecutiveOverloads);
2048
2311
  this.#logger.warn?.('[factory] Relayfile discovery overloaded; backing off before another sweep', {
2049
2312
  status: overload.status,
2050
2313
  reason: overload.reason,
2051
- retryAfterSeconds: overload.retryAfterSeconds,
2052
- delayMs,
2053
- backoffUntilMs,
2054
- consecutiveOverloads,
2314
+ retryAfterSeconds: outcome.retryAfterSeconds,
2315
+ delayMs: outcome.delayMs,
2316
+ backoffUntilMs: outcome.backoffUntilMs,
2317
+ consecutiveOverloads: outcome.consecutiveOverloads,
2318
+ previousOverloads: claim.state.consecutiveOverloads,
2319
+ sweepOverloads: this.#discoverySweepOverloads,
2320
+ sweepProgress: this.#discoverySweepProgress,
2055
2321
  });
2056
2322
  // backoffUntilMs is already durable via deferDiscoverySweep, and the
2057
2323
  // next runOnce() honors it at the pre-claim wait above — sleeping
@@ -2067,6 +2333,9 @@ export class FactoryLoop {
2067
2333
  this.#discoverySweepEpoch = undefined;
2068
2334
  this.#discoverySweepStartedAtMs = undefined;
2069
2335
  this.#discoveryOverloadError = undefined;
2336
+ this.#discoverySweepOverloads = 0;
2337
+ this.#discoverySweepRetryAfterSeconds = undefined;
2338
+ this.#discoverySweepProgress = false;
2070
2339
  // This sweep is over either way (committed, deferred, or lease lost) —
2071
2340
  // a stale `true` here would otherwise make every #listRelayfileTree
2072
2341
  // call outside a fresh claim (Slack lookups, PR confirmation, the
@@ -2078,6 +2347,94 @@ export class FactoryLoop {
2078
2347
  }
2079
2348
  }
2080
2349
  }
2350
+ /**
2351
+ * Commit the sweep, carrying any residual overload backoff into the store.
2352
+ *
2353
+ * `completeDiscoverySweepWithOverload` is optional on the port so a store
2354
+ * written before #297 keeps working. When one of those is injected and this
2355
+ * sweep HAS a residual, the backoff cannot be persisted and the next sweep
2356
+ * would retry immediately — so this says so, loudly and with a counter,
2357
+ * rather than degrading in silence. The sweep itself still commits: losing
2358
+ * the backoff is worse than pre-#297 behaviour only if nobody notices.
2359
+ */
2360
+ async #commitDiscoverySweep(epoch, checkpoint, residual) {
2361
+ if (residual && this.#state.completeDiscoverySweepWithOverload) {
2362
+ return await this.#state.completeDiscoverySweepWithOverload(this.#workspaceId, this.#discoverySweepOwner, epoch, checkpoint, { consecutiveOverloads: residual.consecutiveOverloads, backoffUntilMs: residual.backoffUntilMs });
2363
+ }
2364
+ if (residual) {
2365
+ this.#increment('discoveryOverloadResidualUnsupported');
2366
+ this.#logger.warn?.('[factory] state store cannot persist the Relayfile overload backoff; it will be lost', {
2367
+ store: this.#state.constructor?.name,
2368
+ consecutiveOverloads: residual.consecutiveOverloads,
2369
+ backoffUntilMs: residual.backoffUntilMs,
2370
+ delayMs: residual.delayMs,
2371
+ });
2372
+ }
2373
+ return await this.#state.completeDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, epoch, checkpoint);
2374
+ }
2375
+ /**
2376
+ * How the durable overload ratchet should read after this sweep, or
2377
+ * `undefined` if relayfile never shed anything and the ordinary reset
2378
+ * applies.
2379
+ *
2380
+ * #297, deliverable 3. `consecutiveOverloads` used to clear only on a
2381
+ * *fully* clean sweep, so under sustained mild load the ratchet essentially
2382
+ * never cleared: it climbed to the cap on the first bad sweep and stayed
2383
+ * there, probing for recovery once per cap-length window, long after the
2384
+ * dependency had recovered. Requiring perfection to clear a ratchet means
2385
+ * the ratchet does not clear.
2386
+ *
2387
+ * The signal that replaces "was this sweep perfect" is "did relayfile serve
2388
+ * any of this sweep's work units", because that is what the ratchet is
2389
+ * actually for. A shedding DO rejects ALL background traffic —
2390
+ * `oldest_inflight_age` does this regardless of how few requests are in
2391
+ * flight — so a sweep against a genuinely overloaded workspace gets not one
2392
+ * unit through and still escalates, all the way to the cap. A sweep that did
2393
+ * get units through proves the dependency is serving us, so it decays by one
2394
+ * rung. Decay, not reset: getting work done while being shed is not evidence
2395
+ * that the overload is over, only that it is survivable.
2396
+ */
2397
+ #discoveryOverloadOutcome(previousOverloads, sweepOutcome, error) {
2398
+ const overload = relayfileOverload(error) ?? relayfileOverload(this.#discoveryOverloadError);
2399
+ if (!overload)
2400
+ return undefined;
2401
+ // The longest delay anything in this sweep asked for, not just the one the
2402
+ // latched or terminating error carried — see `#discoverySweepRetryAfterSeconds`.
2403
+ const advertised = [this.#discoverySweepRetryAfterSeconds, overload.retryAfterSeconds]
2404
+ .filter((value) => value !== undefined);
2405
+ const retryAfterSeconds = advertised.length > 0 ? Math.max(...advertised) : undefined;
2406
+ // Deliberately NOT "the sweep reached its end". A sweep in which relayfile
2407
+ // shed EVERY unit still runs the loop to the end, and it got nothing done;
2408
+ // treating that as progress would decay the ratchet in exactly the case it
2409
+ // exists for. (`#runOnceWithDiscoveryFence` turns that sweep into an
2410
+ // aborted one before it can commit, so the escalate branch here is reached
2411
+ // only through `sweepOutcome === 'aborted'` — but the rule is a property of
2412
+ // progress, not of which caller asked, and is written that way.)
2413
+ const consecutiveOverloads = this.#discoverySweepProgress
2414
+ ? Math.max(0, previousOverloads - 1)
2415
+ : previousOverloads + 1;
2416
+ const delayMs = discoveryOverloadBackoffMs(retryAfterSeconds, consecutiveOverloads);
2417
+ const backoffUntilMs = this.#clock.now() + delayMs;
2418
+ this.#increment('discoveryOverloadBackoffs');
2419
+ if (sweepOutcome === 'committed') {
2420
+ this.#logger.warn?.('[factory] discovery sweep committed despite Relayfile overload', {
2421
+ status: overload.status,
2422
+ reason: overload.reason,
2423
+ retryAfterSeconds,
2424
+ sweepOverloads: this.#discoverySweepOverloads,
2425
+ previousOverloads,
2426
+ consecutiveOverloads,
2427
+ delayMs,
2428
+ backoffUntilMs,
2429
+ });
2430
+ }
2431
+ return {
2432
+ consecutiveOverloads,
2433
+ backoffUntilMs,
2434
+ delayMs,
2435
+ ...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }),
2436
+ };
2437
+ }
2081
2438
  async #performRunOnce(opts = {}) {
2082
2439
  const dryRun = opts.dryRun ?? this.#config.dryRun;
2083
2440
  const startedAtMs = this.#clock.now();
@@ -2125,15 +2482,57 @@ export class FactoryLoop {
2125
2482
  let readyIssueReads = 0;
2126
2483
  const issueEntries = [];
2127
2484
  for (const path of paths) {
2128
- const issue = await this.#readIssue(path);
2485
+ let issue;
2486
+ let shed = false;
2487
+ try {
2488
+ issue = await this.#readIssue(path);
2489
+ }
2490
+ catch (error) {
2491
+ // #297: `#readIssue` rethrows relayfile overload and swallows every
2492
+ // other read fault, so this catch only ever sees the backend
2493
+ // shedding THIS issue's read. That is a fact about one work unit:
2494
+ // a sweep that pulled 40 issues and was shed on the 39th must keep
2495
+ // the other 39, exactly as #292 argued for dispatch failures. The
2496
+ // fuse is what still ends a sweep the backend is refusing.
2497
+ const overload = relayfileOverload(error);
2498
+ // Defensive: anything `#readIssue` did not swallow and is not a 429
2499
+ // is not ours to reclassify, and must surface as itself.
2500
+ if (!overload)
2501
+ throw error;
2502
+ const fuse = this.#discoveryOverloadFuseError();
2503
+ if (fuse)
2504
+ throw fuse;
2505
+ shed = true;
2506
+ this.#increment('discoveryOverloadItemsSkipped');
2507
+ this.#logger.warn?.('[factory] relayfile shed a ready-issue read; skipping it and continuing the sweep', {
2508
+ path,
2509
+ status: overload.status,
2510
+ reason: overload.reason,
2511
+ retryAfterSeconds: overload.retryAfterSeconds,
2512
+ sweepOverloads: this.#discoverySweepOverloads,
2513
+ });
2514
+ recordSkip({ issue: issueRefFromPath(path), reason: perItemDispatchSkipReason(error) });
2515
+ }
2129
2516
  readyIssueReads += 1;
2517
+ // Relayfile served this work unit's read: the dependency is shedding
2518
+ // but not refusing, which is what decays the ratchet (#297).
2519
+ //
2520
+ // `issue` is required, not just `!shed`: `#readIssue` returns
2521
+ // `undefined` for a body it could not read at all, and the known
2522
+ // phantom condition — the tree listing issue paths whose bodies are
2523
+ // absent — makes every read return `undefined`. Crediting those would
2524
+ // decay the ratchet on a sweep that served nothing.
2525
+ if (!shed && issue)
2526
+ this.#discoverySweepProgress = true;
2130
2527
  lastReadyReadProgressAtMs = this.#logTimedProgress(this.#config.issueSource === 'github'
2131
2528
  ? '[factory] GitHub ready issue read progress'
2132
2529
  : '[factory] Linear ready issue read progress', startedAtMs, lastReadyReadProgressAtMs, { read: readyIssueReads, total: paths.length, path });
2133
- if (issue && issueSource === 'linear') {
2134
- await this.#recordCanonicalIssueState(issue);
2530
+ if (!shed) {
2531
+ if (issue && issueSource === 'linear') {
2532
+ await this.#recordCanonicalIssueState(issue);
2533
+ }
2534
+ issueEntries.push({ path, issue });
2135
2535
  }
2136
- issueEntries.push({ path, issue });
2137
2536
  await this.#refreshLiveHeartbeatIfDue();
2138
2537
  }
2139
2538
  if (issueSource === 'github') {
@@ -2222,6 +2621,9 @@ export class FactoryLoop {
2222
2621
  // A completed dispatch — even one that parks or escalates the issue —
2223
2622
  // proves the pipeline still works, so the fuse below starts over.
2224
2623
  unclassifiedFailuresSinceDispatch = 0;
2624
+ // ...and proves relayfile is still serving this sweep, which is what
2625
+ // decays the durable overload ratchet (#297).
2626
+ this.#discoverySweepProgress = true;
2225
2627
  if (result.agents.length === 0 && !dryRun) {
2226
2628
  const reason = result.hold?.kind === 'dependency-cycle'
2227
2629
  ? `dependency cycle detected: ${result.hold.cycle?.join(' -> ') ?? 'unknown cycle'}`
@@ -2235,13 +2637,47 @@ export class FactoryLoop {
2235
2637
  }
2236
2638
  }
2237
2639
  catch (error) {
2640
+ // The failure may have left half-spawned agents behind, persisted as
2641
+ // failure handoffs on the way out of `#dispatchUnlocked`. runLoop's
2642
+ // catch used to reap them because every such error aborted the pass;
2643
+ // now that most of them are skipped, the reap has to happen here.
2644
+ //
2645
+ // BEFORE the fatality check, not after (#298 review, round two): the
2646
+ // 429 that trips the overload fuse aborts this pass, and a direct
2647
+ // `runOnce()` — the `factory run-once` CLI, `#reconcileReadyIssues` —
2648
+ // has no runLoop catch behind it, so the unit that trips the fuse
2649
+ // would leak the agents it had already spawned. Reaping first covers
2650
+ // the abort and the skip with one call; it is idempotent, so the
2651
+ // runLoop catch finding nothing left to do is free.
2652
+ if (mayHaveSpawnedBeforeFailing(error)) {
2653
+ await this.#reapDispatchFailureHandoffsNow();
2654
+ }
2238
2655
  // #292: issues in a pass are independent work units, so a failure
2239
2656
  // that is about ONE unit costs that unit and nothing else. Only the
2240
2657
  // conditions named in `#isPassFatalFailure` — the ones where
2241
2658
  // continuing the pass is meaningless — abort the whole sweep.
2242
- if (this.#isPassFatalFailure(error, dryRun))
2243
- throw error;
2244
- if (!isClassifiedPerItemDispatchFailure(error)) {
2659
+ if (this.#isPassFatalFailure(error, dryRun)) {
2660
+ // The overload fuse may have been tripped by a 429 that a caller
2661
+ // swallowed, leaving an unrelated error in hand. The fence keys
2662
+ // the durable backoff off `relayfileOverload(error)`, so hand it
2663
+ // the 429 rather than whatever surfaced last.
2664
+ throw this.#discoveryOverloadFuseError() ?? error;
2665
+ }
2666
+ const overload = relayfileOverload(error);
2667
+ if (overload) {
2668
+ // #297: shedding is a state of the dependency, not a fault of this
2669
+ // work unit, so it stays out of `counters.errors` and gets its own
2670
+ // counter — the same split #293 made for undispatchable units.
2671
+ this.#increment('discoveryOverloadItemsSkipped');
2672
+ this.#logger.warn?.('[factory] relayfile shed this work unit; skipping it and continuing the sweep', {
2673
+ issue: issueRef(issue).key,
2674
+ status: overload.status,
2675
+ reason: overload.reason,
2676
+ retryAfterSeconds: overload.retryAfterSeconds,
2677
+ sweepOverloads: this.#discoverySweepOverloads,
2678
+ });
2679
+ }
2680
+ else if (!isClassifiedPerItemDispatchFailure(error)) {
2245
2681
  unclassifiedFailuresSinceDispatch += 1;
2246
2682
  // A pass-wide fault can arrive disguised as a run of per-item
2247
2683
  // faults. Skipping every unit would then hand back a green report
@@ -2260,11 +2696,6 @@ export class FactoryLoop {
2260
2696
  error: describeError(error).errorMessage,
2261
2697
  });
2262
2698
  this.#error(error, issueRef(issue));
2263
- // The failure may have left half-spawned agents behind. runLoop's
2264
- // catch used to reap them because this error aborted the pass;
2265
- // now that the pass survives, the reap has to happen here or the
2266
- // agents leak until the next failed iteration.
2267
- await this.#reapDispatchFailureHandoffsNow();
2268
2699
  }
2269
2700
  else {
2270
2701
  // Not an error — the unit simply cannot be dispatched right now —
@@ -2313,6 +2744,21 @@ export class FactoryLoop {
2313
2744
  }
2314
2745
  }
2315
2746
  }
2747
+ /**
2748
+ * The 429 that ended this sweep, when relayfile has shed
2749
+ * `DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT` operations and skipping the next work
2750
+ * unit would just be grinding a shedding dependency. Undefined below that.
2751
+ *
2752
+ * Returns the *latched* 429 rather than whatever error is in hand, because
2753
+ * `#runOnceWithDiscoveryFence` keys the durable backoff off
2754
+ * `relayfileOverload(error)` and a caller may have swallowed the 429 that
2755
+ * tripped the fuse.
2756
+ */
2757
+ #discoveryOverloadFuseError() {
2758
+ if (this.#discoverySweepOverloads < DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT)
2759
+ return undefined;
2760
+ return this.#discoveryOverloadError;
2761
+ }
2316
2762
  /**
2317
2763
  * Whether a failure raised while processing ONE work unit must abort the
2318
2764
  * whole readiness pass instead of skipping that unit.
@@ -2333,10 +2779,11 @@ export class FactoryLoop {
2333
2779
  * one would be recorded as an ordinary per-issue skip. The run report
2334
2780
  * would then claim a clean pass over work this process no longer has the
2335
2781
  * right to touch.
2336
- * - Relayfile signalled overload for this sweep. The backend is shedding
2337
- * load; grinding through the remaining units makes it worse, and
2338
- * `#runOnceWithDiscoveryFence` is going to rethrow this at the fence
2339
- * anyway.
2782
+ * - Relayfile has shed `DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT` operations in
2783
+ * this sweep. A single shed operation is per-item and skippable (#297),
2784
+ * but past this the backend is not serving this sweep at all: grinding
2785
+ * through the remaining units makes it worse, and the fence needs the 429
2786
+ * to set the durable backoff.
2340
2787
  * - The factory is stopping. Teardown is in progress and dispatching more
2341
2788
  * agents now leaks them past the shutdown deadline.
2342
2789
  * - The fleet control-plane circuit is no longer closed, **on a live pass**.
@@ -2365,7 +2812,15 @@ export class FactoryLoop {
2365
2812
  #isPassFatalFailure(error, dryRun) {
2366
2813
  // Sweep-scoped: these are about this process's right or ability to run the
2367
2814
  // pass at all, so they hold for a dry run exactly as for a live one.
2368
- if (this.#discoverySweepLeaseLost || this.#discoveryOverloadError !== undefined || this.#stopping) {
2815
+ if (this.#discoverySweepLeaseLost || this.#stopping) {
2816
+ return true;
2817
+ }
2818
+ // #297: relayfile overload used to sit alongside those two, and it did not
2819
+ // belong there. A 429 on ONE work unit is a fact about that unit's read or
2820
+ // write, not about this process's right to run the pass — and because the
2821
+ // flag latched for the whole sweep, the first transient shed also made
2822
+ // every later unit fatal. Only sustained shedding is now pass-fatal.
2823
+ if (this.#discoverySweepOverloads >= DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT) {
2369
2824
  return true;
2370
2825
  }
2371
2826
  // Fleet-scoped, and therefore live-only. See the doc comment above.
@@ -3231,14 +3686,40 @@ export class FactoryLoop {
3231
3686
  return result;
3232
3687
  }
3233
3688
  catch (error) {
3234
- if (relayfileOverload(error) && this.#discoverySweepEpoch !== undefined) {
3689
+ const overload = relayfileOverload(error);
3690
+ if (overload && this.#discoverySweepEpoch !== undefined) {
3235
3691
  this.#discoveryOverloadError ??= error;
3692
+ this.#discoverySweepOverloads += 1;
3693
+ if (overload.retryAfterSeconds !== undefined) {
3694
+ this.#discoverySweepRetryAfterSeconds = Math.max(this.#discoverySweepRetryAfterSeconds ?? 0, overload.retryAfterSeconds);
3695
+ }
3696
+ this.#increment('discoveryOverloadOperations');
3697
+ // #297: relayfile's four overload reason codes — inflight_limit,
3698
+ // oldest_inflight_age, router_inflight_limit, durable_object_overloaded
3699
+ // — all share ONE message string, and they mean four different things:
3700
+ // a DO-local admission cap, one stuck op poisoning every background
3701
+ // caller, a Worker-global isolate cap, and the Cloudflare runtime
3702
+ // shedding the object outright. `relayfileOverload()` has always parsed
3703
+ // the reason; nothing on this path ever printed it, and during the
3704
+ // 2026-08-20 outage that ambiguity was the single biggest obstacle to
3705
+ // diagnosis. Unconditional, unlike the failure warn below: a 429 is
3706
+ // always worth one line.
3707
+ this.#increment(`discoveryOverloadReason:${relayfileOverloadReasonLabel(overload.reason)}`);
3708
+ this.#logger.warn?.('[factory] relayfile shed a discovery operation', {
3709
+ ...metadata,
3710
+ status: overload.status,
3711
+ reason: overload.reason,
3712
+ retryAfterSeconds: overload.retryAfterSeconds,
3713
+ sweepOverloads: this.#discoverySweepOverloads,
3714
+ elapsedMs: this.#elapsedSince(startedAtMs),
3715
+ });
3236
3716
  }
3237
3717
  if (opts.logFailure || waitWarnings > 0) {
3238
3718
  this.#increment('relayfileOperationFailures');
3239
3719
  this.#logger.warn?.('[factory] relayfile operation failed', {
3240
3720
  ...metadata,
3241
3721
  elapsedMs: this.#elapsedSince(startedAtMs),
3722
+ ...(overload ? { status: overload.status, reason: overload.reason } : {}),
3242
3723
  error: describeError(error).errorMessage,
3243
3724
  });
3244
3725
  }
@@ -3587,6 +4068,14 @@ export class FactoryLoop {
3587
4068
  if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
3588
4069
  throw new Error(`Dispatch lifecycle ownership lost immediately before spawning ${dispatchDecision.issue.key}`);
3589
4070
  }
4071
+ // That save stamped `slotHeldSinceAtMs`: the row occupies a batch slot from
4072
+ // here on. Arm its deadline before the first await rather than after a
4073
+ // placement succeeds — `#fleet.spawn` deliberately carries no mutation
4074
+ // timeout, so a first attempt that hangs in an otherwise idle process would
4075
+ // otherwise hold the slot with no timer that can ever fire. That is the
4076
+ // exact shape of #303, reached through the fresh dispatch path instead of
4077
+ // the durable one (#303 review, CodeRabbit).
4078
+ this.#scheduleHeldAgentDeadline(record);
3590
4079
  if (!dryRun)
3591
4080
  await this.#ensureGithubAgentQuestionWatch(record, liveIssue);
3592
4081
  const spawnedForReaperHandoff = [];
@@ -3778,6 +4267,7 @@ export class FactoryLoop {
3778
4267
  slackDegradedReason: this.#slackDegradedReason,
3779
4268
  eventListener: this.#eventListenerStatus(),
3780
4269
  readinessReconcile: this.#readinessReconcileStatus(),
4270
+ dispatchCapacity: this.#dispatchCapacityStatus(),
3781
4271
  heldAgents: batch?.inFlight.flatMap((record) => heldAgentsForRecord(record, nowMs, this.#config.dispatch.agentHoldTimeoutMs, this.#config.terminalState)) ?? [],
3782
4272
  };
3783
4273
  }
@@ -3801,17 +4291,66 @@ export class FactoryLoop {
3801
4291
  }
3802
4292
  #readinessReconcileStatus() {
3803
4293
  const consecutiveFailures = this.#readinessReconcileConsecutiveFailures;
3804
- const state = this.#startMode !== 'live'
4294
+ // #296 owns the numerator here, #295/#300 own the derivation. The earliest
4295
+ // sweep still running — after a deadline expiry that is the abandoned one,
4296
+ // not the current wait, or every expiry would restart the clock and a
4297
+ // permanently stuck pass would read as merely `retrying`.
4298
+ const inFlightSinceMs = Math.min(this.#readinessReconcileInFlightSinceMs ?? Number.POSITIVE_INFINITY, this.#readinessReconcileAbandonedSinceMs ?? Number.POSITIVE_INFINITY);
4299
+ const settled = this.#startMode !== 'live'
3805
4300
  ? 'not-running'
4301
+ // Failure-count ladder only. Precedence against `stalled` belongs to
4302
+ // `derivedReadinessReconcileState`, which outranks everything here: a
4303
+ // stall is the more specific fact, and `consecutiveFailures` ships
4304
+ // alongside so nothing an alarm keyed on `degraded` needs is lost.
3806
4305
  : consecutiveFailures >= READINESS_RECONCILE_FAILURE_THRESHOLD
3807
4306
  ? 'degraded'
3808
4307
  : consecutiveFailures > 0
3809
4308
  ? 'retrying'
3810
4309
  : 'healthy';
4310
+ // The counters above only move when a pass *settles*. A pass that hangs
4311
+ // takes neither path, so `settled` would keep reporting the last finished
4312
+ // pass — `healthy` — for as long as the process is stuck (#295). The
4313
+ // in-flight age is the only field that can express that, so derive the
4314
+ // state from it rather than trusting the last write.
4315
+ const timestamps = {
4316
+ intervalMs: this.#readinessReconcileIntervalMs,
4317
+ ...(Number.isFinite(inFlightSinceMs) ? { inFlightSinceMs } : {}),
4318
+ ...(this.#readinessReconcileLastStartedAtMs !== undefined
4319
+ ? { lastStartedAtMs: this.#readinessReconcileLastStartedAtMs }
4320
+ : {}),
4321
+ ...(this.#readinessReconcileLastCompletedAtMs !== undefined
4322
+ ? { lastCompletedAtMs: this.#readinessReconcileLastCompletedAtMs }
4323
+ : {}),
4324
+ ...(this.#readinessReconcileLastFailureAtMs !== undefined
4325
+ ? { lastFailureAtMs: this.#readinessReconcileLastFailureAtMs }
4326
+ : {}),
4327
+ };
4328
+ const nowMs = this.#clock.now();
4329
+ // Defence in depth (#300 review, CodeRabbit). These derivations are new
4330
+ // code from another module on a path that `status()` and every heartbeat
4331
+ // write depend on. A throw here would take out the liveness signal the
4332
+ // crash reaper reads — the diagnostic causing the outage it exists to
4333
+ // explain — so a failure costs the derived fields and nothing else.
4334
+ let inFlightMs;
4335
+ let derived = settled;
4336
+ try {
4337
+ inFlightMs = readinessReconcileInFlightMs(timestamps, nowMs);
4338
+ derived = derivedReadinessReconcileState({ ...timestamps, state: settled }, nowMs);
4339
+ }
4340
+ catch (error) {
4341
+ this.#logger.warn?.('[factory] readiness health derivation failed; reporting the settled state', {
4342
+ error: describeError(error).errorMessage,
4343
+ });
4344
+ inFlightMs = undefined;
4345
+ derived = settled;
4346
+ }
3811
4347
  return {
3812
- state,
4348
+ state: derived === 'unknown' ? settled : derived,
3813
4349
  consecutiveFailures,
3814
4350
  failureThreshold: READINESS_RECONCILE_FAILURE_THRESHOLD,
4351
+ intervalMs: this.#readinessReconcileIntervalMs,
4352
+ ...(Number.isFinite(inFlightSinceMs) ? { inFlightSinceMs } : {}),
4353
+ ...(inFlightMs !== undefined ? { inFlightMs } : {}),
3815
4354
  ...(this.#readinessReconcileLastDurationMs !== undefined
3816
4355
  ? { lastDurationMs: this.#readinessReconcileLastDurationMs }
3817
4356
  : {}),
@@ -3825,6 +4364,9 @@ export class FactoryLoop {
3825
4364
  ? { lastFailureAtMs: this.#readinessReconcileLastFailureAtMs }
3826
4365
  : {}),
3827
4366
  ...(this.#readinessReconcileLastError ? { lastError: this.#readinessReconcileLastError } : {}),
4367
+ ...(this.#readinessReconcileLastErrorClass
4368
+ ? { lastErrorClass: this.#readinessReconcileLastErrorClass }
4369
+ : {}),
3828
4370
  };
3829
4371
  }
3830
4372
  on(event, listener) {
@@ -4142,7 +4684,7 @@ export class FactoryLoop {
4142
4684
  if (timer)
4143
4685
  clearTimeout(timer);
4144
4686
  this.#dispatchLifecycleRetryTimers.delete(key);
4145
- this.#dispatchLifecycleCapacityWaitLogged.delete(key);
4687
+ this.#dispatchLifecycleCapacityWaits.delete(key);
4146
4688
  clearedKeys.add(key);
4147
4689
  this.#increment('dispatchLifecycleGithubAliasesCollapsed');
4148
4690
  }
@@ -4200,10 +4742,67 @@ export class FactoryLoop {
4200
4742
  }, DISPATCH_LIFECYCLE_RENEW_MS);
4201
4743
  this.#dispatchLifecycleRenewTimer.unref?.();
4202
4744
  }
4745
+ /**
4746
+ * The wall-clock deadline that can free this record's batch slot, if any.
4747
+ *
4748
+ * Two clocks, never both. `heldSinceAtMs` is stamped by the first successful
4749
+ * placement and bounds a team that ran and never reached a terminal state.
4750
+ * A record that has no `heldSinceAtMs` never had a placement at all, and
4751
+ * before #303 that made it permanently unreapable: `#scheduleHeldAgentDeadline`
4752
+ * armed no timer and `#sweepHeldAgentDeadlines` skipped it, so a row that
4753
+ * reached `dispatching` and lost its process held the only batch slot
4754
+ * forever. Such a row is definitionally stuck — nothing but a placement can
4755
+ * move it, and no placement is coming — so it gets the much shorter
4756
+ * `agentlessHoldTimeoutMs` anchored on when it took the slot.
4757
+ *
4758
+ * The anchor is deliberately not `agents.size === 0`: `recordPlanned` writes
4759
+ * the spec before the spawn returns, so a process that died mid-spawn leaves
4760
+ * an agent entry with no result and still no placement.
4761
+ */
4762
+ /**
4763
+ * Does this in-flight record hold a `batchSize` slot right now?
4764
+ *
4765
+ * The same predicate the state stores apply to the durable row, asked of the
4766
+ * in-memory one. Phase alone is not it: once every implementer repo has been
4767
+ * handed to a babysitter, admission stops counting the lifecycle, so
4768
+ * reporting it as an occupant would name slots that are not blocking
4769
+ * anything (#303 review, codex).
4770
+ */
4771
+ #recordOccupiesSlot(record) {
4772
+ return dispatchPhaseOccupiesSlot(record.lifecyclePhase) && !dispatchHandedOffToBabysitters(record.decision.implementers, [...record.agents.values()].map((tracked) => ({ releasedAtMs: tracked.releasedAtMs, spec: tracked.spec })));
4773
+ }
4774
+ #holdDeadline(record) {
4775
+ if (record.dryRun)
4776
+ return undefined;
4777
+ if (record.heldSinceAtMs !== undefined) {
4778
+ const timeoutMs = this.#config.dispatch.agentHoldTimeoutMs;
4779
+ return {
4780
+ kind: 'agents',
4781
+ sinceAtMs: record.heldSinceAtMs,
4782
+ timeoutMs,
4783
+ dueAtMs: record.heldSinceAtMs + timeoutMs,
4784
+ };
4785
+ }
4786
+ // Only a row that is actually holding a slot is worth reaping; a `queued`
4787
+ // or `waiting-for-human` row costs nothing and may wait indefinitely.
4788
+ if (record.slotHeldSinceAtMs === undefined || !this.#recordOccupiesSlot(record)) {
4789
+ return undefined;
4790
+ }
4791
+ const timeoutMs = this.#config.dispatch.agentlessHoldTimeoutMs;
4792
+ return {
4793
+ kind: 'agentless',
4794
+ sinceAtMs: record.slotHeldSinceAtMs,
4795
+ timeoutMs,
4796
+ dueAtMs: record.slotHeldSinceAtMs + timeoutMs,
4797
+ };
4798
+ }
4203
4799
  #scheduleHeldAgentDeadline(record) {
4204
- if (this.#stopping || record.dryRun || record.heldSinceAtMs === undefined || record.agents.size === 0)
4800
+ if (this.#stopping)
4205
4801
  return;
4206
- const dueAtMs = record.heldSinceAtMs + this.#config.dispatch.agentHoldTimeoutMs;
4802
+ const deadline = this.#holdDeadline(record);
4803
+ if (!deadline)
4804
+ return;
4805
+ const dueAtMs = deadline.dueAtMs;
4207
4806
  if (this.#heldAgentDeadlineTimer &&
4208
4807
  this.#heldAgentDeadlineDueAtMs !== undefined &&
4209
4808
  this.#heldAgentDeadlineDueAtMs <= dueAtMs)
@@ -4243,14 +4842,16 @@ export class FactoryLoop {
4243
4842
  }
4244
4843
  async #sweepHeldAgentDeadlines() {
4245
4844
  const nowMs = this.#clock.now();
4246
- const timeoutMs = this.#config.dispatch.agentHoldTimeoutMs;
4247
4845
  for (const record of [...(await this.#batch()).inFlight]) {
4248
- const heldSinceAtMs = record.heldSinceAtMs;
4249
- if (record.dryRun ||
4250
- heldSinceAtMs === undefined ||
4251
- record.agents.size === 0 ||
4252
- nowMs - heldSinceAtMs < timeoutMs)
4846
+ const deadline = this.#holdDeadline(record);
4847
+ if (!deadline || nowMs < deadline.dueAtMs)
4253
4848
  continue;
4849
+ // The durable row wins the classification when there is one. The
4850
+ // in-memory record can lag a placement made in another process, and
4851
+ // reading the stale one would relabel a team that did run as
4852
+ // never-placed — which also excludes its agents from the release below,
4853
+ // leaking live workers (#303 review, CodeRabbit).
4854
+ let effective = deadline;
4254
4855
  const key = issueKey(record.issue);
4255
4856
  if (this.#abandonedDispatchReasons.has(key))
4256
4857
  continue;
@@ -4264,26 +4865,42 @@ export class FactoryLoop {
4264
4865
  await this.#finishDurableRelease(record, lifecycle.releaseReason);
4265
4866
  continue;
4266
4867
  }
4868
+ // Re-derive against the durable row before tearing anything down. The
4869
+ // in-memory record can be a beat behind a placement that just
4870
+ // succeeded in this process or a takeover in another, and the
4871
+ // never-placed deadline exists precisely to catch rows nothing is
4872
+ // moving — it must not be what ends a dispatch that just started
4873
+ // moving (#303 must-not-fire).
4874
+ const durable = this.#holdDeadline(inFlightRecordFromLifecycle(lifecycle));
4875
+ if (!durable || this.#clock.now() < durable.dueAtMs)
4876
+ continue;
4877
+ effective = durable;
4267
4878
  if (!await this.#assertDispatchLifecycleOwner(record))
4268
4879
  continue;
4269
4880
  }
4270
- const heldForMs = Math.max(0, this.#clock.now() - heldSinceAtMs);
4881
+ const agentless = effective.kind === 'agentless';
4882
+ const heldForMs = Math.max(0, this.#clock.now() - effective.sinceAtMs);
4271
4883
  const details = {
4272
4884
  issue: record.issue.key,
4273
4885
  heldForMs,
4274
- holdTimeoutMs: timeoutMs,
4886
+ holdTimeoutMs: effective.timeoutMs,
4275
4887
  waitingForTerminalState: this.#config.terminalState,
4276
- reason: HELD_PAST_DEADLINE_RELEASE_REASON,
4888
+ reason: agentless ? AGENTLESS_SLOT_PAST_DEADLINE_RELEASE_REASON : HELD_PAST_DEADLINE_RELEASE_REASON,
4277
4889
  agents: [...record.agents.keys()].sort(),
4890
+ ...(agentless ? { phase: record.lifecyclePhase } : {}),
4278
4891
  };
4279
- this.#logger.warn?.('[factory] releasing agents held past deadline', details);
4280
- await this.#abandonStuckDispatch(record, HELD_PAST_DEADLINE_RELEASE_REASON);
4892
+ this.#logger.warn?.(agentless
4893
+ ? '[factory] releasing a dispatch lifecycle that never placed an agent'
4894
+ : '[factory] releasing agents held past deadline', details);
4895
+ await this.#abandonStuckDispatch(record, details.reason);
4281
4896
  const lifecycle = this.#usesDurableDispatchLifecycle()
4282
4897
  ? await this.#state.getDispatchLifecycle(this.#workspaceId, key)
4283
4898
  : undefined;
4284
4899
  if (!lifecycle || isTerminalDispatchLifecycle(lifecycle)) {
4285
- this.#increment('heldPastDeadlineReleases');
4286
- this.#logger.warn?.('[factory] released agents held past deadline', details);
4900
+ this.#increment(agentless ? 'agentlessSlotPastDeadlineReleases' : 'heldPastDeadlineReleases');
4901
+ this.#logger.warn?.(agentless
4902
+ ? '[factory] released a dispatch lifecycle that never placed an agent'
4903
+ : '[factory] released agents held past deadline', details);
4287
4904
  }
4288
4905
  }
4289
4906
  }
@@ -4654,6 +5271,14 @@ export class FactoryLoop {
4654
5271
  }
4655
5272
  async #saveDispatchLifecycle(record, phase, pullRequest, releaseReason, releasedAgentNames = new Set(), telemetry = {}) {
4656
5273
  record.lifecyclePhase = phase;
5274
+ // Mirror what the store stamps, so the reaper's never-placed clock is
5275
+ // readable from the in-memory record between durable reads (#303). The
5276
+ // store still owns the authoritative value; this uses the same predicate so
5277
+ // the two cannot disagree.
5278
+ if (this.#recordOccupiesSlot(record))
5279
+ record.slotHeldSinceAtMs ??= this.#clock.now();
5280
+ else
5281
+ record.slotHeldSinceAtMs = undefined;
4657
5282
  if (record.dryRun || !this.#usesDurableDispatchLifecycle())
4658
5283
  return true;
4659
5284
  if (isTerminalDispatchPhase(phase))
@@ -4721,6 +5346,20 @@ export class FactoryLoop {
4721
5346
  if (isTerminalDispatchLifecycle(lifecycle)) {
4722
5347
  this.#dispatchLifecycleEpochs.delete(key);
4723
5348
  }
5349
+ // Wake the capacity waiters exactly when this write gives a slot back —
5350
+ // occupied before, not occupied after — and never otherwise (#303
5351
+ // review, cubic).
5352
+ //
5353
+ // Not "when it goes terminal". `releasing` already does not occupy a
5354
+ // slot, so a normal completion frees it one save *before* `complete`,
5355
+ // and a babysitter handoff frees it without ever going terminal at all.
5356
+ // Keying on the terminal save alone therefore both fires for rows that
5357
+ // freed nothing (a `queued` row abandoned at startup) and misses the
5358
+ // writes that actually freed something. The occupancy transition is the
5359
+ // event; the phase is only a proxy for it.
5360
+ if (previous && dispatchLifecycleOccupiesSlot(previous) && !dispatchLifecycleOccupiesSlot(lifecycle)) {
5361
+ this.#resetDispatchCapacityBackoff();
5362
+ }
4724
5363
  return true;
4725
5364
  });
4726
5365
  }
@@ -4734,7 +5373,130 @@ export class FactoryLoop {
4734
5373
  resolve(phase);
4735
5374
  this.#dispatchTerminalWaiters.delete(key);
4736
5375
  }
4737
- #scheduleDispatchLifecycleRetry(record) {
5376
+ /**
5377
+ * Re-arm delay for a capacity wait: 1 s doubling to a 30 s ceiling (#303).
5378
+ *
5379
+ * Only the capacity path backs off. An ownership wait is already bounded by
5380
+ * `DISPATCH_LIFECYCLE_LEASE_MS`, and every other failure is a real error
5381
+ * whose fast retry is the recovery. A capacity wait has no bound at all —
5382
+ * it ends when some other lifecycle terminates, which may be hours away or,
5383
+ * before this fix, never.
5384
+ */
5385
+ #capacityRetryDelayMs(attempts) {
5386
+ return Math.min(DISPATCH_LIFECYCLE_RETRY_MS * 2 ** Math.max(0, attempts - 1), DISPATCH_LIFECYCLE_RETRY_MAX_MS);
5387
+ }
5388
+ /** Issue keys currently holding a `batchSize` slot, for operator surfaces. */
5389
+ #dispatchSlotOccupants() {
5390
+ return (this.#batchView?.inFlight ?? [])
5391
+ .filter((record) => !record.dryRun && this.#recordOccupiesSlot(record))
5392
+ .map((record) => ({
5393
+ issue: record.issue.key,
5394
+ ...(record.lifecyclePhase ? { phase: record.lifecyclePhase } : {}),
5395
+ agents: record.agents.size,
5396
+ // Specs, not workers: `recordPlanned` writes an entry before the spawn
5397
+ // returns, so `agents > 0` is not proof of a placement (#303 review).
5398
+ placedAgents: [...record.agents.values()].filter((tracked) => tracked.result !== undefined).length,
5399
+ ...(record.heldSinceAtMs !== undefined
5400
+ ? { heldForMs: Math.max(0, this.#clock.now() - record.heldSinceAtMs) }
5401
+ : {}),
5402
+ ...(record.slotHeldSinceAtMs !== undefined
5403
+ ? { slotHeldForMs: Math.max(0, this.#clock.now() - record.slotHeldSinceAtMs) }
5404
+ : {}),
5405
+ }))
5406
+ .sort((left, right) => left.issue.localeCompare(right.issue));
5407
+ }
5408
+ /**
5409
+ * Batch occupancy as an operator-readable fact (#303).
5410
+ *
5411
+ * Before this, a full batch was visible only as the *absence* of dispatch:
5412
+ * `readinessReconcile` stayed green, `consecutiveFailures` stayed 0, and the
5413
+ * one capacity log had fired hours earlier. Publishing occupancy is what
5414
+ * turns "nothing is being dispatched" into a question an operator can answer
5415
+ * without reading the state document.
5416
+ */
5417
+ #dispatchCapacityStatus() {
5418
+ const nowMs = this.#clock.now();
5419
+ const occupants = this.#dispatchSlotOccupants();
5420
+ const waits = [...this.#dispatchLifecycleCapacityWaits.entries()];
5421
+ const longestWaitMs = waits.length === 0
5422
+ ? undefined
5423
+ : Math.max(...waits.map(([, wait]) => Math.max(0, nowMs - wait.sinceAtMs)));
5424
+ return {
5425
+ batchSize: this.#config.batchSize,
5426
+ active: occupants.length,
5427
+ waiting: waits.length,
5428
+ waitWarnMs: this.#config.dispatch.capacityWaitWarnMs,
5429
+ agentlessHoldTimeoutMs: this.#config.dispatch.agentlessHoldTimeoutMs,
5430
+ ...(longestWaitMs !== undefined ? { longestWaitMs } : {}),
5431
+ ...(occupants.length > 0 ? { occupants } : {}),
5432
+ ...(waits.length > 0
5433
+ ? {
5434
+ waitingIssues: waits
5435
+ .sort(([, left], [, right]) => left.sinceAtMs - right.sinceAtMs)
5436
+ .map(([key]) => key),
5437
+ }
5438
+ : {}),
5439
+ };
5440
+ }
5441
+ #recordDispatchCapacityWait(record, key) {
5442
+ const nowMs = this.#clock.now();
5443
+ let wait = this.#dispatchLifecycleCapacityWaits.get(key);
5444
+ if (!wait) {
5445
+ wait = { record, sinceAtMs: nowMs, attempts: 0 };
5446
+ this.#dispatchLifecycleCapacityWaits.set(key, wait);
5447
+ this.#increment('dispatchLifecycleCapacityWaits');
5448
+ }
5449
+ wait.record = record;
5450
+ wait.attempts += 1;
5451
+ const retryMs = this.#capacityRetryDelayMs(wait.attempts);
5452
+ const waitedMs = Math.max(0, nowMs - wait.sinceAtMs);
5453
+ // Escalate on every backoff step, then once a minute after the delay
5454
+ // caps. The old behaviour logged once per key and went silent forever,
5455
+ // which is what made a 14-hour dispatch outage look like an idle Factory.
5456
+ const stepChanged = wait.lastLoggedRetryMs !== retryMs;
5457
+ const overdue = wait.lastLoggedAtMs === undefined ||
5458
+ nowMs - wait.lastLoggedAtMs >= DISPATCH_LIFECYCLE_CAPACITY_WAIT_LOG_MS;
5459
+ if (stepChanged || overdue) {
5460
+ wait.lastLoggedAtMs = nowMs;
5461
+ wait.lastLoggedRetryMs = retryMs;
5462
+ this.#logger.warn?.('[factory] durable dispatch is queued for batch capacity; retries remain active', {
5463
+ issue: record.issue.key,
5464
+ retryMs,
5465
+ attempts: wait.attempts,
5466
+ waitedMs,
5467
+ batchSize: this.#config.batchSize,
5468
+ occupiedBy: this.#dispatchSlotOccupants().map((occupant) => occupant.issue),
5469
+ });
5470
+ }
5471
+ return retryMs;
5472
+ }
5473
+ /**
5474
+ * Put every capacity waiter back on the fast path, because a slot just freed.
5475
+ *
5476
+ * The backoff exists to stop a storm of retries asking a question whose
5477
+ * answer is not changing. When a lifecycle reaches a terminal phase the
5478
+ * answer *has* changed, so parking a waiter behind a 30 s timer would trade
5479
+ * the storm for latency — and for a slot released by another process, that
5480
+ * timer is the only signal this one gets (#303 review follow-up).
5481
+ *
5482
+ * The wait's `sinceAtMs` is deliberately untouched: the issue really has
5483
+ * been waiting that long, and the escalating warning should keep saying so.
5484
+ */
5485
+ #resetDispatchCapacityBackoff() {
5486
+ if (this.#stopping || this.#dispatchLifecycleCapacityWaits.size === 0)
5487
+ return;
5488
+ for (const [key, wait] of this.#dispatchLifecycleCapacityWaits) {
5489
+ wait.attempts = 0;
5490
+ const timer = this.#dispatchLifecycleRetryTimers.get(key);
5491
+ if (!timer)
5492
+ continue;
5493
+ clearTimeout(timer);
5494
+ this.#dispatchLifecycleRetryTimers.delete(key);
5495
+ this.#scheduleDispatchLifecycleRetry(wait.record);
5496
+ }
5497
+ this.#increment('dispatchCapacityBackoffResets');
5498
+ }
5499
+ #scheduleDispatchLifecycleRetry(record, delayMs = DISPATCH_LIFECYCLE_RETRY_MS) {
4738
5500
  const key = issueKey(record.issue);
4739
5501
  if (this.#stopping || this.#dispatchLifecycleRetryTimers.has(key))
4740
5502
  return;
@@ -4742,23 +5504,17 @@ export class FactoryLoop {
4742
5504
  this.#dispatchLifecycleRetryTimers.delete(key);
4743
5505
  const drive = this.#driveDispatchLifecycle(key)
4744
5506
  .then(() => {
4745
- this.#dispatchLifecycleCapacityWaitLogged.delete(key);
5507
+ this.#dispatchLifecycleCapacityWaits.delete(key);
4746
5508
  this.#dispatchLifecycleOwnershipWaitLogged.delete(key);
4747
5509
  })
4748
5510
  .catch((error) => {
5511
+ let nextDelayMs = DISPATCH_LIFECYCLE_RETRY_MS;
4749
5512
  if (error instanceof DispatchLifecycleCapacityError) {
4750
5513
  this.#dispatchLifecycleOwnershipWaitLogged.delete(key);
4751
- if (!this.#dispatchLifecycleCapacityWaitLogged.has(key)) {
4752
- this.#dispatchLifecycleCapacityWaitLogged.add(key);
4753
- this.#increment('dispatchLifecycleCapacityWaits');
4754
- this.#logger.warn?.('[factory] durable dispatch is queued for batch capacity; retries remain active', {
4755
- issue: record.issue.key,
4756
- retryMs: DISPATCH_LIFECYCLE_RETRY_MS,
4757
- });
4758
- }
5514
+ nextDelayMs = this.#recordDispatchCapacityWait(record, key);
4759
5515
  }
4760
5516
  else if (error instanceof DispatchLifecycleOwnedElsewhereError) {
4761
- this.#dispatchLifecycleCapacityWaitLogged.delete(key);
5517
+ this.#dispatchLifecycleCapacityWaits.delete(key);
4762
5518
  if (!this.#dispatchLifecycleOwnershipWaitLogged.has(key)) {
4763
5519
  this.#dispatchLifecycleOwnershipWaitLogged.add(key);
4764
5520
  this.#increment('dispatchLifecycleOwnershipWaits');
@@ -4772,18 +5528,18 @@ export class FactoryLoop {
4772
5528
  }
4773
5529
  }
4774
5530
  else {
4775
- this.#dispatchLifecycleCapacityWaitLogged.delete(key);
5531
+ this.#dispatchLifecycleCapacityWaits.delete(key);
4776
5532
  this.#dispatchLifecycleOwnershipWaitLogged.delete(key);
4777
5533
  this.#logger.warn?.('[factory] durable dispatch lifecycle retry failed', {
4778
5534
  issue: record.issue.key,
4779
5535
  error: describeError(error).errorMessage,
4780
5536
  });
4781
5537
  }
4782
- this.#scheduleDispatchLifecycleRetry(record);
5538
+ this.#scheduleDispatchLifecycleRetry(record, nextDelayMs);
4783
5539
  })
4784
5540
  .finally(() => this.#dispatchLifecycleDrives.delete(drive));
4785
5541
  this.#dispatchLifecycleDrives.add(drive);
4786
- }, DISPATCH_LIFECYCLE_RETRY_MS);
5542
+ }, delayMs);
4787
5543
  this.#dispatchLifecycleRetryTimers.set(key, timer);
4788
5544
  }
4789
5545
  #scheduleReleaseRetry(record, reason) {
@@ -6299,8 +7055,32 @@ export class FactoryLoop {
6299
7055
  registryPath,
6300
7056
  eventListener: this.#eventListenerStatus(),
6301
7057
  readinessReconcile: this.#readinessReconcileStatus(),
7058
+ dispatchCapacity: this.#dispatchCapacityStatus(),
6302
7059
  fleetControlPlane: this.#fleetControlPlane.status(),
6303
7060
  };
7061
+ // The deployed container serves `/healthz` straight out of this file and
7062
+ // has no redaction logic of its own, so publish the already-safe view here
7063
+ // rather than leaving that boundary to whoever reads the file (#295).
7064
+ // Derived against this daemon's clock: every duration in it is a
7065
+ // difference between timestamps this process wrote.
7066
+ //
7067
+ // Guarded (#300 review, CodeRabbit): this heartbeat is what the crash
7068
+ // reaper and `/healthz` read to decide the daemon is alive, and several
7069
+ // callers of this method sit outside any try/catch. A projection failure
7070
+ // must cost the diagnostics block, never the heartbeat — the omitted block
7071
+ // is itself legible, since `factory diagnose` reports a missing one rather
7072
+ // than a false green.
7073
+ try {
7074
+ heartbeat.health = publicHealthFromHeartbeat(heartbeat, {
7075
+ nowMs: updatedAtMs,
7076
+ staleMs: this.#config.loop.heartbeatStaleMs,
7077
+ });
7078
+ }
7079
+ catch (error) {
7080
+ this.#logger.warn?.('[factory] public health projection failed; heartbeat written without it', {
7081
+ error: describeError(error).errorMessage,
7082
+ });
7083
+ }
6304
7084
  await mkdir(dirname(path), { recursive: true });
6305
7085
  await writeFile(path, `${JSON.stringify(heartbeat, null, 2)}\n`, 'utf8');
6306
7086
  await this.#writeInFlightRegistry(registryPath, path);
@@ -6319,11 +7099,18 @@ export class FactoryLoop {
6319
7099
  });
6320
7100
  }
6321
7101
  async #reapDispatchFailureHandoffsNow(heartbeatPath = this.#loopReapPaths?.heartbeatPath ?? this.#config.loop.heartbeatPath, registryPath = this.#loopReapPaths?.registryPath ?? this.#config.loop.registryPath) {
6322
- const handoffs = await this.#state.listFailureHandoffs(this.#workspaceId);
6323
- if (handoffs.length === 0) {
6324
- return;
6325
- }
6326
7102
  try {
7103
+ // Inside the try, not before it (#298 review). Every caller reaps while
7104
+ // already handling a failure and then propagates that failure: the
7105
+ // per-item catch rethrows the dispatch error, runLoop's catch is mid
7106
+ // teardown. A throw from here would REPLACE the error in flight — and a
7107
+ // replaced 429 is no longer recognised as overload at the discovery
7108
+ // fence, silently dropping the advertised backoff. Reaping is
7109
+ // best-effort by construction; the caller's error always wins.
7110
+ const handoffs = await this.#state.listFailureHandoffs(this.#workspaceId);
7111
+ if (handoffs.length === 0) {
7112
+ return;
7113
+ }
6327
7114
  const protectedPids = await this.#protectedPids();
6328
7115
  let registryChanged = false;
6329
7116
  const readyToClear = new Set();
@@ -6699,10 +7486,12 @@ export class FactoryLoop {
6699
7486
  }
6700
7487
  return [...handoffs.values()];
6701
7488
  }
6702
- async #teardownFailedDispatchWorktrees(handoffs, releaseReason = 'dispatch failed') {
7489
+ async #teardownFailedDispatchWorktrees(handoffs, releaseReason = 'dispatch failed', opts = {}) {
6703
7490
  if (!this.#worktrees || !handoffs.some((handoff) => handoff.worktree))
6704
7491
  return false;
6705
- const failed = await this.#releaseAndTerminateAgents(handoffs.map((handoff) => [handoff.name, handoff.tracked]), releaseReason, 'completion');
7492
+ const failed = await this.#releaseAndTerminateAgents(handoffs
7493
+ .filter((handoff) => !opts.skipNeverPlacedAgents || handoff.tracked.result !== undefined)
7494
+ .map((handoff) => [handoff.name, handoff.tracked]), releaseReason, 'completion');
6706
7495
  if (failed.length > 0)
6707
7496
  return false;
6708
7497
  try {
@@ -6878,6 +7667,17 @@ export class FactoryLoop {
6878
7667
  : 'agent_spawn_failed',
6879
7668
  });
6880
7669
  }
7670
+ // The never-placed deadline can fire while this spawn is in flight — that
7671
+ // is the whole point of arming it before the first await, and it makes a
7672
+ // late `spawn` result newly reachable (#303 review, cubic). By now the
7673
+ // reaper may have fenced, released and terminalized the lifecycle, so this
7674
+ // placement belongs to nothing: recording it would attach a live worker to
7675
+ // a record the reaper has finished with, and nothing downstream would ever
7676
+ // release it. Hand it straight to teardown instead.
7677
+ if (!await this.#dispatchLifecycleStillOwned(record)) {
7678
+ await this.#releaseOrphanedLatePlacement(record, spec, result);
7679
+ throw new LatePlacementReleasedError(record.issue.key, result.name ?? spec.name);
7680
+ }
6881
7681
  record.heldSinceAtMs ??= this.#clock.now();
6882
7682
  batch.recordSpawn(record, spec, invocationId, result);
6883
7683
  if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
@@ -6889,6 +7689,68 @@ export class FactoryLoop {
6889
7689
  await this.#reportAgent(record, spawned, 'agent.spawned');
6890
7690
  return { name: result.name };
6891
7691
  }
7692
+ /**
7693
+ * Is this process still the owner of a lifecycle that is not already done?
7694
+ *
7695
+ * Cheap local checks first — a pending abandon reason, or a dropped epoch,
7696
+ * both of which the reaper sets before anything durable is re-read — then the
7697
+ * durable row, which is authoritative when another owner terminalized it.
7698
+ */
7699
+ async #dispatchLifecycleStillOwned(record) {
7700
+ const key = issueKey(record.issue);
7701
+ if (this.#abandonedDispatchReasons.has(key))
7702
+ return false;
7703
+ if (!this.#usesDurableDispatchLifecycle())
7704
+ return true;
7705
+ const epoch = this.#dispatchLifecycleEpochs.get(key);
7706
+ if (epoch === undefined)
7707
+ return false;
7708
+ const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
7709
+ if (!lifecycle || isTerminalDispatchLifecycle(lifecycle))
7710
+ return false;
7711
+ // A live row is not the same as *our* row. Another owner can reclaim an
7712
+ // expired lease and leave it nonterminal, and the cached epoch here would
7713
+ // still say we hold it (#303 review, cubic). This mirrors exactly what
7714
+ // `saveDispatchLifecycle` will accept — owner, epoch and an unexpired
7715
+ // lease — so a placement is recorded only when the write that follows can
7716
+ // actually land. Otherwise the save fails and the worker leaks through the
7717
+ // generic ownership-lost path instead of orphan cleanup.
7718
+ const lease = lifecycle.lease;
7719
+ return lease !== undefined &&
7720
+ lease.owner === this.#dispatchLifecycleOwner &&
7721
+ lease.epoch === epoch &&
7722
+ lease.leaseUntilMs > this.#clock.now();
7723
+ }
7724
+ /**
7725
+ * Tear down a placement that landed after its lifecycle was already released.
7726
+ *
7727
+ * Deliberately not routed through `#abandonStuckDispatch`: that record is
7728
+ * terminal and its batch entry is gone, so there is nothing left to abandon.
7729
+ * The only thing that still exists is a live worker on the fleet.
7730
+ */
7731
+ async #releaseOrphanedLatePlacement(record, spec, result) {
7732
+ const name = result.name ?? spec.name;
7733
+ this.#increment('lateSpawnPlacementsReleased');
7734
+ this.#logger.warn?.('[factory] releasing an agent that finished spawning after its dispatch was released', {
7735
+ issue: record.issue.key,
7736
+ agent: name,
7737
+ role: spec.role,
7738
+ });
7739
+ this.#fleet.markAgentTerminal?.(name, 'dispatch-released-before-placement');
7740
+ try {
7741
+ await this.#fleet.release(name, 'dispatch-released-before-placement');
7742
+ }
7743
+ catch (error) {
7744
+ // The reaper handoff owns anything this could not clean up; failing here
7745
+ // would only replace a released worker with an unreleased one.
7746
+ this.#increment('lateSpawnPlacementReleaseFailures');
7747
+ this.#logger.warn?.('[factory] failed to release a late placement; leaving it to the orphan reaper', {
7748
+ issue: record.issue.key,
7749
+ agent: name,
7750
+ error: describeError(error).errorMessage,
7751
+ });
7752
+ }
7753
+ }
6892
7754
  async #handleAgentExit(name, reason) {
6893
7755
  if (this.#stopping) {
6894
7756
  return;
@@ -7943,7 +8805,14 @@ export class FactoryLoop {
7943
8805
  this.#scheduleAbandonedDispatchRetry(record, reason);
7944
8806
  return;
7945
8807
  }
7946
- const agents = [...record.agents];
8808
+ // A never-placed record carries specs, not workers: `recordPlanned` writes
8809
+ // the spec before the spawn returns, so a dispatch that died mid-spawn
8810
+ // leaves a name the broker never issued. Releasing one fails, which fails
8811
+ // the whole cleanup and re-arms the abandon retry forever — turning the
8812
+ // #303 reap into a second, quieter wedge. Their worktrees are still torn
8813
+ // down below.
8814
+ const neverPlaced = reason === AGENTLESS_SLOT_PAST_DEADLINE_RELEASE_REASON;
8815
+ const agents = [...record.agents].filter(([, tracked]) => !neverPlaced || tracked.result !== undefined);
7947
8816
  for (const [agentName, tracked] of agents) {
7948
8817
  if (!heldPastDeadline && tracked.spec.role === 'implementer')
7949
8818
  continue;
@@ -7963,7 +8832,7 @@ export class FactoryLoop {
7963
8832
  const failed = await this.#releaseAndTerminateAgents(nonWorktreeAgents, agentReleaseReason, 'completion');
7964
8833
  cleanupComplete = failed.length === 0;
7965
8834
  }
7966
- cleanupComplete = await this.#teardownFailedDispatchWorktrees(worktreeHandoffs, agentReleaseReason) && cleanupComplete;
8835
+ cleanupComplete = await this.#teardownFailedDispatchWorktrees(worktreeHandoffs, agentReleaseReason, { skipNeverPlacedAgents: neverPlaced }) && cleanupComplete;
7967
8836
  }
7968
8837
  else if (agents.length > 0) {
7969
8838
  const failed = await this.#releaseAndTerminateAgents(agents, agentReleaseReason, 'completion');
@@ -14941,6 +15810,15 @@ const githubIssueAuthor = (issue) => {
14941
15810
  }
14942
15811
  return source ? undefined : githubAuthorLogin(payload)?.trim() || undefined;
14943
15812
  };
15813
+ /**
15814
+ * An `IssueRef` for a path whose issue body could not be read at all — the
15815
+ * shape a relayfile-shed ready-issue read leaves behind (#297). The key is
15816
+ * what an operator needs to correlate the skip; the uuid falls back to it.
15817
+ */
15818
+ const issueRefFromPath = (path) => {
15819
+ const key = keyFromPath(path);
15820
+ return { uuid: uuidFromPath(path) ?? key, key, path };
15821
+ };
14944
15822
  const issueRef = (issue) => ({ uuid: issue.uuid, key: issue.key, path: issue.path });
14945
15823
  // Preserve the historical Linear state namespace while keeping GitHub-native
14946
15824
  // issue numbers independent across repositories in the same workspace.
@@ -16854,10 +17732,54 @@ const relayfileOverload = (error) => {
16854
17732
  stringValue(flat.code) ?? stringValue(data.code) ?? 'rate_limited';
16855
17733
  return { status, reason, ...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }) };
16856
17734
  };
17735
+ /**
17736
+ * relayfile's overload reason codes, allowlisted.
17737
+ *
17738
+ * `IterationReport.skipped[].reason` is serialized to stdout by `factory
17739
+ * run-once`, so it stays a fixed classification plus a known code — the same
17740
+ * public-surface rule #293 applied to error class names — rather than
17741
+ * whatever string the dependency happened to send.
17742
+ */
17743
+ const RELAYFILE_OVERLOAD_REASONS = new Set([
17744
+ // Admission gate inside the workspace durable object.
17745
+ 'inflight_limit',
17746
+ 'oldest_inflight_age',
17747
+ 'write_admission_limit',
17748
+ // Worker-side backpressure, per isolate rather than per workspace.
17749
+ 'router_inflight_limit',
17750
+ // The Cloudflare runtime shed the object; relayfile only relabels it.
17751
+ 'durable_object_overloaded',
17752
+ // relayfileOverload()'s fallback when the body carried no reason at all.
17753
+ 'rate_limited',
17754
+ ]);
17755
+ /**
17756
+ * Both the run-report reason and the per-reason counter key are built from
17757
+ * this, so an unknown code from the dependency can neither leak into stdout
17758
+ * nor open an unbounded counter namespace.
17759
+ */
17760
+ const relayfileOverloadReasonLabel = (reason) => RELAYFILE_OVERLOAD_REASONS.has(reason) ? reason : 'unrecognized';
17761
+ /**
17762
+ * How long to wait before the next discovery sweep after relayfile shed this
17763
+ * one.
17764
+ *
17765
+ * The advertised `Retry-After` is authoritative in BOTH directions (#297):
17766
+ * it is the first rung, so we never retry sooner than the dependency allows,
17767
+ * and it bounds the ceiling, so we never sleep for minutes because of a
17768
+ * request to wait seconds. Without an advertised delay there is nothing to
17769
+ * respect and the original five-minute ladder governs unchanged.
17770
+ */
16857
17771
  const discoveryOverloadBackoffMs = (retryAfterSeconds, consecutiveOverloads) => {
16858
- const retryAfterMs = Math.max(0, Math.ceil((retryAfterSeconds ?? 0) * 1_000));
16859
- const exponentialMs = Math.min(DISCOVERY_OVERLOAD_BACKOFF_MAX_MS, 5_000 * (2 ** Math.min(10, Math.max(0, consecutiveOverloads - 1))));
16860
- return Math.max(retryAfterMs, exponentialMs);
17772
+ const advertisedMs = retryAfterSeconds === undefined
17773
+ ? undefined
17774
+ : Math.max(DISCOVERY_OVERLOAD_BACKOFF_MIN_MS, Math.ceil(retryAfterSeconds * 1_000));
17775
+ const baseMs = advertisedMs ?? DISCOVERY_OVERLOAD_BACKOFF_BASE_MS;
17776
+ // A dependency that asks for longer than the advertised ceiling still gets
17777
+ // what it asked for; the ceiling only stops the ladder from overshooting it.
17778
+ const ceilingMs = advertisedMs === undefined
17779
+ ? DISCOVERY_OVERLOAD_BACKOFF_MAX_MS
17780
+ : Math.max(advertisedMs, DISCOVERY_OVERLOAD_ADVERTISED_BACKOFF_MAX_MS);
17781
+ const steps = Math.min(10, Math.max(0, consecutiveOverloads - 1));
17782
+ return Math.min(ceilingMs, baseMs * (2 ** steps));
16861
17783
  };
16862
17784
  const eventSequenceNumber = (eventId) => {
16863
17785
  const whole = Number(eventId);
@@ -17086,10 +18008,6 @@ const telemetryCategory = (value) => {
17086
18008
  const normalized = value.trim().toLowerCase().replace(/[^a-z0-9._:/-]+/gu, '-');
17087
18009
  return normalized.slice(0, 120) || undefined;
17088
18010
  };
17089
- const telemetryErrorClass = (error) => {
17090
- const name = error instanceof Error ? error.name : '';
17091
- return /^[A-Za-z][A-Za-z0-9]{0,63}(?:Error|Exception)$/u.test(name) ? name : 'Error';
17092
- };
17093
18011
  const isTimeoutError = (error) => error instanceof Error && (error.name === 'TimeoutError' || error.name === 'AbortError');
17094
18012
  const retryOnTimeout = async (fn, opts) => {
17095
18013
  let lastError;
@@ -17236,6 +18154,7 @@ const lifecycleFromInFlightRecord = (record, runId, phase, updatedAtMs, pullRequ
17236
18154
  ...(releaseReason ? { releaseReason } : {}),
17237
18155
  ...(cost ? { cost: structuredClone(cost) } : {}),
17238
18156
  ...(record.heldSinceAtMs !== undefined ? { heldSinceAtMs: record.heldSinceAtMs } : {}),
18157
+ ...(record.slotHeldSinceAtMs !== undefined ? { slotHeldSinceAtMs: record.slotHeldSinceAtMs } : {}),
17239
18158
  updatedAtMs,
17240
18159
  });
17241
18160
  const inFlightRecordFromLifecycle = (lifecycle) => ({
@@ -17252,9 +18171,14 @@ const inFlightRecordFromLifecycle = (lifecycle) => ({
17252
18171
  invocationIds: new Set(lifecycle.invocationIds),
17253
18172
  result: lifecycle.result ? structuredClone(lifecycle.result) : undefined,
17254
18173
  ...(lifecycle.dispatchClaim ? { dispatchClaim: { ...lifecycle.dispatchClaim } } : {}),
17255
- heldSinceAtMs: lifecycle.heldSinceAtMs ?? (lifecycle.agents.some((agent) => agent.releasedAtMs === undefined)
18174
+ heldSinceAtMs: lifecycle.heldSinceAtMs ?? (
18175
+ // A live placement the durable row predates the `heldSinceAtMs` field for.
18176
+ // `tracked.result` is what distinguishes a placement from a spec that
18177
+ // `recordPlanned` wrote and no spawn ever answered (#303).
18178
+ lifecycle.agents.some((agent) => agent.releasedAtMs === undefined && agent.tracked.result !== undefined)
17256
18179
  ? lifecycle.updatedAtMs
17257
18180
  : undefined),
18181
+ slotHeldSinceAtMs: lifecycle.slotHeldSinceAtMs,
17258
18182
  lifecyclePhase: lifecycle.phase,
17259
18183
  });
17260
18184
  const dispatchResultFromLifecycle = (lifecycle) => lifecycle.result ? structuredClone(lifecycle.result) : {
@@ -17290,6 +18214,29 @@ export class LiveDispatchStateChangedError extends Error {
17290
18214
  this.issueKey = issueKey;
17291
18215
  }
17292
18216
  }
18217
+ /**
18218
+ * A placement that finished spawning after its dispatch had been released.
18219
+ *
18220
+ * The never-placed deadline (#303) can terminalize a lifecycle while
18221
+ * `#fleet.spawn` is still in flight; the worker is released and the dispatch
18222
+ * unwinds. That is a known, named, self-healing race — the issue returns to the
18223
+ * queue and is re-dispatched — so it must be classified rather than counted as
18224
+ * an unexplained fault. It fires precisely under slow spawns, which is the
18225
+ * condition the deadline exists for, so a degraded fleet produces it
18226
+ * repeatedly; left unclassified, five in a row would trip
18227
+ * `UNCLASSIFIED_DISPATCH_FAILURE_LIMIT` and abort the whole readiness pass,
18228
+ * turning a bounded slot into a stopped sweep (#303 review, factory-lead).
18229
+ */
18230
+ export class LatePlacementReleasedError extends Error {
18231
+ issueKey;
18232
+ agentName;
18233
+ constructor(issueKey, agentName) {
18234
+ super(`Dispatch lifecycle for ${issueKey} was released while ${agentName} was still spawning`);
18235
+ this.name = 'LatePlacementReleasedError';
18236
+ this.issueKey = issueKey;
18237
+ this.agentName = agentName;
18238
+ }
18239
+ }
17293
18240
  /** Whether a thrown value is a {@link LiveDispatchStateChangedError}. */
17294
18241
  export function isLiveDispatchStateChangedError(error) {
17295
18242
  return error instanceof LiveDispatchStateChangedError;
@@ -17323,7 +18270,32 @@ const UNCLASSIFIED_DISPATCH_FAILURE_LIMIT = 5;
17323
18270
  * skippable and are exempt from the consecutive-failure fuse.
17324
18271
  */
17325
18272
  const isClassifiedPerItemDispatchFailure = (error) => error instanceof LiveDispatchStateChangedError ||
17326
- error instanceof DispatchLifecycleClaimRefusedError;
18273
+ error instanceof DispatchLifecycleClaimRefusedError ||
18274
+ // #303: the never-placed deadline released this dispatch while its spawn was
18275
+ // still in flight. Named, expected and self-healing — the issue goes back to
18276
+ // the queue — and it recurs under exactly the slow-spawn conditions the
18277
+ // deadline exists for, so leaving it unclassified would let a degraded fleet
18278
+ // trip the pass-abort fuse. Its own counters keep it visible.
18279
+ error instanceof LatePlacementReleasedError ||
18280
+ // Relayfile shedding one operation is a state of the dependency, not an
18281
+ // unexplained fault, and it has its own fuse — see #297 and
18282
+ // DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT.
18283
+ relayfileOverload(error) !== undefined;
18284
+ /**
18285
+ * Whether a per-item failure could have left half-spawned agents behind.
18286
+ *
18287
+ * The two lifecycle refusals are decided before `#dispatchUnlocked` spawns
18288
+ * anything, so there is nothing to reap for them. Everything else can fail
18289
+ * *after* a spawn — including relayfile shedding a post-spawn read — having
18290
+ * persisted the agents as failure handoffs on the way out.
18291
+ *
18292
+ * Deliberately a denylist rather than an allowlist: a new failure mode that
18293
+ * nobody classified should default to "reap it", because the cost of a
18294
+ * needless reap is one no-op pass over an empty handoff list, while the cost
18295
+ * of a missed one is leaked agents and duplicate workers on the next retry.
18296
+ */
18297
+ const mayHaveSpawnedBeforeFailing = (error) => !(error instanceof LiveDispatchStateChangedError) &&
18298
+ !(error instanceof DispatchLifecycleClaimRefusedError);
17327
18299
  /**
17328
18300
  * The run-report reason recorded for a work unit the pass could not dispatch.
17329
18301
  *
@@ -17334,8 +18306,13 @@ const isClassifiedPerItemDispatchFailure = (error) => error instanceof LiveDispa
17334
18306
  * `describeControlPlaneError` makes for circuit state.
17335
18307
  */
17336
18308
  const perItemDispatchSkipReason = (error) => {
18309
+ const overload = relayfileOverload(error);
18310
+ if (overload)
18311
+ return `relayfile overloaded (${relayfileOverloadReasonLabel(overload.reason)})`;
17337
18312
  if (error instanceof LiveDispatchStateChangedError)
17338
18313
  return 'live state changed during dispatch';
18314
+ if (error instanceof LatePlacementReleasedError)
18315
+ return 'dispatch released while its agent was still spawning';
17339
18316
  if (error instanceof DispatchLifecycleClaimRefusedError) {
17340
18317
  return error.refusal === 'terminal'
17341
18318
  ? 'dispatch lifecycle already terminal'