@agent-relay/factory 0.1.64 → 0.1.65

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/README.md +9 -0
  2. package/dist/cli/diagnose.d.ts +92 -0
  3. package/dist/cli/diagnose.d.ts.map +1 -0
  4. package/dist/cli/diagnose.js +346 -0
  5. package/dist/cli/diagnose.js.map +1 -0
  6. package/dist/cli/fleet.d.ts +9 -1
  7. package/dist/cli/fleet.d.ts.map +1 -1
  8. package/dist/cli/fleet.js +87 -4
  9. package/dist/cli/fleet.js.map +1 -1
  10. package/dist/config/schema.d.ts +69 -2
  11. package/dist/config/schema.d.ts.map +1 -1
  12. package/dist/config/schema.js +32 -0
  13. package/dist/config/schema.js.map +1 -1
  14. package/dist/fleet/internal-fleet-client.d.ts +16 -0
  15. package/dist/fleet/internal-fleet-client.d.ts.map +1 -1
  16. package/dist/fleet/internal-fleet-client.js +228 -13
  17. package/dist/fleet/internal-fleet-client.js.map +1 -1
  18. package/dist/hosted/orchestrator.d.ts.map +1 -1
  19. package/dist/hosted/orchestrator.js +1 -4
  20. package/dist/hosted/orchestrator.js.map +1 -1
  21. package/dist/index.d.ts +1 -1
  22. package/dist/index.d.ts.map +1 -1
  23. package/dist/index.js +1 -1
  24. package/dist/index.js.map +1 -1
  25. package/dist/observability/error-class.d.ts +29 -0
  26. package/dist/observability/error-class.d.ts.map +1 -0
  27. package/dist/observability/error-class.js +36 -0
  28. package/dist/observability/error-class.js.map +1 -0
  29. package/dist/observability/index.d.ts +1 -0
  30. package/dist/observability/index.d.ts.map +1 -1
  31. package/dist/observability/index.js +1 -0
  32. package/dist/observability/index.js.map +1 -1
  33. package/dist/orchestrator/factory.d.ts.map +1 -1
  34. package/dist/orchestrator/factory.js +649 -46
  35. package/dist/orchestrator/factory.js.map +1 -1
  36. package/dist/orchestrator/index.d.ts +1 -0
  37. package/dist/orchestrator/index.d.ts.map +1 -1
  38. package/dist/orchestrator/index.js +1 -0
  39. package/dist/orchestrator/index.js.map +1 -1
  40. package/dist/orchestrator/public-health.d.ts +82 -0
  41. package/dist/orchestrator/public-health.d.ts.map +1 -0
  42. package/dist/orchestrator/public-health.js +372 -0
  43. package/dist/orchestrator/public-health.js.map +1 -0
  44. package/dist/ports/state.d.ts +16 -0
  45. package/dist/ports/state.d.ts.map +1 -1
  46. package/dist/state/file-state-store.d.ts +8 -0
  47. package/dist/state/file-state-store.d.ts.map +1 -1
  48. package/dist/state/file-state-store.js +12 -2
  49. package/dist/state/file-state-store.js.map +1 -1
  50. package/dist/state/in-memory-state-store.d.ts +8 -0
  51. package/dist/state/in-memory-state-store.d.ts.map +1 -1
  52. package/dist/state/in-memory-state-store.js +12 -2
  53. package/dist/state/in-memory-state-store.js.map +1 -1
  54. package/dist/types.d.ts +106 -1
  55. package/dist/types.d.ts.map +1 -1
  56. package/package.json +1 -1
@@ -1,7 +1,7 @@
1
1
  import { randomUUID } from 'node:crypto';
2
2
  import { mkdir, readFile, writeFile } from 'node:fs/promises';
3
3
  import { dirname, isAbsolute, resolve } from 'node:path';
4
- import { FactoryConfigSchema } from '../config/schema.js';
4
+ import { DEFAULT_READINESS_RECONCILE_TIMEOUT_MS, FactoryConfigSchema } from '../config/schema.js';
5
5
  import { linearByStatePath, linearByIdPath, linearByUuidPath } from '../constants/linear.js';
6
6
  import { stateResolutionFromIds } from '../linear/state-resolver.js';
7
7
  import { GithubMergeGate, closeProbePr } from '../github/index.js';
@@ -29,6 +29,8 @@ import { CoalescedTaskQueue } from './coalesced-task-queue.js';
29
29
  import { findAgentProcessByName, readProcessIdentity } from './process-identity.js';
30
30
  import { readFactoryInFlightRegistry, terminatePids } from './reaper.js';
31
31
  import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../observability/events.js';
32
+ import { telemetryErrorClass } from '../observability/error-class.js';
33
+ import { derivedReadinessReconcileState, publicHealthFromHeartbeat, readinessReconcileInFlightMs, } from './public-health.js';
32
34
  import { boundedRunCostTotal, CostLedger } from '../cost/ledger.js';
33
35
  import { createTicketDispatchDelivery } from '../delivery/ticket-dispatch.js';
34
36
  import { canonicalTrajectorySessionRef, renderTrajectoryPointer, stripTrajectoryPointers, } from '../trajectory.js';
@@ -185,6 +187,38 @@ const DISCOVERY_SWEEP_RENEW_MS = 30_000;
185
187
  const READINESS_RECONCILE_FAILURE_THRESHOLD = 3;
186
188
  const DISCOVERY_CHANGE_EVENT_LIMIT = 1_000;
187
189
  const DISCOVERY_OVERLOAD_BACKOFF_MAX_MS = 5 * 60_000;
190
+ /** First rung of the ladder when the 429 advertises no `Retry-After`. */
191
+ const DISCOVERY_OVERLOAD_BACKOFF_BASE_MS = 5_000;
192
+ /**
193
+ * Floor for the advertised delay. `Retry-After: 0` would otherwise pin the
194
+ * whole ladder at zero (`0 * 2 ** n` is still zero) and turn respecting the
195
+ * dependency into hammering it.
196
+ */
197
+ const DISCOVERY_OVERLOAD_BACKOFF_MIN_MS = 1_000;
198
+ /**
199
+ * Ceiling for the ladder once the dependency has told us how long to wait.
200
+ *
201
+ * relayfile sheds an overloaded workspace DO with a 429 in milliseconds
202
+ * carrying `Retry-After: 5`, and #297 is what happened when the ladder ignored
203
+ * that and climbed to `DISCOVERY_OVERLOAD_BACKOFF_MAX_MS` anyway: the
204
+ * dependency asked for five seconds, Factory slept for five minutes, probed
205
+ * for recovery once per cap-length window, and presented a transient upstream
206
+ * blip as a sustained outage. The ratchet still escalates — it is just bounded
207
+ * by roughly what was actually asked for, and never *below* it, so this is a
208
+ * ceiling and not a licence to retry sooner than the dependency allows.
209
+ */
210
+ const DISCOVERY_OVERLOAD_ADVERTISED_BACKOFF_MAX_MS = 30_000;
211
+ /**
212
+ * How many relayfile operations one sweep may have shed before the sweep is
213
+ * abandoned rather than continued.
214
+ *
215
+ * Per-item overload skips that item and keeps going (#297, the same principle
216
+ * as #292), but skipping must not degenerate into grinding a shedding
217
+ * dependency through an entire backlog one 429 at a time. Past this many, the
218
+ * dependency is not serving this sweep at all: abort, back off, and let the
219
+ * ratchet do its job.
220
+ */
221
+ const DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT = 5;
188
222
  const GITHUB_FACTORY_LABEL = 'factory';
189
223
  const GITHUB_LIFECYCLE_LABELS = new Set(['factory:in-progress', 'factory:human-review']);
190
224
  const GITHUB_MIRROR_TITLE_PREFIX = '[factory]';
@@ -259,6 +293,20 @@ class DispatchLifecycleClaimRefusedError extends Error {
259
293
  this.name = 'DispatchLifecycleClaimRefusedError';
260
294
  }
261
295
  }
296
+ /**
297
+ * The deadline that makes a hung sweep reachable by the existing recovery path.
298
+ * Its message is fully internal (one integer), so it is safe to persist into
299
+ * the operator-facing `readinessReconcile.lastError`.
300
+ */
301
+ class ReadinessReconcileTimeoutError extends Error {
302
+ timeoutMs;
303
+ code = 'FACTORY_READINESS_RECONCILE_TIMEOUT';
304
+ constructor(timeoutMs) {
305
+ super(`readiness reconcile sweep exceeded its ${timeoutMs}ms deadline`);
306
+ this.timeoutMs = timeoutMs;
307
+ this.name = 'ReadinessReconcileTimeoutError';
308
+ }
309
+ }
262
310
  const realClock = {
263
311
  now: () => Date.now(),
264
312
  sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
@@ -408,12 +456,41 @@ export class FactoryLoop {
408
456
  #readinessReconcileTimer;
409
457
  #readinessReconcileInFlight;
410
458
  #readinessReconcileIntervalMs = 60_000;
459
+ #readinessReconcileTimeoutMs = DEFAULT_READINESS_RECONCILE_TIMEOUT_MS;
460
+ // Set for exactly as long as a sweep is running. `state` is derived from
461
+ // this, so an in-flight pass can no longer masquerade as the last settled one.
462
+ #readinessReconcileInFlightSinceMs;
463
+ /**
464
+ * The work a deadline gave up waiting on. The deadline bounds the wait, not
465
+ * the sweep, so this is still live: shutdown has to drain it, and `state` has
466
+ * to keep counting from when it actually started.
467
+ *
468
+ * Two fields rather than a collection, because every live abandoned wait
469
+ * converges on the same sweep. They are all `runOnce()` calls with the same
470
+ * `dryRun`, so whatever they are queued behind, the first one out starts the
471
+ * sweep and the rest coalesce onto it — they settle together. So the newest
472
+ * wait is a sufficient drain target, and the earliest start is the honest
473
+ * age. Both matter (#301 review): keeping only the newest start advanced the
474
+ * age by two intervals every two intervals, so at `reconcileTimeoutMs ===
475
+ * reconcileIntervalMs` it never reached three and `stalled` was never
476
+ * reported; keeping one record per wait grew without bound in exactly the
477
+ * never-settling case this change exists for.
478
+ *
479
+ * The wait, deliberately, and never `#runOnceInFlight`: a mismatched-`dryRun`
480
+ * sweep is waited BEHIND rather than coalesced onto, so that handle can name
481
+ * an unrelated sweep, and the readiness pass would then start its own work
482
+ * after shutdown believed it had drained everything. The wait covers the
483
+ * queueing and the sweep it eventually runs, in every branch.
484
+ */
485
+ #readinessReconcileAbandonedWait;
486
+ #readinessReconcileAbandonedSinceMs;
411
487
  #readinessReconcileConsecutiveFailures = 0;
412
488
  #readinessReconcileLastDurationMs;
413
489
  #readinessReconcileLastStartedAtMs;
414
490
  #readinessReconcileLastCompletedAtMs;
415
491
  #readinessReconcileLastFailureAtMs;
416
492
  #readinessReconcileLastError;
493
+ #readinessReconcileLastErrorClass;
417
494
  #liveEventQueue = [];
418
495
  #liveEventDrainScheduled = false;
419
496
  #liveEventDrainActive = false;
@@ -501,7 +578,34 @@ export class FactoryLoop {
501
578
  // so the loop's catch no longer runs the failure-handoff reaper for it; the
502
579
  // pass reaps inline and must write to the same paths runLoop would.
503
580
  #loopReapPaths;
581
+ /**
582
+ * The first 429 relayfile raised during this sweep, kept for its
583
+ * `Retry-After` and reason when the sweep decides how long to back off.
584
+ *
585
+ * Before #297 this doubled as a sweep-wide kill switch: any 429 from any
586
+ * relayfile call latched here and `#runOnceWithDiscoveryFence` then threw it
587
+ * away along with everything the sweep had already accomplished. It is now
588
+ * only evidence, never a verdict — see `#discoverySweepOverloads` for the
589
+ * fuse that still ends a sweep the dependency is genuinely refusing to serve.
590
+ */
504
591
  #discoveryOverloadError;
592
+ /** Relayfile operations this sweep has been shed on. */
593
+ #discoverySweepOverloads = 0;
594
+ /**
595
+ * The longest `Retry-After` any operation in this sweep advertised.
596
+ *
597
+ * `#discoveryOverloadError` latches the FIRST 429, so deriving the backoff
598
+ * from it alone would let the durable window expire before a later, longer
599
+ * advertised delay permits — breaking the very guarantee #297 is about. The
600
+ * backoff takes the maximum instead.
601
+ */
602
+ #discoverySweepRetryAfterSeconds;
603
+ /**
604
+ * Whether relayfile served at least one ready work unit end to end during
605
+ * this sweep — its issue read, or a dispatch built on it. This is what
606
+ * decays the durable overload ratchet; see `#discoveryOverloadOutcome`.
607
+ */
608
+ #discoverySweepProgress = false;
505
609
  #resolvedIssueSource;
506
610
  #integrationInstructions;
507
611
  #integrationInstructionsRefresh;
@@ -917,6 +1021,14 @@ export class FactoryLoop {
917
1021
  clearTimeout(this.#previewSweepTimer);
918
1022
  this.#previewSweepTimer = undefined;
919
1023
  await this.#readinessReconcileInFlight;
1024
+ // #301 review: the deadline ends the *wait*, so `#readinessReconcileInFlight`
1025
+ // can settle with its `runOnce()` still live. Shutdown releases dispatch
1026
+ // lifecycle leases and disposes ports below, and `#isPassFatalFailure` only
1027
+ // fences a stopping sweep once something in it throws — so a sweep whose
1028
+ // dependency recovers cleanly would otherwise dispatch through torn-down
1029
+ // state. Draining here restores exactly the pre-deadline shutdown contract:
1030
+ // stop() outlives the sweep it started.
1031
+ await this.#readinessReconcileAbandonedWait;
920
1032
  await this.#previewSweepInFlight;
921
1033
  this.#stoppingHeartbeatRefreshActive = await this.#stopLiveHeartbeat('stopping');
922
1034
  try {
@@ -1067,6 +1179,9 @@ export class FactoryLoop {
1067
1179
  const options = this.#liveOptions(overrides);
1068
1180
  this.#liveTransport = options.transport;
1069
1181
  this.#readinessReconcileIntervalMs = options.reconcileIntervalMs;
1182
+ // `start()` overrides skip the schema's cross-field check, so re-apply its
1183
+ // floor here: a deadline under one interval would kill every pass.
1184
+ this.#readinessReconcileTimeoutMs = Math.max(options.reconcileTimeoutMs, options.reconcileIntervalMs);
1070
1185
  this.#liveConnectStartedAtMs = this.#clock.now();
1071
1186
  this.#liveReplaySkewMarginMs = options.replaySkewMarginMs;
1072
1187
  const highWatermark = await this.#currentEventHighWatermark();
@@ -1103,10 +1218,27 @@ export class FactoryLoop {
1103
1218
  this.#logger.info?.('[factory] running startup ready-issue backfill before draining buffered events', {
1104
1219
  highWatermarkRouteUnavailable: highWatermark.routeUnavailable,
1105
1220
  });
1221
+ // Review follow-up on #300 (P1, cubic). The startup backfill is a
1222
+ // discovery pass like any other, and it is the one most likely to hang:
1223
+ // #36 measured 61 minutes here while the Relayfile mirror hydrated on a
1224
+ // cold container. Stamping it means a wedged FIRST pass is visible as
1225
+ // in-flight, instead of leaving the timestamps empty and the derived
1226
+ // state reading `healthy` forever.
1227
+ //
1228
+ // Only the timestamps. `consecutiveFailures` and `lastError` belong to
1229
+ // the reconcile loop's own failure accounting, which owns the degraded
1230
+ // threshold and the #297 reason allowlist; a startup failure is already
1231
+ // counted by `liveStartupBackfillErrors` and reported through `#error`.
1232
+ const backfillStartedAtMs = this.#clock.now();
1233
+ this.#readinessReconcileLastStartedAtMs = backfillStartedAtMs;
1106
1234
  try {
1107
1235
  await this.runOnce();
1236
+ this.#readinessReconcileLastDurationMs = this.#elapsedSince(backfillStartedAtMs);
1237
+ this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
1108
1238
  }
1109
1239
  catch (error) {
1240
+ this.#readinessReconcileLastDurationMs = this.#elapsedSince(backfillStartedAtMs);
1241
+ this.#readinessReconcileLastFailureAtMs = this.#clock.now();
1110
1242
  // A startup backfill failure must not abort the daemon: log it and fall
1111
1243
  // back to the live event stream (plus any buffered events) instead of
1112
1244
  // leaving the factory down.
@@ -1198,6 +1330,7 @@ export class FactoryLoop {
1198
1330
  eventLimit: overrides.eventLimit ?? this.#config.liveSubscription.eventLimit,
1199
1331
  replaySkewMarginMs: overrides.replaySkewMarginMs ?? this.#config.liveSubscription.replaySkewMarginMs,
1200
1332
  reconcileIntervalMs: overrides.reconcileIntervalMs ?? this.#config.liveSubscription.reconcileIntervalMs,
1333
+ reconcileTimeoutMs: overrides.reconcileTimeoutMs ?? this.#config.liveSubscription.reconcileTimeoutMs,
1201
1334
  };
1202
1335
  }
1203
1336
  async #currentEventCursor(limit) {
@@ -1246,19 +1379,82 @@ export class FactoryLoop {
1246
1379
  }, delayMs);
1247
1380
  this.#readinessReconcileTimer.unref?.();
1248
1381
  }
1382
+ /**
1383
+ * Runs one sweep under a deadline (#296).
1384
+ *
1385
+ * The sweep itself cannot be cancelled — `runOnce()` owns a durable discovery
1386
+ * lease and abandoning it mid-flight is not safe — so expiry rejects *this*
1387
+ * wait and leaves the underlying pass to finish on its own. That is enough:
1388
+ * the rejection is what reaches the failure path, which re-arms the timer.
1389
+ * A later reconcile pass coalesces onto the still-running `runOnce()` and
1390
+ * fails on its own deadline too, so a persistent hang keeps counting up to
1391
+ * `degraded` instead of going quiet.
1392
+ */
1393
+ async #runOnceWithReadinessDeadline() {
1394
+ const timeoutMs = this.#readinessReconcileTimeoutMs;
1395
+ const startedAtMs = this.#clock.now();
1396
+ const sweep = this.runOnce();
1397
+ let timer;
1398
+ try {
1399
+ return await new Promise((resolve, reject) => {
1400
+ timer = setTimeout(() => {
1401
+ this.#increment('readinessReconcileDeadlineExceeded');
1402
+ if (this.#readinessReconcileAbandonedSinceMs === undefined) {
1403
+ // `state` ages from the FIRST wait that gave up on this work, not
1404
+ // from whenever the latest one began.
1405
+ this.#readinessReconcileAbandonedSinceMs = startedAtMs;
1406
+ // The abandoned pass is still running against the live control
1407
+ // plane. Report where it lands, so an operator can tell a
1408
+ // dependency that recovered late from one that never answered.
1409
+ // Attached once, so a wedge is reported once and not per expiry.
1410
+ void sweep.then((report) => this.#logger.warn?.('[factory] abandoned readiness sweep completed after its deadline', {
1411
+ timeoutMs,
1412
+ overrunMs: this.#elapsedSince(startedAtMs) - timeoutMs,
1413
+ dispatched: report.dispatched.length,
1414
+ }), (error) => this.#logger.warn?.('[factory] abandoned readiness sweep failed after its deadline', {
1415
+ timeoutMs,
1416
+ overrunMs: this.#elapsedSince(startedAtMs) - timeoutMs,
1417
+ error: describeError(error).errorMessage,
1418
+ })).catch(() => undefined);
1419
+ }
1420
+ // Newest wait wins as the drain target: it settles no earlier than
1421
+ // the ones before it, and clearing on it clears them all.
1422
+ const wait = sweep.catch(() => undefined).then(() => {
1423
+ if (this.#readinessReconcileAbandonedWait !== wait)
1424
+ return;
1425
+ this.#readinessReconcileAbandonedWait = undefined;
1426
+ this.#readinessReconcileAbandonedSinceMs = undefined;
1427
+ });
1428
+ this.#readinessReconcileAbandonedWait = wait;
1429
+ reject(new ReadinessReconcileTimeoutError(timeoutMs));
1430
+ }, timeoutMs);
1431
+ timer.unref?.();
1432
+ // Attaching handlers here is also what keeps a late rejection from the
1433
+ // abandoned pass from surfacing as an unhandled rejection.
1434
+ sweep.then(resolve, reject);
1435
+ });
1436
+ }
1437
+ finally {
1438
+ if (timer)
1439
+ clearTimeout(timer);
1440
+ }
1441
+ }
1249
1442
  async #reconcileReadyIssues() {
1250
1443
  const startedAtMs = this.#clock.now();
1251
1444
  this.#readinessReconcileLastStartedAtMs = startedAtMs;
1445
+ this.#readinessReconcileInFlightSinceMs = startedAtMs;
1252
1446
  this.#increment('readinessReconcileSweeps');
1253
1447
  this.#logger.info?.('[factory] periodic readiness reconciliation started', {
1254
1448
  intervalMs: this.#readinessReconcileIntervalMs,
1449
+ timeoutMs: this.#readinessReconcileTimeoutMs,
1255
1450
  });
1256
1451
  try {
1257
- const report = await this.runOnce();
1452
+ const report = await this.#runOnceWithReadinessDeadline();
1258
1453
  this.#readinessReconcileConsecutiveFailures = 0;
1259
1454
  this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
1260
1455
  this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
1261
1456
  this.#readinessReconcileLastError = undefined;
1457
+ this.#readinessReconcileLastErrorClass = undefined;
1262
1458
  this.#logger.info?.('[factory] periodic readiness reconciliation completed', {
1263
1459
  durationMs: this.#readinessReconcileLastDurationMs,
1264
1460
  candidates: report.pulled.length,
@@ -1267,11 +1463,28 @@ export class FactoryLoop {
1267
1463
  });
1268
1464
  }
1269
1465
  catch (error) {
1270
- const errorMessage = describeError(error).errorMessage;
1466
+ // #297: all four relayfile overload reason codes share one message, and
1467
+ // `lastError` is what an operator reads from /evidence. Without the
1468
+ // reason, "workspace durable object is busy" cannot be told apart from
1469
+ // three other conditions with three different remedies.
1470
+ //
1471
+ // Allowlisted, because this is a persisted operator-facing surface and
1472
+ // not just a log line: `lastError` is returned from `status()` and
1473
+ // written into the loop heartbeat file, so an unbounded
1474
+ // dependency-controlled string would land on disk.
1475
+ const overload = relayfileOverload(error);
1476
+ const errorMessage = overload
1477
+ ? `${describeError(error).errorMessage} ` +
1478
+ `[relayfile ${overload.status} ${relayfileOverloadReasonLabel(overload.reason)}` +
1479
+ `${overload.retryAfterSeconds === undefined ? '' : `; retry-after=${overload.retryAfterSeconds}s`}]`
1480
+ : describeError(error).errorMessage;
1271
1481
  this.#readinessReconcileConsecutiveFailures += 1;
1272
1482
  this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
1273
1483
  this.#readinessReconcileLastFailureAtMs = this.#clock.now();
1274
1484
  this.#readinessReconcileLastError = errorMessage;
1485
+ // The class, unlike the message, is publishable: #295 puts it on the
1486
+ // unauthenticated health surface through the same allowlist.
1487
+ this.#readinessReconcileLastErrorClass = telemetryErrorClass(error);
1275
1488
  this.#increment('readinessReconcileErrors');
1276
1489
  this.#logger.warn?.('[factory] periodic readiness reconciliation failed; retry remains scheduled', {
1277
1490
  error: errorMessage,
@@ -1280,6 +1493,11 @@ export class FactoryLoop {
1280
1493
  degraded: this.#readinessReconcileConsecutiveFailures >= READINESS_RECONCILE_FAILURE_THRESHOLD,
1281
1494
  });
1282
1495
  }
1496
+ finally {
1497
+ // Cleared before the heartbeat write below, so a slow-but-successful pass
1498
+ // does not stamp its own tail as `stalled`.
1499
+ this.#readinessReconcileInFlightSinceMs = undefined;
1500
+ }
1283
1501
  await this.#refreshLiveHeartbeat();
1284
1502
  }
1285
1503
  #scheduleLivePoll(delayMs, options) {
@@ -2008,13 +2226,29 @@ export class FactoryLoop {
2008
2226
  this.#discoverySweepStartedAtMs = sweepStartedAtMs;
2009
2227
  this.#discoverySweepLeaseLost = false;
2010
2228
  this.#discoveryOverloadError = undefined;
2229
+ this.#discoverySweepOverloads = 0;
2230
+ this.#discoverySweepRetryAfterSeconds = undefined;
2231
+ this.#discoverySweepProgress = false;
2011
2232
  this.#startDiscoverySweepRenewal(claim.lease.epoch);
2012
2233
  let leaseReleased = false;
2013
2234
  try {
2014
2235
  this.#discoverySession = await this.#prepareDiscoverySession(claim);
2236
+ // #297: a 429 raised anywhere in the sweep used to latch and be rethrown
2237
+ // here, discarding a completed pass — every issue read, every dispatch —
2238
+ // because of one transient shed operation. The work this sweep did is
2239
+ // now kept instead, and the ratchet below records that the dependency is
2240
+ // shedding but still serving.
2015
2241
  const report = await this.#performRunOnce(opts);
2016
- if (this.#discoveryOverloadError)
2242
+ // The exception, and the reason skipping shed units cannot make a sweep
2243
+ // unconditionally green: a sweep that was shed AND got no work unit
2244
+ // through accomplished nothing. There is no progress to preserve, and
2245
+ // committing it would report a clean sweep over a dependency that served
2246
+ // none of it — leaving `readinessReconcile` healthy while Factory
2247
+ // dispatches nothing, which is the #292 wedge wearing the other costume.
2248
+ // Fail it so the ratchet escalates and readiness reflects reality.
2249
+ if (this.#discoveryOverloadError !== undefined && !this.#discoverySweepProgress) {
2017
2250
  throw this.#discoveryOverloadError;
2251
+ }
2018
2252
  const checkpoint = await this.#finalizeDiscoveryCheckpoint();
2019
2253
  // Do not clear the durable lease while a renewal can still be waiting on
2020
2254
  // the same state-file lock. A late renewal that observes the completed
@@ -2024,7 +2258,8 @@ export class FactoryLoop {
2024
2258
  if (this.#discoverySweepLeaseLost) {
2025
2259
  throw new Error('discovery sweep lease was lost before checkpoint commit');
2026
2260
  }
2027
- const completed = await this.#state.completeDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, checkpoint);
2261
+ const residual = this.#discoveryOverloadOutcome(claim.state.consecutiveOverloads, 'committed');
2262
+ const completed = await this.#commitDiscoverySweep(claim.lease.epoch, checkpoint, residual);
2028
2263
  leaseReleased = completed;
2029
2264
  if (!completed)
2030
2265
  throw new Error('discovery sweep lease was lost before completion');
@@ -2040,18 +2275,18 @@ export class FactoryLoop {
2040
2275
  await this.#stopDiscoverySweepRenewal();
2041
2276
  const overload = relayfileOverload(error);
2042
2277
  if (overload) {
2043
- const consecutiveOverloads = claim.state.consecutiveOverloads + 1;
2044
- const delayMs = discoveryOverloadBackoffMs(overload.retryAfterSeconds, consecutiveOverloads);
2045
- const backoffUntilMs = this.#clock.now() + delayMs;
2046
- leaseReleased = await this.#state.deferDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, backoffUntilMs, consecutiveOverloads);
2047
- this.#increment('discoveryOverloadBackoffs');
2278
+ const outcome = this.#discoveryOverloadOutcome(claim.state.consecutiveOverloads, 'aborted', error);
2279
+ leaseReleased = await this.#state.deferDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, outcome.backoffUntilMs, outcome.consecutiveOverloads);
2048
2280
  this.#logger.warn?.('[factory] Relayfile discovery overloaded; backing off before another sweep', {
2049
2281
  status: overload.status,
2050
2282
  reason: overload.reason,
2051
- retryAfterSeconds: overload.retryAfterSeconds,
2052
- delayMs,
2053
- backoffUntilMs,
2054
- consecutiveOverloads,
2283
+ retryAfterSeconds: outcome.retryAfterSeconds,
2284
+ delayMs: outcome.delayMs,
2285
+ backoffUntilMs: outcome.backoffUntilMs,
2286
+ consecutiveOverloads: outcome.consecutiveOverloads,
2287
+ previousOverloads: claim.state.consecutiveOverloads,
2288
+ sweepOverloads: this.#discoverySweepOverloads,
2289
+ sweepProgress: this.#discoverySweepProgress,
2055
2290
  });
2056
2291
  // backoffUntilMs is already durable via deferDiscoverySweep, and the
2057
2292
  // next runOnce() honors it at the pre-claim wait above — sleeping
@@ -2067,6 +2302,9 @@ export class FactoryLoop {
2067
2302
  this.#discoverySweepEpoch = undefined;
2068
2303
  this.#discoverySweepStartedAtMs = undefined;
2069
2304
  this.#discoveryOverloadError = undefined;
2305
+ this.#discoverySweepOverloads = 0;
2306
+ this.#discoverySweepRetryAfterSeconds = undefined;
2307
+ this.#discoverySweepProgress = false;
2070
2308
  // This sweep is over either way (committed, deferred, or lease lost) —
2071
2309
  // a stale `true` here would otherwise make every #listRelayfileTree
2072
2310
  // call outside a fresh claim (Slack lookups, PR confirmation, the
@@ -2078,6 +2316,94 @@ export class FactoryLoop {
2078
2316
  }
2079
2317
  }
2080
2318
  }
2319
+ /**
2320
+ * Commit the sweep, carrying any residual overload backoff into the store.
2321
+ *
2322
+ * `completeDiscoverySweepWithOverload` is optional on the port so a store
2323
+ * written before #297 keeps working. When one of those is injected and this
2324
+ * sweep HAS a residual, the backoff cannot be persisted and the next sweep
2325
+ * would retry immediately — so this says so, loudly and with a counter,
2326
+ * rather than degrading in silence. The sweep itself still commits: losing
2327
+ * the backoff is worse than pre-#297 behaviour only if nobody notices.
2328
+ */
2329
+ async #commitDiscoverySweep(epoch, checkpoint, residual) {
2330
+ if (residual && this.#state.completeDiscoverySweepWithOverload) {
2331
+ return await this.#state.completeDiscoverySweepWithOverload(this.#workspaceId, this.#discoverySweepOwner, epoch, checkpoint, { consecutiveOverloads: residual.consecutiveOverloads, backoffUntilMs: residual.backoffUntilMs });
2332
+ }
2333
+ if (residual) {
2334
+ this.#increment('discoveryOverloadResidualUnsupported');
2335
+ this.#logger.warn?.('[factory] state store cannot persist the Relayfile overload backoff; it will be lost', {
2336
+ store: this.#state.constructor?.name,
2337
+ consecutiveOverloads: residual.consecutiveOverloads,
2338
+ backoffUntilMs: residual.backoffUntilMs,
2339
+ delayMs: residual.delayMs,
2340
+ });
2341
+ }
2342
+ return await this.#state.completeDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, epoch, checkpoint);
2343
+ }
2344
+ /**
2345
+ * How the durable overload ratchet should read after this sweep, or
2346
+ * `undefined` if relayfile never shed anything and the ordinary reset
2347
+ * applies.
2348
+ *
2349
+ * #297, deliverable 3. `consecutiveOverloads` used to clear only on a
2350
+ * *fully* clean sweep, so under sustained mild load the ratchet essentially
2351
+ * never cleared: it climbed to the cap on the first bad sweep and stayed
2352
+ * there, probing for recovery once per cap-length window, long after the
2353
+ * dependency had recovered. Requiring perfection to clear a ratchet means
2354
+ * the ratchet does not clear.
2355
+ *
2356
+ * The signal that replaces "was this sweep perfect" is "did relayfile serve
2357
+ * any of this sweep's work units", because that is what the ratchet is
2358
+ * actually for. A shedding DO rejects ALL background traffic —
2359
+ * `oldest_inflight_age` does this regardless of how few requests are in
2360
+ * flight — so a sweep against a genuinely overloaded workspace gets not one
2361
+ * unit through and still escalates, all the way to the cap. A sweep that did
2362
+ * get units through proves the dependency is serving us, so it decays by one
2363
+ * rung. Decay, not reset: getting work done while being shed is not evidence
2364
+ * that the overload is over, only that it is survivable.
2365
+ */
2366
+ #discoveryOverloadOutcome(previousOverloads, sweepOutcome, error) {
2367
+ const overload = relayfileOverload(error) ?? relayfileOverload(this.#discoveryOverloadError);
2368
+ if (!overload)
2369
+ return undefined;
2370
+ // The longest delay anything in this sweep asked for, not just the one the
2371
+ // latched or terminating error carried — see `#discoverySweepRetryAfterSeconds`.
2372
+ const advertised = [this.#discoverySweepRetryAfterSeconds, overload.retryAfterSeconds]
2373
+ .filter((value) => value !== undefined);
2374
+ const retryAfterSeconds = advertised.length > 0 ? Math.max(...advertised) : undefined;
2375
+ // Deliberately NOT "the sweep reached its end". A sweep in which relayfile
2376
+ // shed EVERY unit still runs the loop to the end, and it got nothing done;
2377
+ // treating that as progress would decay the ratchet in exactly the case it
2378
+ // exists for. (`#runOnceWithDiscoveryFence` turns that sweep into an
2379
+ // aborted one before it can commit, so the escalate branch here is reached
2380
+ // only through `sweepOutcome === 'aborted'` — but the rule is a property of
2381
+ // progress, not of which caller asked, and is written that way.)
2382
+ const consecutiveOverloads = this.#discoverySweepProgress
2383
+ ? Math.max(0, previousOverloads - 1)
2384
+ : previousOverloads + 1;
2385
+ const delayMs = discoveryOverloadBackoffMs(retryAfterSeconds, consecutiveOverloads);
2386
+ const backoffUntilMs = this.#clock.now() + delayMs;
2387
+ this.#increment('discoveryOverloadBackoffs');
2388
+ if (sweepOutcome === 'committed') {
2389
+ this.#logger.warn?.('[factory] discovery sweep committed despite Relayfile overload', {
2390
+ status: overload.status,
2391
+ reason: overload.reason,
2392
+ retryAfterSeconds,
2393
+ sweepOverloads: this.#discoverySweepOverloads,
2394
+ previousOverloads,
2395
+ consecutiveOverloads,
2396
+ delayMs,
2397
+ backoffUntilMs,
2398
+ });
2399
+ }
2400
+ return {
2401
+ consecutiveOverloads,
2402
+ backoffUntilMs,
2403
+ delayMs,
2404
+ ...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }),
2405
+ };
2406
+ }
2081
2407
  async #performRunOnce(opts = {}) {
2082
2408
  const dryRun = opts.dryRun ?? this.#config.dryRun;
2083
2409
  const startedAtMs = this.#clock.now();
@@ -2125,15 +2451,57 @@ export class FactoryLoop {
2125
2451
  let readyIssueReads = 0;
2126
2452
  const issueEntries = [];
2127
2453
  for (const path of paths) {
2128
- const issue = await this.#readIssue(path);
2454
+ let issue;
2455
+ let shed = false;
2456
+ try {
2457
+ issue = await this.#readIssue(path);
2458
+ }
2459
+ catch (error) {
2460
+ // #297: `#readIssue` rethrows relayfile overload and swallows every
2461
+ // other read fault, so this catch only ever sees the backend
2462
+ // shedding THIS issue's read. That is a fact about one work unit:
2463
+ // a sweep that pulled 40 issues and was shed on the 39th must keep
2464
+ // the other 39, exactly as #292 argued for dispatch failures. The
2465
+ // fuse is what still ends a sweep the backend is refusing.
2466
+ const overload = relayfileOverload(error);
2467
+ // Defensive: anything `#readIssue` did not swallow and is not a 429
2468
+ // is not ours to reclassify, and must surface as itself.
2469
+ if (!overload)
2470
+ throw error;
2471
+ const fuse = this.#discoveryOverloadFuseError();
2472
+ if (fuse)
2473
+ throw fuse;
2474
+ shed = true;
2475
+ this.#increment('discoveryOverloadItemsSkipped');
2476
+ this.#logger.warn?.('[factory] relayfile shed a ready-issue read; skipping it and continuing the sweep', {
2477
+ path,
2478
+ status: overload.status,
2479
+ reason: overload.reason,
2480
+ retryAfterSeconds: overload.retryAfterSeconds,
2481
+ sweepOverloads: this.#discoverySweepOverloads,
2482
+ });
2483
+ recordSkip({ issue: issueRefFromPath(path), reason: perItemDispatchSkipReason(error) });
2484
+ }
2129
2485
  readyIssueReads += 1;
2486
+ // Relayfile served this work unit's read: the dependency is shedding
2487
+ // but not refusing, which is what decays the ratchet (#297).
2488
+ //
2489
+ // `issue` is required, not just `!shed`: `#readIssue` returns
2490
+ // `undefined` for a body it could not read at all, and the known
2491
+ // phantom condition — the tree listing issue paths whose bodies are
2492
+ // absent — makes every read return `undefined`. Crediting those would
2493
+ // decay the ratchet on a sweep that served nothing.
2494
+ if (!shed && issue)
2495
+ this.#discoverySweepProgress = true;
2130
2496
  lastReadyReadProgressAtMs = this.#logTimedProgress(this.#config.issueSource === 'github'
2131
2497
  ? '[factory] GitHub ready issue read progress'
2132
2498
  : '[factory] Linear ready issue read progress', startedAtMs, lastReadyReadProgressAtMs, { read: readyIssueReads, total: paths.length, path });
2133
- if (issue && issueSource === 'linear') {
2134
- await this.#recordCanonicalIssueState(issue);
2499
+ if (!shed) {
2500
+ if (issue && issueSource === 'linear') {
2501
+ await this.#recordCanonicalIssueState(issue);
2502
+ }
2503
+ issueEntries.push({ path, issue });
2135
2504
  }
2136
- issueEntries.push({ path, issue });
2137
2505
  await this.#refreshLiveHeartbeatIfDue();
2138
2506
  }
2139
2507
  if (issueSource === 'github') {
@@ -2222,6 +2590,9 @@ export class FactoryLoop {
2222
2590
  // A completed dispatch — even one that parks or escalates the issue —
2223
2591
  // proves the pipeline still works, so the fuse below starts over.
2224
2592
  unclassifiedFailuresSinceDispatch = 0;
2593
+ // ...and proves relayfile is still serving this sweep, which is what
2594
+ // decays the durable overload ratchet (#297).
2595
+ this.#discoverySweepProgress = true;
2225
2596
  if (result.agents.length === 0 && !dryRun) {
2226
2597
  const reason = result.hold?.kind === 'dependency-cycle'
2227
2598
  ? `dependency cycle detected: ${result.hold.cycle?.join(' -> ') ?? 'unknown cycle'}`
@@ -2235,13 +2606,47 @@ export class FactoryLoop {
2235
2606
  }
2236
2607
  }
2237
2608
  catch (error) {
2609
+ // The failure may have left half-spawned agents behind, persisted as
2610
+ // failure handoffs on the way out of `#dispatchUnlocked`. runLoop's
2611
+ // catch used to reap them because every such error aborted the pass;
2612
+ // now that most of them are skipped, the reap has to happen here.
2613
+ //
2614
+ // BEFORE the fatality check, not after (#298 review, round two): the
2615
+ // 429 that trips the overload fuse aborts this pass, and a direct
2616
+ // `runOnce()` — the `factory run-once` CLI, `#reconcileReadyIssues` —
2617
+ // has no runLoop catch behind it, so the unit that trips the fuse
2618
+ // would leak the agents it had already spawned. Reaping first covers
2619
+ // the abort and the skip with one call; it is idempotent, so the
2620
+ // runLoop catch finding nothing left to do is free.
2621
+ if (mayHaveSpawnedBeforeFailing(error)) {
2622
+ await this.#reapDispatchFailureHandoffsNow();
2623
+ }
2238
2624
  // #292: issues in a pass are independent work units, so a failure
2239
2625
  // that is about ONE unit costs that unit and nothing else. Only the
2240
2626
  // conditions named in `#isPassFatalFailure` — the ones where
2241
2627
  // continuing the pass is meaningless — abort the whole sweep.
2242
- if (this.#isPassFatalFailure(error, dryRun))
2243
- throw error;
2244
- if (!isClassifiedPerItemDispatchFailure(error)) {
2628
+ if (this.#isPassFatalFailure(error, dryRun)) {
2629
+ // The overload fuse may have been tripped by a 429 that a caller
2630
+ // swallowed, leaving an unrelated error in hand. The fence keys
2631
+ // the durable backoff off `relayfileOverload(error)`, so hand it
2632
+ // the 429 rather than whatever surfaced last.
2633
+ throw this.#discoveryOverloadFuseError() ?? error;
2634
+ }
2635
+ const overload = relayfileOverload(error);
2636
+ if (overload) {
2637
+ // #297: shedding is a state of the dependency, not a fault of this
2638
+ // work unit, so it stays out of `counters.errors` and gets its own
2639
+ // counter — the same split #293 made for undispatchable units.
2640
+ this.#increment('discoveryOverloadItemsSkipped');
2641
+ this.#logger.warn?.('[factory] relayfile shed this work unit; skipping it and continuing the sweep', {
2642
+ issue: issueRef(issue).key,
2643
+ status: overload.status,
2644
+ reason: overload.reason,
2645
+ retryAfterSeconds: overload.retryAfterSeconds,
2646
+ sweepOverloads: this.#discoverySweepOverloads,
2647
+ });
2648
+ }
2649
+ else if (!isClassifiedPerItemDispatchFailure(error)) {
2245
2650
  unclassifiedFailuresSinceDispatch += 1;
2246
2651
  // A pass-wide fault can arrive disguised as a run of per-item
2247
2652
  // faults. Skipping every unit would then hand back a green report
@@ -2260,11 +2665,6 @@ export class FactoryLoop {
2260
2665
  error: describeError(error).errorMessage,
2261
2666
  });
2262
2667
  this.#error(error, issueRef(issue));
2263
- // The failure may have left half-spawned agents behind. runLoop's
2264
- // catch used to reap them because this error aborted the pass;
2265
- // now that the pass survives, the reap has to happen here or the
2266
- // agents leak until the next failed iteration.
2267
- await this.#reapDispatchFailureHandoffsNow();
2268
2668
  }
2269
2669
  else {
2270
2670
  // Not an error — the unit simply cannot be dispatched right now —
@@ -2313,6 +2713,21 @@ export class FactoryLoop {
2313
2713
  }
2314
2714
  }
2315
2715
  }
2716
+ /**
2717
+ * The 429 that ended this sweep, when relayfile has shed
2718
+ * `DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT` operations and skipping the next work
2719
+ * unit would just be grinding a shedding dependency. Undefined below that.
2720
+ *
2721
+ * Returns the *latched* 429 rather than whatever error is in hand, because
2722
+ * `#runOnceWithDiscoveryFence` keys the durable backoff off
2723
+ * `relayfileOverload(error)` and a caller may have swallowed the 429 that
2724
+ * tripped the fuse.
2725
+ */
2726
+ #discoveryOverloadFuseError() {
2727
+ if (this.#discoverySweepOverloads < DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT)
2728
+ return undefined;
2729
+ return this.#discoveryOverloadError;
2730
+ }
2316
2731
  /**
2317
2732
  * Whether a failure raised while processing ONE work unit must abort the
2318
2733
  * whole readiness pass instead of skipping that unit.
@@ -2333,10 +2748,11 @@ export class FactoryLoop {
2333
2748
  * one would be recorded as an ordinary per-issue skip. The run report
2334
2749
  * would then claim a clean pass over work this process no longer has the
2335
2750
  * right to touch.
2336
- * - Relayfile signalled overload for this sweep. The backend is shedding
2337
- * load; grinding through the remaining units makes it worse, and
2338
- * `#runOnceWithDiscoveryFence` is going to rethrow this at the fence
2339
- * anyway.
2751
+ * - Relayfile has shed `DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT` operations in
2752
+ * this sweep. A single shed operation is per-item and skippable (#297),
2753
+ * but past this the backend is not serving this sweep at all: grinding
2754
+ * through the remaining units makes it worse, and the fence needs the 429
2755
+ * to set the durable backoff.
2340
2756
  * - The factory is stopping. Teardown is in progress and dispatching more
2341
2757
  * agents now leaks them past the shutdown deadline.
2342
2758
  * - The fleet control-plane circuit is no longer closed, **on a live pass**.
@@ -2365,7 +2781,15 @@ export class FactoryLoop {
2365
2781
  #isPassFatalFailure(error, dryRun) {
2366
2782
  // Sweep-scoped: these are about this process's right or ability to run the
2367
2783
  // pass at all, so they hold for a dry run exactly as for a live one.
2368
- if (this.#discoverySweepLeaseLost || this.#discoveryOverloadError !== undefined || this.#stopping) {
2784
+ if (this.#discoverySweepLeaseLost || this.#stopping) {
2785
+ return true;
2786
+ }
2787
+ // #297: relayfile overload used to sit alongside those two, and it did not
2788
+ // belong there. A 429 on ONE work unit is a fact about that unit's read or
2789
+ // write, not about this process's right to run the pass — and because the
2790
+ // flag latched for the whole sweep, the first transient shed also made
2791
+ // every later unit fatal. Only sustained shedding is now pass-fatal.
2792
+ if (this.#discoverySweepOverloads >= DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT) {
2369
2793
  return true;
2370
2794
  }
2371
2795
  // Fleet-scoped, and therefore live-only. See the doc comment above.
@@ -3231,14 +3655,40 @@ export class FactoryLoop {
3231
3655
  return result;
3232
3656
  }
3233
3657
  catch (error) {
3234
- if (relayfileOverload(error) && this.#discoverySweepEpoch !== undefined) {
3658
+ const overload = relayfileOverload(error);
3659
+ if (overload && this.#discoverySweepEpoch !== undefined) {
3235
3660
  this.#discoveryOverloadError ??= error;
3661
+ this.#discoverySweepOverloads += 1;
3662
+ if (overload.retryAfterSeconds !== undefined) {
3663
+ this.#discoverySweepRetryAfterSeconds = Math.max(this.#discoverySweepRetryAfterSeconds ?? 0, overload.retryAfterSeconds);
3664
+ }
3665
+ this.#increment('discoveryOverloadOperations');
3666
+ // #297: relayfile's four overload reason codes — inflight_limit,
3667
+ // oldest_inflight_age, router_inflight_limit, durable_object_overloaded
3668
+ // — all share ONE message string, and they mean four different things:
3669
+ // a DO-local admission cap, one stuck op poisoning every background
3670
+ // caller, a Worker-global isolate cap, and the Cloudflare runtime
3671
+ // shedding the object outright. `relayfileOverload()` has always parsed
3672
+ // the reason; nothing on this path ever printed it, and during the
3673
+ // 2026-08-20 outage that ambiguity was the single biggest obstacle to
3674
+ // diagnosis. Unconditional, unlike the failure warn below: a 429 is
3675
+ // always worth one line.
3676
+ this.#increment(`discoveryOverloadReason:${relayfileOverloadReasonLabel(overload.reason)}`);
3677
+ this.#logger.warn?.('[factory] relayfile shed a discovery operation', {
3678
+ ...metadata,
3679
+ status: overload.status,
3680
+ reason: overload.reason,
3681
+ retryAfterSeconds: overload.retryAfterSeconds,
3682
+ sweepOverloads: this.#discoverySweepOverloads,
3683
+ elapsedMs: this.#elapsedSince(startedAtMs),
3684
+ });
3236
3685
  }
3237
3686
  if (opts.logFailure || waitWarnings > 0) {
3238
3687
  this.#increment('relayfileOperationFailures');
3239
3688
  this.#logger.warn?.('[factory] relayfile operation failed', {
3240
3689
  ...metadata,
3241
3690
  elapsedMs: this.#elapsedSince(startedAtMs),
3691
+ ...(overload ? { status: overload.status, reason: overload.reason } : {}),
3242
3692
  error: describeError(error).errorMessage,
3243
3693
  });
3244
3694
  }
@@ -3801,17 +4251,66 @@ export class FactoryLoop {
3801
4251
  }
3802
4252
  #readinessReconcileStatus() {
3803
4253
  const consecutiveFailures = this.#readinessReconcileConsecutiveFailures;
3804
- const state = this.#startMode !== 'live'
4254
+ // #296 owns the numerator here, #295/#300 own the derivation. The earliest
4255
+ // sweep still running — after a deadline expiry that is the abandoned one,
4256
+ // not the current wait, or every expiry would restart the clock and a
4257
+ // permanently stuck pass would read as merely `retrying`.
4258
+ const inFlightSinceMs = Math.min(this.#readinessReconcileInFlightSinceMs ?? Number.POSITIVE_INFINITY, this.#readinessReconcileAbandonedSinceMs ?? Number.POSITIVE_INFINITY);
4259
+ const settled = this.#startMode !== 'live'
3805
4260
  ? 'not-running'
4261
+ // Failure-count ladder only. Precedence against `stalled` belongs to
4262
+ // `derivedReadinessReconcileState`, which outranks everything here: a
4263
+ // stall is the more specific fact, and `consecutiveFailures` ships
4264
+ // alongside so nothing an alarm keyed on `degraded` needs is lost.
3806
4265
  : consecutiveFailures >= READINESS_RECONCILE_FAILURE_THRESHOLD
3807
4266
  ? 'degraded'
3808
4267
  : consecutiveFailures > 0
3809
4268
  ? 'retrying'
3810
4269
  : 'healthy';
4270
+ // The counters above only move when a pass *settles*. A pass that hangs
4271
+ // takes neither path, so `settled` would keep reporting the last finished
4272
+ // pass — `healthy` — for as long as the process is stuck (#295). The
4273
+ // in-flight age is the only field that can express that, so derive the
4274
+ // state from it rather than trusting the last write.
4275
+ const timestamps = {
4276
+ intervalMs: this.#readinessReconcileIntervalMs,
4277
+ ...(Number.isFinite(inFlightSinceMs) ? { inFlightSinceMs } : {}),
4278
+ ...(this.#readinessReconcileLastStartedAtMs !== undefined
4279
+ ? { lastStartedAtMs: this.#readinessReconcileLastStartedAtMs }
4280
+ : {}),
4281
+ ...(this.#readinessReconcileLastCompletedAtMs !== undefined
4282
+ ? { lastCompletedAtMs: this.#readinessReconcileLastCompletedAtMs }
4283
+ : {}),
4284
+ ...(this.#readinessReconcileLastFailureAtMs !== undefined
4285
+ ? { lastFailureAtMs: this.#readinessReconcileLastFailureAtMs }
4286
+ : {}),
4287
+ };
4288
+ const nowMs = this.#clock.now();
4289
+ // Defence in depth (#300 review, CodeRabbit). These derivations are new
4290
+ // code from another module on a path that `status()` and every heartbeat
4291
+ // write depend on. A throw here would take out the liveness signal the
4292
+ // crash reaper reads — the diagnostic causing the outage it exists to
4293
+ // explain — so a failure costs the derived fields and nothing else.
4294
+ let inFlightMs;
4295
+ let derived = settled;
4296
+ try {
4297
+ inFlightMs = readinessReconcileInFlightMs(timestamps, nowMs);
4298
+ derived = derivedReadinessReconcileState({ ...timestamps, state: settled }, nowMs);
4299
+ }
4300
+ catch (error) {
4301
+ this.#logger.warn?.('[factory] readiness health derivation failed; reporting the settled state', {
4302
+ error: describeError(error).errorMessage,
4303
+ });
4304
+ inFlightMs = undefined;
4305
+ derived = settled;
4306
+ }
3811
4307
  return {
3812
- state,
4308
+ state: derived === 'unknown' ? settled : derived,
3813
4309
  consecutiveFailures,
3814
4310
  failureThreshold: READINESS_RECONCILE_FAILURE_THRESHOLD,
4311
+ intervalMs: this.#readinessReconcileIntervalMs,
4312
+ ...(Number.isFinite(inFlightSinceMs) ? { inFlightSinceMs } : {}),
4313
+ ...(inFlightMs !== undefined ? { inFlightMs } : {}),
3815
4314
  ...(this.#readinessReconcileLastDurationMs !== undefined
3816
4315
  ? { lastDurationMs: this.#readinessReconcileLastDurationMs }
3817
4316
  : {}),
@@ -3825,6 +4324,9 @@ export class FactoryLoop {
3825
4324
  ? { lastFailureAtMs: this.#readinessReconcileLastFailureAtMs }
3826
4325
  : {}),
3827
4326
  ...(this.#readinessReconcileLastError ? { lastError: this.#readinessReconcileLastError } : {}),
4327
+ ...(this.#readinessReconcileLastErrorClass
4328
+ ? { lastErrorClass: this.#readinessReconcileLastErrorClass }
4329
+ : {}),
3828
4330
  };
3829
4331
  }
3830
4332
  on(event, listener) {
@@ -6301,6 +6803,29 @@ export class FactoryLoop {
6301
6803
  readinessReconcile: this.#readinessReconcileStatus(),
6302
6804
  fleetControlPlane: this.#fleetControlPlane.status(),
6303
6805
  };
6806
+ // The deployed container serves `/healthz` straight out of this file and
6807
+ // has no redaction logic of its own, so publish the already-safe view here
6808
+ // rather than leaving that boundary to whoever reads the file (#295).
6809
+ // Derived against this daemon's clock: every duration in it is a
6810
+ // difference between timestamps this process wrote.
6811
+ //
6812
+ // Guarded (#300 review, CodeRabbit): this heartbeat is what the crash
6813
+ // reaper and `/healthz` read to decide the daemon is alive, and several
6814
+ // callers of this method sit outside any try/catch. A projection failure
6815
+ // must cost the diagnostics block, never the heartbeat — the omitted block
6816
+ // is itself legible, since `factory diagnose` reports a missing one rather
6817
+ // than a false green.
6818
+ try {
6819
+ heartbeat.health = publicHealthFromHeartbeat(heartbeat, {
6820
+ nowMs: updatedAtMs,
6821
+ staleMs: this.#config.loop.heartbeatStaleMs,
6822
+ });
6823
+ }
6824
+ catch (error) {
6825
+ this.#logger.warn?.('[factory] public health projection failed; heartbeat written without it', {
6826
+ error: describeError(error).errorMessage,
6827
+ });
6828
+ }
6304
6829
  await mkdir(dirname(path), { recursive: true });
6305
6830
  await writeFile(path, `${JSON.stringify(heartbeat, null, 2)}\n`, 'utf8');
6306
6831
  await this.#writeInFlightRegistry(registryPath, path);
@@ -6319,11 +6844,18 @@ export class FactoryLoop {
6319
6844
  });
6320
6845
  }
6321
6846
  async #reapDispatchFailureHandoffsNow(heartbeatPath = this.#loopReapPaths?.heartbeatPath ?? this.#config.loop.heartbeatPath, registryPath = this.#loopReapPaths?.registryPath ?? this.#config.loop.registryPath) {
6322
- const handoffs = await this.#state.listFailureHandoffs(this.#workspaceId);
6323
- if (handoffs.length === 0) {
6324
- return;
6325
- }
6326
6847
  try {
6848
+ // Inside the try, not before it (#298 review). Every caller reaps while
6849
+ // already handling a failure and then propagates that failure: the
6850
+ // per-item catch rethrows the dispatch error, runLoop's catch is mid
6851
+ // teardown. A throw from here would REPLACE the error in flight — and a
6852
+ // replaced 429 is no longer recognised as overload at the discovery
6853
+ // fence, silently dropping the advertised backoff. Reaping is
6854
+ // best-effort by construction; the caller's error always wins.
6855
+ const handoffs = await this.#state.listFailureHandoffs(this.#workspaceId);
6856
+ if (handoffs.length === 0) {
6857
+ return;
6858
+ }
6327
6859
  const protectedPids = await this.#protectedPids();
6328
6860
  let registryChanged = false;
6329
6861
  const readyToClear = new Set();
@@ -14941,6 +15473,15 @@ const githubIssueAuthor = (issue) => {
14941
15473
  }
14942
15474
  return source ? undefined : githubAuthorLogin(payload)?.trim() || undefined;
14943
15475
  };
15476
+ /**
15477
+ * An `IssueRef` for a path whose issue body could not be read at all — the
15478
+ * shape a relayfile-shed ready-issue read leaves behind (#297). The key is
15479
+ * what an operator needs to correlate the skip; the uuid falls back to it.
15480
+ */
15481
+ const issueRefFromPath = (path) => {
15482
+ const key = keyFromPath(path);
15483
+ return { uuid: uuidFromPath(path) ?? key, key, path };
15484
+ };
14944
15485
  const issueRef = (issue) => ({ uuid: issue.uuid, key: issue.key, path: issue.path });
14945
15486
  // Preserve the historical Linear state namespace while keeping GitHub-native
14946
15487
  // issue numbers independent across repositories in the same workspace.
@@ -16854,10 +17395,54 @@ const relayfileOverload = (error) => {
16854
17395
  stringValue(flat.code) ?? stringValue(data.code) ?? 'rate_limited';
16855
17396
  return { status, reason, ...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }) };
16856
17397
  };
17398
+ /**
17399
+ * relayfile's overload reason codes, allowlisted.
17400
+ *
17401
+ * `IterationReport.skipped[].reason` is serialized to stdout by `factory
17402
+ * run-once`, so it stays a fixed classification plus a known code — the same
17403
+ * public-surface rule #293 applied to error class names — rather than
17404
+ * whatever string the dependency happened to send.
17405
+ */
17406
+ const RELAYFILE_OVERLOAD_REASONS = new Set([
17407
+ // Admission gate inside the workspace durable object.
17408
+ 'inflight_limit',
17409
+ 'oldest_inflight_age',
17410
+ 'write_admission_limit',
17411
+ // Worker-side backpressure, per isolate rather than per workspace.
17412
+ 'router_inflight_limit',
17413
+ // The Cloudflare runtime shed the object; relayfile only relabels it.
17414
+ 'durable_object_overloaded',
17415
+ // relayfileOverload()'s fallback when the body carried no reason at all.
17416
+ 'rate_limited',
17417
+ ]);
17418
+ /**
17419
+ * Both the run-report reason and the per-reason counter key are built from
17420
+ * this, so an unknown code from the dependency can neither leak into stdout
17421
+ * nor open an unbounded counter namespace.
17422
+ */
17423
+ const relayfileOverloadReasonLabel = (reason) => RELAYFILE_OVERLOAD_REASONS.has(reason) ? reason : 'unrecognized';
17424
+ /**
17425
+ * How long to wait before the next discovery sweep after relayfile shed this
17426
+ * one.
17427
+ *
17428
+ * The advertised `Retry-After` is authoritative in BOTH directions (#297):
17429
+ * it is the first rung, so we never retry sooner than the dependency allows,
17430
+ * and it bounds the ceiling, so we never sleep for minutes because of a
17431
+ * request to wait seconds. Without an advertised delay there is nothing to
17432
+ * respect and the original five-minute ladder governs unchanged.
17433
+ */
16857
17434
  const discoveryOverloadBackoffMs = (retryAfterSeconds, consecutiveOverloads) => {
16858
- const retryAfterMs = Math.max(0, Math.ceil((retryAfterSeconds ?? 0) * 1_000));
16859
- const exponentialMs = Math.min(DISCOVERY_OVERLOAD_BACKOFF_MAX_MS, 5_000 * (2 ** Math.min(10, Math.max(0, consecutiveOverloads - 1))));
16860
- return Math.max(retryAfterMs, exponentialMs);
17435
+ const advertisedMs = retryAfterSeconds === undefined
17436
+ ? undefined
17437
+ : Math.max(DISCOVERY_OVERLOAD_BACKOFF_MIN_MS, Math.ceil(retryAfterSeconds * 1_000));
17438
+ const baseMs = advertisedMs ?? DISCOVERY_OVERLOAD_BACKOFF_BASE_MS;
17439
+ // A dependency that asks for longer than the advertised ceiling still gets
17440
+ // what it asked for; the ceiling only stops the ladder from overshooting it.
17441
+ const ceilingMs = advertisedMs === undefined
17442
+ ? DISCOVERY_OVERLOAD_BACKOFF_MAX_MS
17443
+ : Math.max(advertisedMs, DISCOVERY_OVERLOAD_ADVERTISED_BACKOFF_MAX_MS);
17444
+ const steps = Math.min(10, Math.max(0, consecutiveOverloads - 1));
17445
+ return Math.min(ceilingMs, baseMs * (2 ** steps));
16861
17446
  };
16862
17447
  const eventSequenceNumber = (eventId) => {
16863
17448
  const whole = Number(eventId);
@@ -17086,10 +17671,6 @@ const telemetryCategory = (value) => {
17086
17671
  const normalized = value.trim().toLowerCase().replace(/[^a-z0-9._:/-]+/gu, '-');
17087
17672
  return normalized.slice(0, 120) || undefined;
17088
17673
  };
17089
- const telemetryErrorClass = (error) => {
17090
- const name = error instanceof Error ? error.name : '';
17091
- return /^[A-Za-z][A-Za-z0-9]{0,63}(?:Error|Exception)$/u.test(name) ? name : 'Error';
17092
- };
17093
17674
  const isTimeoutError = (error) => error instanceof Error && (error.name === 'TimeoutError' || error.name === 'AbortError');
17094
17675
  const retryOnTimeout = async (fn, opts) => {
17095
17676
  let lastError;
@@ -17323,7 +17904,26 @@ const UNCLASSIFIED_DISPATCH_FAILURE_LIMIT = 5;
17323
17904
  * skippable and are exempt from the consecutive-failure fuse.
17324
17905
  */
17325
17906
  const isClassifiedPerItemDispatchFailure = (error) => error instanceof LiveDispatchStateChangedError ||
17326
- error instanceof DispatchLifecycleClaimRefusedError;
17907
+ error instanceof DispatchLifecycleClaimRefusedError ||
17908
+ // Relayfile shedding one operation is a state of the dependency, not an
17909
+ // unexplained fault, and it has its own fuse — see #297 and
17910
+ // DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT.
17911
+ relayfileOverload(error) !== undefined;
17912
+ /**
17913
+ * Whether a per-item failure could have left half-spawned agents behind.
17914
+ *
17915
+ * The two lifecycle refusals are decided before `#dispatchUnlocked` spawns
17916
+ * anything, so there is nothing to reap for them. Everything else can fail
17917
+ * *after* a spawn — including relayfile shedding a post-spawn read — having
17918
+ * persisted the agents as failure handoffs on the way out.
17919
+ *
17920
+ * Deliberately a denylist rather than an allowlist: a new failure mode that
17921
+ * nobody classified should default to "reap it", because the cost of a
17922
+ * needless reap is one no-op pass over an empty handoff list, while the cost
17923
+ * of a missed one is leaked agents and duplicate workers on the next retry.
17924
+ */
17925
+ const mayHaveSpawnedBeforeFailing = (error) => !(error instanceof LiveDispatchStateChangedError) &&
17926
+ !(error instanceof DispatchLifecycleClaimRefusedError);
17327
17927
  /**
17328
17928
  * The run-report reason recorded for a work unit the pass could not dispatch.
17329
17929
  *
@@ -17334,6 +17934,9 @@ const isClassifiedPerItemDispatchFailure = (error) => error instanceof LiveDispa
17334
17934
  * `describeControlPlaneError` makes for circuit state.
17335
17935
  */
17336
17936
  const perItemDispatchSkipReason = (error) => {
17937
+ const overload = relayfileOverload(error);
17938
+ if (overload)
17939
+ return `relayfile overloaded (${relayfileOverloadReasonLabel(overload.reason)})`;
17337
17940
  if (error instanceof LiveDispatchStateChangedError)
17338
17941
  return 'live state changed during dispatch';
17339
17942
  if (error instanceof DispatchLifecycleClaimRefusedError) {