@agent-relay/factory 0.1.63 → 0.1.65

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/README.md +9 -0
  2. package/dist/cli/diagnose.d.ts +92 -0
  3. package/dist/cli/diagnose.d.ts.map +1 -0
  4. package/dist/cli/diagnose.js +346 -0
  5. package/dist/cli/diagnose.js.map +1 -0
  6. package/dist/cli/fleet.d.ts +13 -1
  7. package/dist/cli/fleet.d.ts.map +1 -1
  8. package/dist/cli/fleet.js +118 -11
  9. package/dist/cli/fleet.js.map +1 -1
  10. package/dist/config/schema.d.ts +69 -2
  11. package/dist/config/schema.d.ts.map +1 -1
  12. package/dist/config/schema.js +32 -0
  13. package/dist/config/schema.js.map +1 -1
  14. package/dist/fleet/control-plane-circuit.d.ts.map +1 -1
  15. package/dist/fleet/control-plane-circuit.js +13 -1
  16. package/dist/fleet/control-plane-circuit.js.map +1 -1
  17. package/dist/fleet/internal-fleet-client.d.ts +16 -0
  18. package/dist/fleet/internal-fleet-client.d.ts.map +1 -1
  19. package/dist/fleet/internal-fleet-client.js +228 -13
  20. package/dist/fleet/internal-fleet-client.js.map +1 -1
  21. package/dist/hosted/orchestrator.d.ts.map +1 -1
  22. package/dist/hosted/orchestrator.js +1 -4
  23. package/dist/hosted/orchestrator.js.map +1 -1
  24. package/dist/index.d.ts +3 -1
  25. package/dist/index.d.ts.map +1 -1
  26. package/dist/index.js +2 -1
  27. package/dist/index.js.map +1 -1
  28. package/dist/mount/relayfile-cloud-mount-client.d.ts +23 -1
  29. package/dist/mount/relayfile-cloud-mount-client.d.ts.map +1 -1
  30. package/dist/mount/relayfile-cloud-mount-client.js +38 -9
  31. package/dist/mount/relayfile-cloud-mount-client.js.map +1 -1
  32. package/dist/observability/cloud-reporter.d.ts +1 -0
  33. package/dist/observability/cloud-reporter.d.ts.map +1 -1
  34. package/dist/observability/cloud-reporter.js +182 -50
  35. package/dist/observability/cloud-reporter.js.map +1 -1
  36. package/dist/observability/error-class.d.ts +29 -0
  37. package/dist/observability/error-class.d.ts.map +1 -0
  38. package/dist/observability/error-class.js +36 -0
  39. package/dist/observability/error-class.js.map +1 -0
  40. package/dist/observability/index.d.ts +1 -0
  41. package/dist/observability/index.d.ts.map +1 -1
  42. package/dist/observability/index.js +1 -0
  43. package/dist/observability/index.js.map +1 -1
  44. package/dist/orchestrator/factory.d.ts.map +1 -1
  45. package/dist/orchestrator/factory.js +1461 -84
  46. package/dist/orchestrator/factory.js.map +1 -1
  47. package/dist/orchestrator/index.d.ts +1 -0
  48. package/dist/orchestrator/index.d.ts.map +1 -1
  49. package/dist/orchestrator/index.js +1 -0
  50. package/dist/orchestrator/index.js.map +1 -1
  51. package/dist/orchestrator/public-health.d.ts +82 -0
  52. package/dist/orchestrator/public-health.d.ts.map +1 -0
  53. package/dist/orchestrator/public-health.js +372 -0
  54. package/dist/orchestrator/public-health.js.map +1 -0
  55. package/dist/ports/state.d.ts +76 -3
  56. package/dist/ports/state.d.ts.map +1 -1
  57. package/dist/state/document-store.d.ts +2 -1
  58. package/dist/state/document-store.d.ts.map +1 -1
  59. package/dist/state/document-store.js.map +1 -1
  60. package/dist/state/file-state-store.d.ts +24 -5
  61. package/dist/state/file-state-store.d.ts.map +1 -1
  62. package/dist/state/file-state-store.js +136 -10
  63. package/dist/state/file-state-store.js.map +1 -1
  64. package/dist/state/in-memory-state-store.d.ts +21 -2
  65. package/dist/state/in-memory-state-store.d.ts.map +1 -1
  66. package/dist/state/in-memory-state-store.js +99 -6
  67. package/dist/state/in-memory-state-store.js.map +1 -1
  68. package/dist/state/watch-state-document.d.ts.map +1 -1
  69. package/dist/state/watch-state-document.js +45 -3
  70. package/dist/state/watch-state-document.js.map +1 -1
  71. package/dist/trajectory.d.ts +25 -0
  72. package/dist/trajectory.d.ts.map +1 -0
  73. package/dist/trajectory.js +51 -0
  74. package/dist/trajectory.js.map +1 -0
  75. package/dist/types.d.ts +106 -1
  76. package/dist/types.d.ts.map +1 -1
  77. package/package.json +1 -1
@@ -1,7 +1,7 @@
1
1
  import { randomUUID } from 'node:crypto';
2
2
  import { mkdir, readFile, writeFile } from 'node:fs/promises';
3
3
  import { dirname, isAbsolute, resolve } from 'node:path';
4
- import { FactoryConfigSchema } from '../config/schema.js';
4
+ import { DEFAULT_READINESS_RECONCILE_TIMEOUT_MS, FactoryConfigSchema } from '../config/schema.js';
5
5
  import { linearByStatePath, linearByIdPath, linearByUuidPath } from '../constants/linear.js';
6
6
  import { stateResolutionFromIds } from '../linear/state-resolver.js';
7
7
  import { GithubMergeGate, closeProbePr } from '../github/index.js';
@@ -29,8 +29,11 @@ import { CoalescedTaskQueue } from './coalesced-task-queue.js';
29
29
  import { findAgentProcessByName, readProcessIdentity } from './process-identity.js';
30
30
  import { readFactoryInFlightRegistry, terminatePids } from './reaper.js';
31
31
  import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../observability/events.js';
32
+ import { telemetryErrorClass } from '../observability/error-class.js';
33
+ import { derivedReadinessReconcileState, publicHealthFromHeartbeat, readinessReconcileInFlightMs, } from './public-health.js';
32
34
  import { boundedRunCostTotal, CostLedger } from '../cost/ledger.js';
33
35
  import { createTicketDispatchDelivery } from '../delivery/ticket-dispatch.js';
36
+ import { canonicalTrajectorySessionRef, renderTrajectoryPointer, stripTrajectoryPointers, } from '../trajectory.js';
34
37
  import { FleetControlPlaneCircuit, FleetControlPlaneCircuitOpenError, guardFleetControlPlane, } from '../fleet/control-plane-circuit.js';
35
38
  class ClarificationWakeLeaseLostError extends Error {
36
39
  }
@@ -146,8 +149,32 @@ const STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS = 30_000;
146
149
  const RECONCILED_AGENT_EXIT_CONCURRENCY = 4;
147
150
  const SLACK_EVENT_WATERMARK_CACHE_MS = 60_000;
148
151
  const SLACK_CONVERSATION_TURN_LEASE_MS = 60_000;
152
+ // Both receipt leases guard an in-flight Slack writeback, and no fixed lease can
153
+ // cover one: MountSlackWriteback budgets 90s for the confirm alone, on top of an
154
+ // unbounded writeFile. Sizing them past that worst case would only trade a stolen
155
+ // claim for a stranded one — a lease long enough to survive the slowest write is
156
+ // equally long enough to hold the receipt hostage to a dead holder. So these
157
+ // bound the *idle* claim and #withRenewedProviderLease extends them for exactly
158
+ // as long as the write they cover is still running.
159
+ const SLACK_REPLY_ACKNOWLEDGEMENT_LEASE_MS = 60_000;
160
+ const SLACK_TERMINAL_RECEIPT_CLAIM_LEASE_MS = 60_000;
161
+ // Renewal without a ceiling is the same defect from the other side: a heartbeat
162
+ // that extends the claim for as long as the write runs also extends it forever
163
+ // when the write never returns, and nothing else can reclaim the receipt short
164
+ // of a restart. So renewal is bounded past the slowest write this daemon budgets
165
+ // for — MountSlackWriteback's 90s confirm on top of its writeFile — and beyond
166
+ // that the write is not slow, it is wedged: the heartbeat stops, the idle lease
167
+ // runs out, and the retry that owns the queued replies can take them back.
168
+ const SLACK_PROVIDER_LEASE_MAX_RENEWAL_MS = 5 * 60_000;
149
169
  const SLACK_CONVERSATION_TURN_RETRY_MS = 1_000;
150
170
  const SLACK_REPLY_ROUTE_RETRY_MS = 1_000;
171
+ // One pass drains the whole chain (#slackReplyRoutes holds only the newest
172
+ // route per key and every route awaits its predecessor). The extra passes only
173
+ // exist so the drain can prove quiescence rather than assume it.
174
+ const SLACK_REPLY_ROUTE_DRAIN_PASSES = 8;
175
+ const SLACK_TERMINAL_THREAD_GRACE_MS = 24 * 60 * 60_000;
176
+ const SLACK_TERMINAL_RECEIPT_RETRY_MS = 1_000;
177
+ const SLACK_TERMINAL_RECEIPT_RETRY_MAX_MS = 5 * 60_000;
151
178
  const MERGE_GATE_MAX_ATTEMPTS = 12;
152
179
  const MERGE_GATE_POLL_DELAY_MS = 10_000;
153
180
  const MAX_LABEL_IMPLEMENTERS = 4;
@@ -160,6 +187,38 @@ const DISCOVERY_SWEEP_RENEW_MS = 30_000;
160
187
  const READINESS_RECONCILE_FAILURE_THRESHOLD = 3;
161
188
  const DISCOVERY_CHANGE_EVENT_LIMIT = 1_000;
162
189
  const DISCOVERY_OVERLOAD_BACKOFF_MAX_MS = 5 * 60_000;
190
+ /** First rung of the ladder when the 429 advertises no `Retry-After`. */
191
+ const DISCOVERY_OVERLOAD_BACKOFF_BASE_MS = 5_000;
192
+ /**
193
+ * Floor for the advertised delay. `Retry-After: 0` would otherwise pin the
194
+ * whole ladder at zero (`0 * 2 ** n` is still zero) and turn respecting the
195
+ * dependency into hammering it.
196
+ */
197
+ const DISCOVERY_OVERLOAD_BACKOFF_MIN_MS = 1_000;
198
+ /**
199
+ * Ceiling for the ladder once the dependency has told us how long to wait.
200
+ *
201
+ * relayfile sheds an overloaded workspace DO with a 429 in milliseconds
202
+ * carrying `Retry-After: 5`, and #297 is what happened when the ladder ignored
203
+ * that and climbed to `DISCOVERY_OVERLOAD_BACKOFF_MAX_MS` anyway: the
204
+ * dependency asked for five seconds, Factory slept for five minutes, probed
205
+ * for recovery once per cap-length window, and presented a transient upstream
206
+ * blip as a sustained outage. The ratchet still escalates — it is just bounded
207
+ * by roughly what was actually asked for, and never *below* it, so this is a
208
+ * ceiling and not a licence to retry sooner than the dependency allows.
209
+ */
210
+ const DISCOVERY_OVERLOAD_ADVERTISED_BACKOFF_MAX_MS = 30_000;
211
+ /**
212
+ * How many relayfile operations one sweep may have shed before the sweep is
213
+ * abandoned rather than continued.
214
+ *
215
+ * Per-item overload skips that item and keeps going (#297, the same principle
216
+ * as #292), but skipping must not degenerate into grinding a shedding
217
+ * dependency through an entire backlog one 429 at a time. Past this many, the
218
+ * dependency is not serving this sweep at all: abort, back off, and let the
219
+ * ratchet do its job.
220
+ */
221
+ const DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT = 5;
163
222
  const GITHUB_FACTORY_LABEL = 'factory';
164
223
  const GITHUB_LIFECYCLE_LABELS = new Set(['factory:in-progress', 'factory:human-review']);
165
224
  const GITHUB_MIRROR_TITLE_PREFIX = '[factory]';
@@ -215,6 +274,39 @@ class DispatchLifecycleOwnedElsewhereError extends Error {
215
274
  this.leaseUntilMs = leaseUntilMs;
216
275
  }
217
276
  }
277
+ /**
278
+ * The durable dispatch-lifecycle claim was refused for one work unit: its
279
+ * record is already terminal, or another publisher currently holds the lease.
280
+ * Both are facts about that single unit — the rest of the pass is unaffected —
281
+ * so the readiness loop skips it and keeps going (#292).
282
+ *
283
+ * Typed rather than left as a plain `Error` so the loop can classify it by
284
+ * construction instead of by matching on `Refusing to dispatch ...` text.
285
+ */
286
+ class DispatchLifecycleClaimRefusedError extends Error {
287
+ issueKey;
288
+ refusal;
289
+ constructor(issueKey, refusal, message) {
290
+ super(message);
291
+ this.issueKey = issueKey;
292
+ this.refusal = refusal;
293
+ this.name = 'DispatchLifecycleClaimRefusedError';
294
+ }
295
+ }
296
+ /**
297
+ * The deadline that makes a hung sweep reachable by the existing recovery path.
298
+ * Its message is fully internal (one integer), so it is safe to persist into
299
+ * the operator-facing `readinessReconcile.lastError`.
300
+ */
301
+ class ReadinessReconcileTimeoutError extends Error {
302
+ timeoutMs;
303
+ code = 'FACTORY_READINESS_RECONCILE_TIMEOUT';
304
+ constructor(timeoutMs) {
305
+ super(`readiness reconcile sweep exceeded its ${timeoutMs}ms deadline`);
306
+ this.timeoutMs = timeoutMs;
307
+ this.name = 'ReadinessReconcileTimeoutError';
308
+ }
309
+ }
218
310
  const realClock = {
219
311
  now: () => Date.now(),
220
312
  sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
@@ -266,6 +358,19 @@ export class FactoryLoop {
266
358
  #dispatchInFlight = new Map();
267
359
  #slackWatchers = new Map();
268
360
  #slackWatcherStarts = new Map();
361
+ #slackTerminalWatchExpiryTimers = new Map();
362
+ #slackTerminalReceiptRetryTimers = new Map();
363
+ #terminalSlackWatchIssues = new Set();
364
+ /**
365
+ * The one in-memory record of "this work unit has an in-flight Slack side
366
+ * effect". Both ordinary reply routes and the writebacks the terminal fence
367
+ * issues on their behalf register here, because this map is what the terminal
368
+ * drain waits on: anything that touches Slack for a work unit without
369
+ * registering is invisible to the drain, and the watcher teardown that follows
370
+ * a successful drain then pulls that effect's retry timer out from under it.
371
+ */
372
+ #slackReplyRoutes = new Map();
373
+ #slackReplyRouteDrains = new Set();
269
374
  #slackConversationTurns;
270
375
  #slackConversationOwner = `${process.pid}:${randomUUID()}`;
271
376
  #githubIssueCommentWatchers = new Map();
@@ -351,12 +456,41 @@ export class FactoryLoop {
351
456
  #readinessReconcileTimer;
352
457
  #readinessReconcileInFlight;
353
458
  #readinessReconcileIntervalMs = 60_000;
459
+ #readinessReconcileTimeoutMs = DEFAULT_READINESS_RECONCILE_TIMEOUT_MS;
460
+ // Set for exactly as long as a sweep is running. `state` is derived from
461
+ // this, so an in-flight pass can no longer masquerade as the last settled one.
462
+ #readinessReconcileInFlightSinceMs;
463
+ /**
464
+ * The work a deadline gave up waiting on. The deadline bounds the wait, not
465
+ * the sweep, so this is still live: shutdown has to drain it, and `state` has
466
+ * to keep counting from when it actually started.
467
+ *
468
+ * Two fields rather than a collection, because every live abandoned wait
469
+ * converges on the same sweep. They are all `runOnce()` calls with the same
470
+ * `dryRun`, so whatever they are queued behind, the first one out starts the
471
+ * sweep and the rest coalesce onto it — they settle together. So the newest
472
+ * wait is a sufficient drain target, and the earliest start is the honest
473
+ * age. Both matter (#301 review): keeping only the newest start advanced the
474
+ * age by two intervals every two intervals, so at `reconcileTimeoutMs ===
475
+ * reconcileIntervalMs` it never reached three and `stalled` was never
476
+ * reported; keeping one record per wait grew without bound in exactly the
477
+ * never-settling case this change exists for.
478
+ *
479
+ * The wait, deliberately, and never `#runOnceInFlight`: a mismatched-`dryRun`
480
+ * sweep is waited BEHIND rather than coalesced onto, so that handle can name
481
+ * an unrelated sweep, and the readiness pass would then start its own work
482
+ * after shutdown believed it had drained everything. The wait covers the
483
+ * queueing and the sweep it eventually runs, in every branch.
484
+ */
485
+ #readinessReconcileAbandonedWait;
486
+ #readinessReconcileAbandonedSinceMs;
354
487
  #readinessReconcileConsecutiveFailures = 0;
355
488
  #readinessReconcileLastDurationMs;
356
489
  #readinessReconcileLastStartedAtMs;
357
490
  #readinessReconcileLastCompletedAtMs;
358
491
  #readinessReconcileLastFailureAtMs;
359
492
  #readinessReconcileLastError;
493
+ #readinessReconcileLastErrorClass;
360
494
  #liveEventQueue = [];
361
495
  #liveEventDrainScheduled = false;
362
496
  #liveEventDrainActive = false;
@@ -439,7 +573,39 @@ export class FactoryLoop {
439
573
  #discoverySweepRenewTimer;
440
574
  #discoverySweepRenewalInFlight;
441
575
  #discoverySweepLeaseLost = false;
576
+ // Registry/heartbeat paths the in-flight runLoop iteration would use. A
577
+ // per-item dispatch failure now skips instead of aborting the pass (#292),
578
+ // so the loop's catch no longer runs the failure-handoff reaper for it; the
579
+ // pass reaps inline and must write to the same paths runLoop would.
580
+ #loopReapPaths;
581
+ /**
582
+ * The first 429 relayfile raised during this sweep, kept for its
583
+ * `Retry-After` and reason when the sweep decides how long to back off.
584
+ *
585
+ * Before #297 this doubled as a sweep-wide kill switch: any 429 from any
586
+ * relayfile call latched here and `#runOnceWithDiscoveryFence` then threw it
587
+ * away along with everything the sweep had already accomplished. It is now
588
+ * only evidence, never a verdict — see `#discoverySweepOverloads` for the
589
+ * fuse that still ends a sweep the dependency is genuinely refusing to serve.
590
+ */
442
591
  #discoveryOverloadError;
592
+ /** Relayfile operations this sweep has been shed on. */
593
+ #discoverySweepOverloads = 0;
594
+ /**
595
+ * The longest `Retry-After` any operation in this sweep advertised.
596
+ *
597
+ * `#discoveryOverloadError` latches the FIRST 429, so deriving the backoff
598
+ * from it alone would let the durable window expire before a later, longer
599
+ * advertised delay permits — breaking the very guarantee #297 is about. The
600
+ * backoff takes the maximum instead.
601
+ */
602
+ #discoverySweepRetryAfterSeconds;
603
+ /**
604
+ * Whether relayfile served at least one ready work unit end to end during
605
+ * this sweep — its issue read, or a dispatch built on it. This is what
606
+ * decays the durable overload ratchet; see `#discoveryOverloadOutcome`.
607
+ */
608
+ #discoverySweepProgress = false;
443
609
  #resolvedIssueSource;
444
610
  #integrationInstructions;
445
611
  #integrationInstructionsRefresh;
@@ -855,6 +1021,14 @@ export class FactoryLoop {
855
1021
  clearTimeout(this.#previewSweepTimer);
856
1022
  this.#previewSweepTimer = undefined;
857
1023
  await this.#readinessReconcileInFlight;
1024
+ // #301 review: the deadline ends the *wait*, so `#readinessReconcileInFlight`
1025
+ // can settle with its `runOnce()` still live. Shutdown releases dispatch
1026
+ // lifecycle leases and disposes ports below, and `#isPassFatalFailure` only
1027
+ // fences a stopping sweep once something in it throws — so a sweep whose
1028
+ // dependency recovers cleanly would otherwise dispatch through torn-down
1029
+ // state. Draining here restores exactly the pre-deadline shutdown contract:
1030
+ // stop() outlives the sweep it started.
1031
+ await this.#readinessReconcileAbandonedWait;
858
1032
  await this.#previewSweepInFlight;
859
1033
  this.#stoppingHeartbeatRefreshActive = await this.#stopLiveHeartbeat('stopping');
860
1034
  try {
@@ -907,6 +1081,14 @@ export class FactoryLoop {
907
1081
  await this.#boundedStopTeardown('factory subscription unsubscribe', () => subscription?.unsubscribe());
908
1082
  await Promise.all([...this.#slackWatchers.values()].map((watcher) => watcher.stop()));
909
1083
  this.#slackWatchers.clear();
1084
+ for (const timer of this.#slackTerminalWatchExpiryTimers.values())
1085
+ clearTimeout(timer);
1086
+ this.#slackTerminalWatchExpiryTimers.clear();
1087
+ for (const timer of this.#slackTerminalReceiptRetryTimers.values())
1088
+ clearTimeout(timer);
1089
+ this.#slackTerminalReceiptRetryTimers.clear();
1090
+ this.#terminalSlackWatchIssues.clear();
1091
+ this.#slackReplyRouteDrains.clear();
910
1092
  await Promise.all([...this.#githubIssueCommentWatchers.values()].map((watcher) => watcher.stop()));
911
1093
  this.#githubIssueCommentWatchers.clear();
912
1094
  this.#githubIssueCommentWatchStates.clear();
@@ -997,6 +1179,9 @@ export class FactoryLoop {
997
1179
  const options = this.#liveOptions(overrides);
998
1180
  this.#liveTransport = options.transport;
999
1181
  this.#readinessReconcileIntervalMs = options.reconcileIntervalMs;
1182
+ // `start()` overrides skip the schema's cross-field check, so re-apply its
1183
+ // floor here: a deadline under one interval would kill every pass.
1184
+ this.#readinessReconcileTimeoutMs = Math.max(options.reconcileTimeoutMs, options.reconcileIntervalMs);
1000
1185
  this.#liveConnectStartedAtMs = this.#clock.now();
1001
1186
  this.#liveReplaySkewMarginMs = options.replaySkewMarginMs;
1002
1187
  const highWatermark = await this.#currentEventHighWatermark();
@@ -1033,10 +1218,27 @@ export class FactoryLoop {
1033
1218
  this.#logger.info?.('[factory] running startup ready-issue backfill before draining buffered events', {
1034
1219
  highWatermarkRouteUnavailable: highWatermark.routeUnavailable,
1035
1220
  });
1221
+ // Review follow-up on #300 (P1, cubic). The startup backfill is a
1222
+ // discovery pass like any other, and it is the one most likely to hang:
1223
+ // #36 measured 61 minutes here while the Relayfile mirror hydrated on a
1224
+ // cold container. Stamping it means a wedged FIRST pass is visible as
1225
+ // in-flight, instead of leaving the timestamps empty and the derived
1226
+ // state reading `healthy` forever.
1227
+ //
1228
+ // Only the timestamps. `consecutiveFailures` and `lastError` belong to
1229
+ // the reconcile loop's own failure accounting, which owns the degraded
1230
+ // threshold and the #297 reason allowlist; a startup failure is already
1231
+ // counted by `liveStartupBackfillErrors` and reported through `#error`.
1232
+ const backfillStartedAtMs = this.#clock.now();
1233
+ this.#readinessReconcileLastStartedAtMs = backfillStartedAtMs;
1036
1234
  try {
1037
1235
  await this.runOnce();
1236
+ this.#readinessReconcileLastDurationMs = this.#elapsedSince(backfillStartedAtMs);
1237
+ this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
1038
1238
  }
1039
1239
  catch (error) {
1240
+ this.#readinessReconcileLastDurationMs = this.#elapsedSince(backfillStartedAtMs);
1241
+ this.#readinessReconcileLastFailureAtMs = this.#clock.now();
1040
1242
  // A startup backfill failure must not abort the daemon: log it and fall
1041
1243
  // back to the live event stream (plus any buffered events) instead of
1042
1244
  // leaving the factory down.
@@ -1128,6 +1330,7 @@ export class FactoryLoop {
1128
1330
  eventLimit: overrides.eventLimit ?? this.#config.liveSubscription.eventLimit,
1129
1331
  replaySkewMarginMs: overrides.replaySkewMarginMs ?? this.#config.liveSubscription.replaySkewMarginMs,
1130
1332
  reconcileIntervalMs: overrides.reconcileIntervalMs ?? this.#config.liveSubscription.reconcileIntervalMs,
1333
+ reconcileTimeoutMs: overrides.reconcileTimeoutMs ?? this.#config.liveSubscription.reconcileTimeoutMs,
1131
1334
  };
1132
1335
  }
1133
1336
  async #currentEventCursor(limit) {
@@ -1176,19 +1379,82 @@ export class FactoryLoop {
1176
1379
  }, delayMs);
1177
1380
  this.#readinessReconcileTimer.unref?.();
1178
1381
  }
1382
+ /**
1383
+ * Runs one sweep under a deadline (#296).
1384
+ *
1385
+ * The sweep itself cannot be cancelled — `runOnce()` owns a durable discovery
1386
+ * lease and abandoning it mid-flight is not safe — so expiry rejects *this*
1387
+ * wait and leaves the underlying pass to finish on its own. That is enough:
1388
+ * the rejection is what reaches the failure path, which re-arms the timer.
1389
+ * A later reconcile pass coalesces onto the still-running `runOnce()` and
1390
+ * fails on its own deadline too, so a persistent hang keeps counting up to
1391
+ * `degraded` instead of going quiet.
1392
+ */
1393
+ async #runOnceWithReadinessDeadline() {
1394
+ const timeoutMs = this.#readinessReconcileTimeoutMs;
1395
+ const startedAtMs = this.#clock.now();
1396
+ const sweep = this.runOnce();
1397
+ let timer;
1398
+ try {
1399
+ return await new Promise((resolve, reject) => {
1400
+ timer = setTimeout(() => {
1401
+ this.#increment('readinessReconcileDeadlineExceeded');
1402
+ if (this.#readinessReconcileAbandonedSinceMs === undefined) {
1403
+ // `state` ages from the FIRST wait that gave up on this work, not
1404
+ // from whenever the latest one began.
1405
+ this.#readinessReconcileAbandonedSinceMs = startedAtMs;
1406
+ // The abandoned pass is still running against the live control
1407
+ // plane. Report where it lands, so an operator can tell a
1408
+ // dependency that recovered late from one that never answered.
1409
+ // Attached once, so a wedge is reported once and not per expiry.
1410
+ void sweep.then((report) => this.#logger.warn?.('[factory] abandoned readiness sweep completed after its deadline', {
1411
+ timeoutMs,
1412
+ overrunMs: this.#elapsedSince(startedAtMs) - timeoutMs,
1413
+ dispatched: report.dispatched.length,
1414
+ }), (error) => this.#logger.warn?.('[factory] abandoned readiness sweep failed after its deadline', {
1415
+ timeoutMs,
1416
+ overrunMs: this.#elapsedSince(startedAtMs) - timeoutMs,
1417
+ error: describeError(error).errorMessage,
1418
+ })).catch(() => undefined);
1419
+ }
1420
+ // Newest wait wins as the drain target: it settles no earlier than
1421
+ // the ones before it, and clearing on it clears them all.
1422
+ const wait = sweep.catch(() => undefined).then(() => {
1423
+ if (this.#readinessReconcileAbandonedWait !== wait)
1424
+ return;
1425
+ this.#readinessReconcileAbandonedWait = undefined;
1426
+ this.#readinessReconcileAbandonedSinceMs = undefined;
1427
+ });
1428
+ this.#readinessReconcileAbandonedWait = wait;
1429
+ reject(new ReadinessReconcileTimeoutError(timeoutMs));
1430
+ }, timeoutMs);
1431
+ timer.unref?.();
1432
+ // Attaching handlers here is also what keeps a late rejection from the
1433
+ // abandoned pass from surfacing as an unhandled rejection.
1434
+ sweep.then(resolve, reject);
1435
+ });
1436
+ }
1437
+ finally {
1438
+ if (timer)
1439
+ clearTimeout(timer);
1440
+ }
1441
+ }
1179
1442
  async #reconcileReadyIssues() {
1180
1443
  const startedAtMs = this.#clock.now();
1181
1444
  this.#readinessReconcileLastStartedAtMs = startedAtMs;
1445
+ this.#readinessReconcileInFlightSinceMs = startedAtMs;
1182
1446
  this.#increment('readinessReconcileSweeps');
1183
1447
  this.#logger.info?.('[factory] periodic readiness reconciliation started', {
1184
1448
  intervalMs: this.#readinessReconcileIntervalMs,
1449
+ timeoutMs: this.#readinessReconcileTimeoutMs,
1185
1450
  });
1186
1451
  try {
1187
- const report = await this.runOnce();
1452
+ const report = await this.#runOnceWithReadinessDeadline();
1188
1453
  this.#readinessReconcileConsecutiveFailures = 0;
1189
1454
  this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
1190
1455
  this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
1191
1456
  this.#readinessReconcileLastError = undefined;
1457
+ this.#readinessReconcileLastErrorClass = undefined;
1192
1458
  this.#logger.info?.('[factory] periodic readiness reconciliation completed', {
1193
1459
  durationMs: this.#readinessReconcileLastDurationMs,
1194
1460
  candidates: report.pulled.length,
@@ -1197,11 +1463,28 @@ export class FactoryLoop {
1197
1463
  });
1198
1464
  }
1199
1465
  catch (error) {
1200
- const errorMessage = describeError(error).errorMessage;
1466
+ // #297: all four relayfile overload reason codes share one message, and
1467
+ // `lastError` is what an operator reads from /evidence. Without the
1468
+ // reason, "workspace durable object is busy" cannot be told apart from
1469
+ // three other conditions with three different remedies.
1470
+ //
1471
+ // Allowlisted, because this is a persisted operator-facing surface and
1472
+ // not just a log line: `lastError` is returned from `status()` and
1473
+ // written into the loop heartbeat file, so an unbounded
1474
+ // dependency-controlled string would land on disk.
1475
+ const overload = relayfileOverload(error);
1476
+ const errorMessage = overload
1477
+ ? `${describeError(error).errorMessage} ` +
1478
+ `[relayfile ${overload.status} ${relayfileOverloadReasonLabel(overload.reason)}` +
1479
+ `${overload.retryAfterSeconds === undefined ? '' : `; retry-after=${overload.retryAfterSeconds}s`}]`
1480
+ : describeError(error).errorMessage;
1201
1481
  this.#readinessReconcileConsecutiveFailures += 1;
1202
1482
  this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
1203
1483
  this.#readinessReconcileLastFailureAtMs = this.#clock.now();
1204
1484
  this.#readinessReconcileLastError = errorMessage;
1485
+ // The class, unlike the message, is publishable: #295 puts it on the
1486
+ // unauthenticated health surface through the same allowlist.
1487
+ this.#readinessReconcileLastErrorClass = telemetryErrorClass(error);
1205
1488
  this.#increment('readinessReconcileErrors');
1206
1489
  this.#logger.warn?.('[factory] periodic readiness reconciliation failed; retry remains scheduled', {
1207
1490
  error: errorMessage,
@@ -1210,6 +1493,11 @@ export class FactoryLoop {
1210
1493
  degraded: this.#readinessReconcileConsecutiveFailures >= READINESS_RECONCILE_FAILURE_THRESHOLD,
1211
1494
  });
1212
1495
  }
1496
+ finally {
1497
+ // Cleared before the heartbeat write below, so a slow-but-successful pass
1498
+ // does not stamp its own tail as `stalled`.
1499
+ this.#readinessReconcileInFlightSinceMs = undefined;
1500
+ }
1213
1501
  await this.#refreshLiveHeartbeat();
1214
1502
  }
1215
1503
  #scheduleLivePoll(delayMs, options) {
@@ -1938,13 +2226,29 @@ export class FactoryLoop {
1938
2226
  this.#discoverySweepStartedAtMs = sweepStartedAtMs;
1939
2227
  this.#discoverySweepLeaseLost = false;
1940
2228
  this.#discoveryOverloadError = undefined;
2229
+ this.#discoverySweepOverloads = 0;
2230
+ this.#discoverySweepRetryAfterSeconds = undefined;
2231
+ this.#discoverySweepProgress = false;
1941
2232
  this.#startDiscoverySweepRenewal(claim.lease.epoch);
1942
2233
  let leaseReleased = false;
1943
2234
  try {
1944
2235
  this.#discoverySession = await this.#prepareDiscoverySession(claim);
2236
+ // #297: a 429 raised anywhere in the sweep used to latch and be rethrown
2237
+ // here, discarding a completed pass — every issue read, every dispatch —
2238
+ // because of one transient shed operation. The work this sweep did is
2239
+ // now kept instead, and the ratchet below records that the dependency is
2240
+ // shedding but still serving.
1945
2241
  const report = await this.#performRunOnce(opts);
1946
- if (this.#discoveryOverloadError)
2242
+ // The exception, and the reason skipping shed units cannot make a sweep
2243
+ // unconditionally green: a sweep that was shed AND got no work unit
2244
+ // through accomplished nothing. There is no progress to preserve, and
2245
+ // committing it would report a clean sweep over a dependency that served
2246
+ // none of it — leaving `readinessReconcile` healthy while Factory
2247
+ // dispatches nothing, which is the #292 wedge wearing the other costume.
2248
+ // Fail it so the ratchet escalates and readiness reflects reality.
2249
+ if (this.#discoveryOverloadError !== undefined && !this.#discoverySweepProgress) {
1947
2250
  throw this.#discoveryOverloadError;
2251
+ }
1948
2252
  const checkpoint = await this.#finalizeDiscoveryCheckpoint();
1949
2253
  // Do not clear the durable lease while a renewal can still be waiting on
1950
2254
  // the same state-file lock. A late renewal that observes the completed
@@ -1954,7 +2258,8 @@ export class FactoryLoop {
1954
2258
  if (this.#discoverySweepLeaseLost) {
1955
2259
  throw new Error('discovery sweep lease was lost before checkpoint commit');
1956
2260
  }
1957
- const completed = await this.#state.completeDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, checkpoint);
2261
+ const residual = this.#discoveryOverloadOutcome(claim.state.consecutiveOverloads, 'committed');
2262
+ const completed = await this.#commitDiscoverySweep(claim.lease.epoch, checkpoint, residual);
1958
2263
  leaseReleased = completed;
1959
2264
  if (!completed)
1960
2265
  throw new Error('discovery sweep lease was lost before completion');
@@ -1970,18 +2275,18 @@ export class FactoryLoop {
1970
2275
  await this.#stopDiscoverySweepRenewal();
1971
2276
  const overload = relayfileOverload(error);
1972
2277
  if (overload) {
1973
- const consecutiveOverloads = claim.state.consecutiveOverloads + 1;
1974
- const delayMs = discoveryOverloadBackoffMs(overload.retryAfterSeconds, consecutiveOverloads);
1975
- const backoffUntilMs = this.#clock.now() + delayMs;
1976
- leaseReleased = await this.#state.deferDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, backoffUntilMs, consecutiveOverloads);
1977
- this.#increment('discoveryOverloadBackoffs');
2278
+ const outcome = this.#discoveryOverloadOutcome(claim.state.consecutiveOverloads, 'aborted', error);
2279
+ leaseReleased = await this.#state.deferDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, claim.lease.epoch, outcome.backoffUntilMs, outcome.consecutiveOverloads);
1978
2280
  this.#logger.warn?.('[factory] Relayfile discovery overloaded; backing off before another sweep', {
1979
2281
  status: overload.status,
1980
2282
  reason: overload.reason,
1981
- retryAfterSeconds: overload.retryAfterSeconds,
1982
- delayMs,
1983
- backoffUntilMs,
1984
- consecutiveOverloads,
2283
+ retryAfterSeconds: outcome.retryAfterSeconds,
2284
+ delayMs: outcome.delayMs,
2285
+ backoffUntilMs: outcome.backoffUntilMs,
2286
+ consecutiveOverloads: outcome.consecutiveOverloads,
2287
+ previousOverloads: claim.state.consecutiveOverloads,
2288
+ sweepOverloads: this.#discoverySweepOverloads,
2289
+ sweepProgress: this.#discoverySweepProgress,
1985
2290
  });
1986
2291
  // backoffUntilMs is already durable via deferDiscoverySweep, and the
1987
2292
  // next runOnce() honors it at the pre-claim wait above — sleeping
@@ -1997,6 +2302,9 @@ export class FactoryLoop {
1997
2302
  this.#discoverySweepEpoch = undefined;
1998
2303
  this.#discoverySweepStartedAtMs = undefined;
1999
2304
  this.#discoveryOverloadError = undefined;
2305
+ this.#discoverySweepOverloads = 0;
2306
+ this.#discoverySweepRetryAfterSeconds = undefined;
2307
+ this.#discoverySweepProgress = false;
2000
2308
  // This sweep is over either way (committed, deferred, or lease lost) —
2001
2309
  // a stale `true` here would otherwise make every #listRelayfileTree
2002
2310
  // call outside a fresh claim (Slack lookups, PR confirmation, the
@@ -2008,6 +2316,94 @@ export class FactoryLoop {
2008
2316
  }
2009
2317
  }
2010
2318
  }
2319
+ /**
2320
+ * Commit the sweep, carrying any residual overload backoff into the store.
2321
+ *
2322
+ * `completeDiscoverySweepWithOverload` is optional on the port so a store
2323
+ * written before #297 keeps working. When one of those is injected and this
2324
+ * sweep HAS a residual, the backoff cannot be persisted and the next sweep
2325
+ * would retry immediately — so this says so, loudly and with a counter,
2326
+ * rather than degrading in silence. The sweep itself still commits: losing
2327
+ * the backoff is worse than pre-#297 behaviour only if nobody notices.
2328
+ */
2329
+ async #commitDiscoverySweep(epoch, checkpoint, residual) {
2330
+ if (residual && this.#state.completeDiscoverySweepWithOverload) {
2331
+ return await this.#state.completeDiscoverySweepWithOverload(this.#workspaceId, this.#discoverySweepOwner, epoch, checkpoint, { consecutiveOverloads: residual.consecutiveOverloads, backoffUntilMs: residual.backoffUntilMs });
2332
+ }
2333
+ if (residual) {
2334
+ this.#increment('discoveryOverloadResidualUnsupported');
2335
+ this.#logger.warn?.('[factory] state store cannot persist the Relayfile overload backoff; it will be lost', {
2336
+ store: this.#state.constructor?.name,
2337
+ consecutiveOverloads: residual.consecutiveOverloads,
2338
+ backoffUntilMs: residual.backoffUntilMs,
2339
+ delayMs: residual.delayMs,
2340
+ });
2341
+ }
2342
+ return await this.#state.completeDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, epoch, checkpoint);
2343
+ }
2344
+ /**
2345
+ * How the durable overload ratchet should read after this sweep, or
2346
+ * `undefined` if relayfile never shed anything and the ordinary reset
2347
+ * applies.
2348
+ *
2349
+ * #297, deliverable 3. `consecutiveOverloads` used to clear only on a
2350
+ * *fully* clean sweep, so under sustained mild load the ratchet essentially
2351
+ * never cleared: it climbed to the cap on the first bad sweep and stayed
2352
+ * there, probing for recovery once per cap-length window, long after the
2353
+ * dependency had recovered. Requiring perfection to clear a ratchet means
2354
+ * the ratchet does not clear.
2355
+ *
2356
+ * The signal that replaces "was this sweep perfect" is "did relayfile serve
2357
+ * any of this sweep's work units", because that is what the ratchet is
2358
+ * actually for. A shedding DO rejects ALL background traffic —
2359
+ * `oldest_inflight_age` does this regardless of how few requests are in
2360
+ * flight — so a sweep against a genuinely overloaded workspace gets not one
2361
+ * unit through and still escalates, all the way to the cap. A sweep that did
2362
+ * get units through proves the dependency is serving us, so it decays by one
2363
+ * rung. Decay, not reset: getting work done while being shed is not evidence
2364
+ * that the overload is over, only that it is survivable.
2365
+ */
2366
+ #discoveryOverloadOutcome(previousOverloads, sweepOutcome, error) {
2367
+ const overload = relayfileOverload(error) ?? relayfileOverload(this.#discoveryOverloadError);
2368
+ if (!overload)
2369
+ return undefined;
2370
+ // The longest delay anything in this sweep asked for, not just the one the
2371
+ // latched or terminating error carried — see `#discoverySweepRetryAfterSeconds`.
2372
+ const advertised = [this.#discoverySweepRetryAfterSeconds, overload.retryAfterSeconds]
2373
+ .filter((value) => value !== undefined);
2374
+ const retryAfterSeconds = advertised.length > 0 ? Math.max(...advertised) : undefined;
2375
+ // Deliberately NOT "the sweep reached its end". A sweep in which relayfile
2376
+ // shed EVERY unit still runs the loop to the end, and it got nothing done;
2377
+ // treating that as progress would decay the ratchet in exactly the case it
2378
+ // exists for. (`#runOnceWithDiscoveryFence` turns that sweep into an
2379
+ // aborted one before it can commit, so the escalate branch here is reached
2380
+ // only through `sweepOutcome === 'aborted'` — but the rule is a property of
2381
+ // progress, not of which caller asked, and is written that way.)
2382
+ const consecutiveOverloads = this.#discoverySweepProgress
2383
+ ? Math.max(0, previousOverloads - 1)
2384
+ : previousOverloads + 1;
2385
+ const delayMs = discoveryOverloadBackoffMs(retryAfterSeconds, consecutiveOverloads);
2386
+ const backoffUntilMs = this.#clock.now() + delayMs;
2387
+ this.#increment('discoveryOverloadBackoffs');
2388
+ if (sweepOutcome === 'committed') {
2389
+ this.#logger.warn?.('[factory] discovery sweep committed despite Relayfile overload', {
2390
+ status: overload.status,
2391
+ reason: overload.reason,
2392
+ retryAfterSeconds,
2393
+ sweepOverloads: this.#discoverySweepOverloads,
2394
+ previousOverloads,
2395
+ consecutiveOverloads,
2396
+ delayMs,
2397
+ backoffUntilMs,
2398
+ });
2399
+ }
2400
+ return {
2401
+ consecutiveOverloads,
2402
+ backoffUntilMs,
2403
+ delayMs,
2404
+ ...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }),
2405
+ };
2406
+ }
2011
2407
  async #performRunOnce(opts = {}) {
2012
2408
  const dryRun = opts.dryRun ?? this.#config.dryRun;
2013
2409
  const startedAtMs = this.#clock.now();
@@ -2047,19 +2443,65 @@ export class FactoryLoop {
2047
2443
  reason: entry.reason,
2048
2444
  });
2049
2445
  };
2446
+ // Backstop for the skip-by-default catch below: see #292. Reset only by
2447
+ // a completed dispatch, so the name says "since a dispatch" rather than
2448
+ // "consecutive" — a benign classified skip in between does not clear it.
2449
+ let unclassifiedFailuresSinceDispatch = 0;
2050
2450
  let lastReadyReadProgressAtMs = this.#clock.now();
2051
2451
  let readyIssueReads = 0;
2052
2452
  const issueEntries = [];
2053
2453
  for (const path of paths) {
2054
- const issue = await this.#readIssue(path);
2454
+ let issue;
2455
+ let shed = false;
2456
+ try {
2457
+ issue = await this.#readIssue(path);
2458
+ }
2459
+ catch (error) {
2460
+ // #297: `#readIssue` rethrows relayfile overload and swallows every
2461
+ // other read fault, so this catch only ever sees the backend
2462
+ // shedding THIS issue's read. That is a fact about one work unit:
2463
+ // a sweep that pulled 40 issues and was shed on the 39th must keep
2464
+ // the other 39, exactly as #292 argued for dispatch failures. The
2465
+ // fuse is what still ends a sweep the backend is refusing.
2466
+ const overload = relayfileOverload(error);
2467
+ // Defensive: anything `#readIssue` did not swallow and is not a 429
2468
+ // is not ours to reclassify, and must surface as itself.
2469
+ if (!overload)
2470
+ throw error;
2471
+ const fuse = this.#discoveryOverloadFuseError();
2472
+ if (fuse)
2473
+ throw fuse;
2474
+ shed = true;
2475
+ this.#increment('discoveryOverloadItemsSkipped');
2476
+ this.#logger.warn?.('[factory] relayfile shed a ready-issue read; skipping it and continuing the sweep', {
2477
+ path,
2478
+ status: overload.status,
2479
+ reason: overload.reason,
2480
+ retryAfterSeconds: overload.retryAfterSeconds,
2481
+ sweepOverloads: this.#discoverySweepOverloads,
2482
+ });
2483
+ recordSkip({ issue: issueRefFromPath(path), reason: perItemDispatchSkipReason(error) });
2484
+ }
2055
2485
  readyIssueReads += 1;
2486
+ // Relayfile served this work unit's read: the dependency is shedding
2487
+ // but not refusing, which is what decays the ratchet (#297).
2488
+ //
2489
+ // `issue` is required, not just `!shed`: `#readIssue` returns
2490
+ // `undefined` for a body it could not read at all, and the known
2491
+ // phantom condition — the tree listing issue paths whose bodies are
2492
+ // absent — makes every read return `undefined`. Crediting those would
2493
+ // decay the ratchet on a sweep that served nothing.
2494
+ if (!shed && issue)
2495
+ this.#discoverySweepProgress = true;
2056
2496
  lastReadyReadProgressAtMs = this.#logTimedProgress(this.#config.issueSource === 'github'
2057
2497
  ? '[factory] GitHub ready issue read progress'
2058
2498
  : '[factory] Linear ready issue read progress', startedAtMs, lastReadyReadProgressAtMs, { read: readyIssueReads, total: paths.length, path });
2059
- if (issue && issueSource === 'linear') {
2060
- await this.#recordCanonicalIssueState(issue);
2499
+ if (!shed) {
2500
+ if (issue && issueSource === 'linear') {
2501
+ await this.#recordCanonicalIssueState(issue);
2502
+ }
2503
+ issueEntries.push({ path, issue });
2061
2504
  }
2062
- issueEntries.push({ path, issue });
2063
2505
  await this.#refreshLiveHeartbeatIfDue();
2064
2506
  }
2065
2507
  if (issueSource === 'github') {
@@ -2144,19 +2586,13 @@ export class FactoryLoop {
2144
2586
  }
2145
2587
  const decision = await this.triageIssue(issue);
2146
2588
  triaged.push(decision);
2147
- let result;
2148
- try {
2149
- result = await this.dispatch(decision, { dryRun });
2150
- }
2151
- catch (error) {
2152
- if (!(error instanceof LiveDispatchStateChangedError))
2153
- throw error;
2154
- recordSkip({ issue: decision.issue, reason: 'live state changed during dispatch' });
2155
- this.#logger.info?.('[factory] skipped issue whose live state changed during dispatch', {
2156
- issue: decision.issue.key,
2157
- });
2158
- continue;
2159
- }
2589
+ const result = await this.dispatch(decision, { dryRun });
2590
+ // A completed dispatch — even one that parks or escalates the issue —
2591
+ // proves the pipeline still works, so the fuse below starts over.
2592
+ unclassifiedFailuresSinceDispatch = 0;
2593
+ // ...and proves relayfile is still serving this sweep, which is what
2594
+ // decays the durable overload ratchet (#297).
2595
+ this.#discoverySweepProgress = true;
2160
2596
  if (result.agents.length === 0 && !dryRun) {
2161
2597
  const reason = result.hold?.kind === 'dependency-cycle'
2162
2598
  ? `dependency cycle detected: ${result.hold.cycle?.join(' -> ') ?? 'unknown cycle'}`
@@ -2169,6 +2605,81 @@ export class FactoryLoop {
2169
2605
  dispatched.push(result);
2170
2606
  }
2171
2607
  }
2608
+ catch (error) {
2609
+ // The failure may have left half-spawned agents behind, persisted as
2610
+ // failure handoffs on the way out of `#dispatchUnlocked`. runLoop's
2611
+ // catch used to reap them because every such error aborted the pass;
2612
+ // now that most of them are skipped, the reap has to happen here.
2613
+ //
2614
+ // BEFORE the fatality check, not after (#298 review, round two): the
2615
+ // 429 that trips the overload fuse aborts this pass, and a direct
2616
+ // `runOnce()` — the `factory run-once` CLI, `#reconcileReadyIssues` —
2617
+ // has no runLoop catch behind it, so the unit that trips the fuse
2618
+ // would leak the agents it had already spawned. Reaping first covers
2619
+ // the abort and the skip with one call; it is idempotent, so the
2620
+ // runLoop catch finding nothing left to do is free.
2621
+ if (mayHaveSpawnedBeforeFailing(error)) {
2622
+ await this.#reapDispatchFailureHandoffsNow();
2623
+ }
2624
+ // #292: issues in a pass are independent work units, so a failure
2625
+ // that is about ONE unit costs that unit and nothing else. Only the
2626
+ // conditions named in `#isPassFatalFailure` — the ones where
2627
+ // continuing the pass is meaningless — abort the whole sweep.
2628
+ if (this.#isPassFatalFailure(error, dryRun)) {
2629
+ // The overload fuse may have been tripped by a 429 that a caller
2630
+ // swallowed, leaving an unrelated error in hand. The fence keys
2631
+ // the durable backoff off `relayfileOverload(error)`, so hand it
2632
+ // the 429 rather than whatever surfaced last.
2633
+ throw this.#discoveryOverloadFuseError() ?? error;
2634
+ }
2635
+ const overload = relayfileOverload(error);
2636
+ if (overload) {
2637
+ // #297: shedding is a state of the dependency, not a fault of this
2638
+ // work unit, so it stays out of `counters.errors` and gets its own
2639
+ // counter — the same split #293 made for undispatchable units.
2640
+ this.#increment('discoveryOverloadItemsSkipped');
2641
+ this.#logger.warn?.('[factory] relayfile shed this work unit; skipping it and continuing the sweep', {
2642
+ issue: issueRef(issue).key,
2643
+ status: overload.status,
2644
+ reason: overload.reason,
2645
+ retryAfterSeconds: overload.retryAfterSeconds,
2646
+ sweepOverloads: this.#discoverySweepOverloads,
2647
+ });
2648
+ }
2649
+ else if (!isClassifiedPerItemDispatchFailure(error)) {
2650
+ unclassifiedFailuresSinceDispatch += 1;
2651
+ // A pass-wide fault can arrive disguised as a run of per-item
2652
+ // faults. Skipping every unit would then hand back a green report
2653
+ // that dispatched nothing, which is the same silent wedge #292 is
2654
+ // about, wearing the opposite costume. Fail the pass loudly so
2655
+ // `readinessReconcile.lastError` carries the cause.
2656
+ if (unclassifiedFailuresSinceDispatch >= UNCLASSIFIED_DISPATCH_FAILURE_LIMIT) {
2657
+ throw contextualError(`Aborting readiness pass after ${unclassifiedFailuresSinceDispatch} unclassified dispatch failures without a successful dispatch`, error);
2658
+ }
2659
+ this.#increment('dispatchItemFailuresSkipped');
2660
+ // The raw message is operator-facing only; the run report carries
2661
+ // the sanitized classification from `perItemDispatchSkipReason`.
2662
+ this.#logger.warn?.('[factory] skipped a work unit whose dispatch failed; continuing the pass', {
2663
+ issue: issueRef(issue).key,
2664
+ unclassifiedFailuresSinceDispatch,
2665
+ error: describeError(error).errorMessage,
2666
+ });
2667
+ this.#error(error, issueRef(issue));
2668
+ }
2669
+ else {
2670
+ // Not an error — the unit simply cannot be dispatched right now —
2671
+ // so this stays out of `counters.errors` and gets its own counter
2672
+ // instead, or a terminal-lifecycle backlog would be invisible to
2673
+ // anyone watching only `dispatchItemFailuresSkipped`.
2674
+ this.#increment('dispatchItemsSkippedUndispatchable');
2675
+ this.#logger.info?.('[factory] skipped a work unit that cannot be dispatched right now', {
2676
+ issue: issueRef(issue).key,
2677
+ error: describeError(error).errorMessage,
2678
+ });
2679
+ }
2680
+ recordSkip({ issue: issueRef(issue), reason: perItemDispatchSkipReason(error) });
2681
+ continue;
2682
+ }
2172
2683
  finally {
2173
2684
  if (recoveredIdentity)
2174
2685
  this.#reconciledGithubInProgress.delete(recoveredIdentity);
@@ -2202,6 +2713,103 @@ export class FactoryLoop {
2202
2713
  }
2203
2714
  }
2204
2715
  }
2716
+ /**
2717
+ * The 429 that ended this sweep, when relayfile has shed
2718
+ * `DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT` operations and skipping the next work
2719
+ * unit would just be grinding a shedding dependency. Undefined below that.
2720
+ *
2721
+ * Returns the *latched* 429 rather than whatever error is in hand, because
2722
+ * `#runOnceWithDiscoveryFence` keys the durable backoff off
2723
+ * `relayfileOverload(error)` and a caller may have swallowed the 429 that
2724
+ * tripped the fuse.
2725
+ */
2726
+ #discoveryOverloadFuseError() {
2727
+ if (this.#discoverySweepOverloads < DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT)
2728
+ return undefined;
2729
+ return this.#discoveryOverloadError;
2730
+ }
2731
+ /**
2732
+ * Whether a failure raised while processing ONE work unit must abort the
2733
+ * whole readiness pass instead of skipping that unit.
2734
+ *
2735
+ * The default is the opposite, and that inversion is the fix for #292.
2736
+ * Issues in a pass are independent work units: a failure that is *about one
2737
+ * unit* — its dispatch-lifecycle record, its live state, a provider fault on
2738
+ * its own writeback — costs that unit and nothing else. Before this, every
2739
+ * error except `LiveDispatchStateChangedError` escaped the `for` loop, so a
2740
+ * single issue whose lifecycle record had gone terminal stopped all dispatch
2741
+ * indefinitely, every pass.
2742
+ *
2743
+ * A condition belongs here only when continuing the pass is meaningless or
2744
+ * actively harmful — when the failure is about the *pass*, not the item:
2745
+ *
2746
+ * - The discovery sweep lease is gone. Another process now owns this
2747
+ * workspace's sweep, so every remaining read throws the same way and each
2748
+ * one would be recorded as an ordinary per-issue skip. The run report
2749
+ * would then claim a clean pass over work this process no longer has the
2750
+ * right to touch.
2751
+ * - Relayfile has shed `DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT` operations in
2752
+ * this sweep. A single shed operation is per-item and skippable (#297),
2753
+ * but past this the backend is not serving this sweep at all: grinding
2754
+ * through the remaining units makes it worse, and the fence needs the 429
2755
+ * to set the durable backoff.
2756
+ * - The factory is stopping. Teardown is in progress and dispatching more
2757
+ * agents now leaks them past the shutdown deadline.
2758
+ * - The fleet control-plane circuit is no longer closed, **on a live pass**.
2759
+ * Dispatch is globally paused — the same condition
2760
+ * `#assertFleetControlPlaneAvailable` refuses to *start* a live pass on,
2761
+ * so it must also stop one already in flight. A dry run is exempt: it
2762
+ * never calls that admission gate and never spawns, so a paused control
2763
+ * plane is irrelevant to it rather than fatal to it. Without the
2764
+ * exemption, one live pass that trips the circuit would poison every
2765
+ * later dry run — including the boot gate's own `run-once --dry-run`
2766
+ * probe, turning a recoverable circuit-open condition into a failed boot.
2767
+ * That is a nastier version of the wedge this whole change removes.
2768
+ *
2769
+ * Deliberately NOT here: JavaScript builtin error types. Classifying
2770
+ * "programmer faults" such as `TypeError` as fatal is the obvious next rule
2771
+ * and it is a trap — Node reports a failed `fetch` as `TypeError: fetch
2772
+ * failed`, which is precisely the transient per-item roster lookup that
2773
+ * wedged the second instance (#291). A rule keyed on builtin types would
2774
+ * have preserved that outage verbatim.
2775
+ *
2776
+ * Everything else — a refused lifecycle claim, a terminal lifecycle record,
2777
+ * a transient network fault on one issue — is per-item: record a skip and
2778
+ * keep going. The unclassified-failure fuse in `#performRunOnce` is the
2779
+ * backstop for a pass-wide fault that does not announce itself as one.
2780
+ */
2781
+ #isPassFatalFailure(error, dryRun) {
2782
+ // Sweep-scoped: these are about this process's right or ability to run the
2783
+ // pass at all, so they hold for a dry run exactly as for a live one.
2784
+ if (this.#discoverySweepLeaseLost || this.#stopping) {
2785
+ return true;
2786
+ }
2787
+ // #297: relayfile overload used to sit alongside those two, and it did not
2788
+ // belong there. A 429 on ONE work unit is a fact about that unit's read or
2789
+ // write, not about this process's right to run the pass — and because the
2790
+ // flag latched for the whole sweep, the first transient shed also made
2791
+ // every later unit fatal. Only sustained shedding is now pass-fatal.
2792
+ if (this.#discoverySweepOverloads >= DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT) {
2793
+ return true;
2794
+ }
2795
+ // Fleet-scoped, and therefore live-only. See the doc comment above.
2796
+ return !dryRun && this.#isFleetControlPlaneHalted(error);
2797
+ }
2798
+ /**
2799
+ * Whether dispatch is globally paused by the fleet control-plane circuit.
2800
+ *
2801
+ * Two reads, because the circuit announces itself two different ways. The
2802
+ * state read covers `guardedMutation`, which records a mutation's own
2803
+ * transport failure and rethrows the *original* error rather than the
2804
+ * circuit-open type — converting it there would be wrong, since the mutation
2805
+ * may already have reached the broker and callers key spawn-failure handling
2806
+ * off that original error. The type check covers a rejection raised without
2807
+ * any state transition, such as an already-open circuit refusing admission.
2808
+ */
2809
+ #isFleetControlPlaneHalted(error) {
2810
+ return this.#fleetControlPlane.status().state !== 'closed' ||
2811
+ wrapsErrorOfType(error, FleetControlPlaneCircuitOpenError);
2812
+ }
2205
2813
  #startDiscoverySweepRenewal(epoch) {
2206
2814
  this.#discoverySweepRenewTimer = setInterval(() => {
2207
2815
  if (this.#discoverySweepRenewalInFlight || this.#discoverySweepLeaseLost)
@@ -3047,14 +3655,40 @@ export class FactoryLoop {
3047
3655
  return result;
3048
3656
  }
3049
3657
  catch (error) {
3050
- if (relayfileOverload(error) && this.#discoverySweepEpoch !== undefined) {
3658
+ const overload = relayfileOverload(error);
3659
+ if (overload && this.#discoverySweepEpoch !== undefined) {
3051
3660
  this.#discoveryOverloadError ??= error;
3661
+ this.#discoverySweepOverloads += 1;
3662
+ if (overload.retryAfterSeconds !== undefined) {
3663
+ this.#discoverySweepRetryAfterSeconds = Math.max(this.#discoverySweepRetryAfterSeconds ?? 0, overload.retryAfterSeconds);
3664
+ }
3665
+ this.#increment('discoveryOverloadOperations');
3666
+ // #297: relayfile's four overload reason codes — inflight_limit,
3667
+ // oldest_inflight_age, router_inflight_limit, durable_object_overloaded
3668
+ // — all share ONE message string, and they mean four different things:
3669
+ // a DO-local admission cap, one stuck op poisoning every background
3670
+ // caller, a Worker-global isolate cap, and the Cloudflare runtime
3671
+ // shedding the object outright. `relayfileOverload()` has always parsed
3672
+ // the reason; nothing on this path ever printed it, and during the
3673
+ // 2026-08-20 outage that ambiguity was the single biggest obstacle to
3674
+ // diagnosis. Unconditional, unlike the failure warn below: a 429 is
3675
+ // always worth one line.
3676
+ this.#increment(`discoveryOverloadReason:${relayfileOverloadReasonLabel(overload.reason)}`);
3677
+ this.#logger.warn?.('[factory] relayfile shed a discovery operation', {
3678
+ ...metadata,
3679
+ status: overload.status,
3680
+ reason: overload.reason,
3681
+ retryAfterSeconds: overload.retryAfterSeconds,
3682
+ sweepOverloads: this.#discoverySweepOverloads,
3683
+ elapsedMs: this.#elapsedSince(startedAtMs),
3684
+ });
3052
3685
  }
3053
3686
  if (opts.logFailure || waitWarnings > 0) {
3054
3687
  this.#increment('relayfileOperationFailures');
3055
3688
  this.#logger.warn?.('[factory] relayfile operation failed', {
3056
3689
  ...metadata,
3057
3690
  elapsedMs: this.#elapsedSince(startedAtMs),
3691
+ ...(overload ? { status: overload.status, reason: overload.reason } : {}),
3058
3692
  error: describeError(error).errorMessage,
3059
3693
  });
3060
3694
  }
@@ -3085,6 +3719,7 @@ export class FactoryLoop {
3085
3719
  const maxConsecutiveFailures = Math.min(5, Math.max(1, Math.trunc(opts.maxConsecutiveFailures ?? this.#config.loop.maxConsecutiveFailures)));
3086
3720
  const heartbeatPath = opts.heartbeatPath ?? this.#config.loop.heartbeatPath;
3087
3721
  const registryPath = opts.registryPath ?? this.#config.loop.registryPath;
3722
+ this.#loopReapPaths = { heartbeatPath, registryPath };
3088
3723
  const reports = [];
3089
3724
  let consecutiveFailures = 0;
3090
3725
  let completed = false;
@@ -3140,6 +3775,7 @@ export class FactoryLoop {
3140
3775
  return reports;
3141
3776
  }
3142
3777
  finally {
3778
+ this.#loopReapPaths = undefined;
3143
3779
  if (!completed) {
3144
3780
  await this.#writeLoopHeartbeat(heartbeatPath, registryPath, 'stopping', reports.length, maxIterations);
3145
3781
  }
@@ -3615,17 +4251,66 @@ export class FactoryLoop {
3615
4251
  }
3616
4252
  #readinessReconcileStatus() {
3617
4253
  const consecutiveFailures = this.#readinessReconcileConsecutiveFailures;
3618
- const state = this.#startMode !== 'live'
4254
+ // #296 owns the numerator here, #295/#300 own the derivation. The earliest
4255
+ // sweep still running — after a deadline expiry that is the abandoned one,
4256
+ // not the current wait, or every expiry would restart the clock and a
4257
+ // permanently stuck pass would read as merely `retrying`.
4258
+ const inFlightSinceMs = Math.min(this.#readinessReconcileInFlightSinceMs ?? Number.POSITIVE_INFINITY, this.#readinessReconcileAbandonedSinceMs ?? Number.POSITIVE_INFINITY);
4259
+ const settled = this.#startMode !== 'live'
3619
4260
  ? 'not-running'
4261
+ // Failure-count ladder only. Precedence against `stalled` belongs to
4262
+ // `derivedReadinessReconcileState`, which outranks everything here: a
4263
+ // stall is the more specific fact, and `consecutiveFailures` ships
4264
+ // alongside so nothing an alarm keyed on `degraded` needs is lost.
3620
4265
  : consecutiveFailures >= READINESS_RECONCILE_FAILURE_THRESHOLD
3621
4266
  ? 'degraded'
3622
4267
  : consecutiveFailures > 0
3623
4268
  ? 'retrying'
3624
4269
  : 'healthy';
4270
+ // The counters above only move when a pass *settles*. A pass that hangs
4271
+ // takes neither path, so `settled` would keep reporting the last finished
4272
+ // pass — `healthy` — for as long as the process is stuck (#295). The
4273
+ // in-flight age is the only field that can express that, so derive the
4274
+ // state from it rather than trusting the last write.
4275
+ const timestamps = {
4276
+ intervalMs: this.#readinessReconcileIntervalMs,
4277
+ ...(Number.isFinite(inFlightSinceMs) ? { inFlightSinceMs } : {}),
4278
+ ...(this.#readinessReconcileLastStartedAtMs !== undefined
4279
+ ? { lastStartedAtMs: this.#readinessReconcileLastStartedAtMs }
4280
+ : {}),
4281
+ ...(this.#readinessReconcileLastCompletedAtMs !== undefined
4282
+ ? { lastCompletedAtMs: this.#readinessReconcileLastCompletedAtMs }
4283
+ : {}),
4284
+ ...(this.#readinessReconcileLastFailureAtMs !== undefined
4285
+ ? { lastFailureAtMs: this.#readinessReconcileLastFailureAtMs }
4286
+ : {}),
4287
+ };
4288
+ const nowMs = this.#clock.now();
4289
+ // Defence in depth (#300 review, CodeRabbit). These derivations are new
4290
+ // code from another module on a path that `status()` and every heartbeat
4291
+ // write depend on. A throw here would take out the liveness signal the
4292
+ // crash reaper reads — the diagnostic causing the outage it exists to
4293
+ // explain — so a failure costs the derived fields and nothing else.
4294
+ let inFlightMs;
4295
+ let derived = settled;
4296
+ try {
4297
+ inFlightMs = readinessReconcileInFlightMs(timestamps, nowMs);
4298
+ derived = derivedReadinessReconcileState({ ...timestamps, state: settled }, nowMs);
4299
+ }
4300
+ catch (error) {
4301
+ this.#logger.warn?.('[factory] readiness health derivation failed; reporting the settled state', {
4302
+ error: describeError(error).errorMessage,
4303
+ });
4304
+ inFlightMs = undefined;
4305
+ derived = settled;
4306
+ }
3625
4307
  return {
3626
- state,
4308
+ state: derived === 'unknown' ? settled : derived,
3627
4309
  consecutiveFailures,
3628
4310
  failureThreshold: READINESS_RECONCILE_FAILURE_THRESHOLD,
4311
+ intervalMs: this.#readinessReconcileIntervalMs,
4312
+ ...(Number.isFinite(inFlightSinceMs) ? { inFlightSinceMs } : {}),
4313
+ ...(inFlightMs !== undefined ? { inFlightMs } : {}),
3629
4314
  ...(this.#readinessReconcileLastDurationMs !== undefined
3630
4315
  ? { lastDurationMs: this.#readinessReconcileLastDurationMs }
3631
4316
  : {}),
@@ -3639,6 +4324,9 @@ export class FactoryLoop {
3639
4324
  ? { lastFailureAtMs: this.#readinessReconcileLastFailureAtMs }
3640
4325
  : {}),
3641
4326
  ...(this.#readinessReconcileLastError ? { lastError: this.#readinessReconcileLastError } : {}),
4327
+ ...(this.#readinessReconcileLastErrorClass
4328
+ ? { lastErrorClass: this.#readinessReconcileLastErrorClass }
4329
+ : {}),
3642
4330
  };
3643
4331
  }
3644
4332
  on(event, listener) {
@@ -4142,10 +4830,11 @@ export class FactoryLoop {
4142
4830
  seed.decision = decisionWithLifecycleBranches(seed.decision, seed.runId);
4143
4831
  const claim = await this.#state.claimDispatchLifecycle(this.#workspaceId, key, seed, this.#dispatchLifecycleOwner, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS);
4144
4832
  if (!claim.acquired || !claim.lease) {
4145
- const reason = isTerminalDispatchLifecycle(claim.lifecycle)
4833
+ const terminal = isTerminalDispatchLifecycle(claim.lifecycle);
4834
+ const reason = terminal
4146
4835
  ? 'dispatch lifecycle is already terminal'
4147
4836
  : `dispatch lifecycle is owned by ${claim.lifecycle.lease?.owner ?? 'another publisher'}`;
4148
- throw new Error(`Refusing to dispatch ${decision.issue.key}: ${reason}`);
4837
+ throw new DispatchLifecycleClaimRefusedError(decision.issue.key, terminal ? 'terminal' : 'owned-elsewhere', `Refusing to dispatch ${decision.issue.key}: ${reason}`);
4149
4838
  }
4150
4839
  this.#dispatchLifecycleEpochs.set(claim.key ?? key, claim.lease.epoch);
4151
4840
  this.#hydrateCostLedger(claim.lifecycle);
@@ -5008,7 +5697,7 @@ export class FactoryLoop {
5008
5697
  for (const [name] of record.agents) {
5009
5698
  this.#fleet.markAgentTerminal?.(name, 'durable-dispatch-abandoned');
5010
5699
  }
5011
- await this.#stopSlackWatcher(record.issue);
5700
+ await this.#retireSlackWatcher(record);
5012
5701
  await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
5013
5702
  await this.#writeInFlightRegistry();
5014
5703
  this.#increment('dispatchLifecycleStaleIssuesAbandoned');
@@ -6114,6 +6803,29 @@ export class FactoryLoop {
6114
6803
  readinessReconcile: this.#readinessReconcileStatus(),
6115
6804
  fleetControlPlane: this.#fleetControlPlane.status(),
6116
6805
  };
6806
+ // The deployed container serves `/healthz` straight out of this file and
6807
+ // has no redaction logic of its own, so publish the already-safe view here
6808
+ // rather than leaving that boundary to whoever reads the file (#295).
6809
+ // Derived against this daemon's clock: every duration in it is a
6810
+ // difference between timestamps this process wrote.
6811
+ //
6812
+ // Guarded (#300 review, CodeRabbit): this heartbeat is what the crash
6813
+ // reaper and `/healthz` read to decide the daemon is alive, and several
6814
+ // callers of this method sit outside any try/catch. A projection failure
6815
+ // must cost the diagnostics block, never the heartbeat — the omitted block
6816
+ // is itself legible, since `factory diagnose` reports a missing one rather
6817
+ // than a false green.
6818
+ try {
6819
+ heartbeat.health = publicHealthFromHeartbeat(heartbeat, {
6820
+ nowMs: updatedAtMs,
6821
+ staleMs: this.#config.loop.heartbeatStaleMs,
6822
+ });
6823
+ }
6824
+ catch (error) {
6825
+ this.#logger.warn?.('[factory] public health projection failed; heartbeat written without it', {
6826
+ error: describeError(error).errorMessage,
6827
+ });
6828
+ }
6117
6829
  await mkdir(dirname(path), { recursive: true });
6118
6830
  await writeFile(path, `${JSON.stringify(heartbeat, null, 2)}\n`, 'utf8');
6119
6831
  await this.#writeInFlightRegistry(registryPath, path);
@@ -6131,12 +6843,19 @@ export class FactoryLoop {
6131
6843
  },
6132
6844
  });
6133
6845
  }
6134
- async #reapDispatchFailureHandoffsNow(heartbeatPath, registryPath) {
6135
- const handoffs = await this.#state.listFailureHandoffs(this.#workspaceId);
6136
- if (handoffs.length === 0) {
6137
- return;
6138
- }
6846
+ async #reapDispatchFailureHandoffsNow(heartbeatPath = this.#loopReapPaths?.heartbeatPath ?? this.#config.loop.heartbeatPath, registryPath = this.#loopReapPaths?.registryPath ?? this.#config.loop.registryPath) {
6139
6847
  try {
6848
+ // Inside the try, not before it (#298 review). Every caller reaps while
6849
+ // already handling a failure and then propagates that failure: the
6850
+ // per-item catch rethrows the dispatch error, runLoop's catch is mid
6851
+ // teardown. A throw from here would REPLACE the error in flight — and a
6852
+ // replaced 429 is no longer recognised as overload at the discovery
6853
+ // fence, silently dropping the advertised backoff. Reaping is
6854
+ // best-effort by construction; the caller's error always wins.
6855
+ const handoffs = await this.#state.listFailureHandoffs(this.#workspaceId);
6856
+ if (handoffs.length === 0) {
6857
+ return;
6858
+ }
6140
6859
  const protectedPids = await this.#protectedPids();
6141
6860
  let registryChanged = false;
6142
6861
  const readyToClear = new Set();
@@ -7149,6 +7868,7 @@ export class FactoryLoop {
7149
7868
  }
7150
7869
  async #publishImplementerPullRequest(record, implementer, opts = {}) {
7151
7870
  const key = `${issueKey(record.issue)}:${implementer.spec.repo}`;
7871
+ const trajectorySessionRef = canonicalTrajectorySessionRef(implementer.sessionRef);
7152
7872
  const expectedHeadRef = implementer.spec.branch;
7153
7873
  if (!expectedHeadRef) {
7154
7874
  throw new Error(`Refusing to publish ${record.issue.key}: implementer has no Factory-derived branch`);
@@ -7214,7 +7934,7 @@ export class FactoryLoop {
7214
7934
  expectedHeadRef,
7215
7935
  baseRef,
7216
7936
  title: `${issue.key}: ${issue.title}`,
7217
- body: githubPullRequestBody(issue, implementer.spec.preview),
7937
+ body: githubPullRequestBody(issue, implementer.spec.preview, trajectorySessionRef),
7218
7938
  ...(implementer.sessionRef ? { sessionRef: implementer.sessionRef } : {}),
7219
7939
  });
7220
7940
  const published = result.author
@@ -7803,7 +8523,7 @@ export class FactoryLoop {
7803
8523
  await this.#recordDispatchTerminal(record.issue);
7804
8524
  const next = (await this.#batch()).complete(record.issue);
7805
8525
  await this.#drainReadyClarificationWake();
7806
- await this.#stopSlackWatcher(record.issue);
8526
+ await this.#retireSlackWatcher(record);
7807
8527
  await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
7808
8528
  await this.#writeInFlightRegistry();
7809
8529
  if (next) {
@@ -11763,7 +12483,7 @@ export class FactoryLoop {
11763
12483
  }
11764
12484
  if (!await this.#saveDispatchLifecycle(record, 'releasing', undefined, releaseReason))
11765
12485
  return;
11766
- await this.#stopSlackWatcher(record.issue);
12486
+ await this.#retireSlackWatcher(record);
11767
12487
  await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
11768
12488
  await this.#recordDispatchTerminal(record.issue);
11769
12489
  await this.#finishDurableRelease(record, releaseReason);
@@ -12241,6 +12961,21 @@ export class FactoryLoop {
12241
12961
  return;
12242
12962
  }
12243
12963
  const key = issueKey(record.issue);
12964
+ const previousWatch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
12965
+ .find(([watchKey]) => watchKey === key)?.[1];
12966
+ if (previousWatch?.kind === 'terminal-grace') {
12967
+ // A reopened work unit needs a fresh dispatch notification and a fresh
12968
+ // conversation. Do not let the old grace-period watcher (or its expiry
12969
+ // timer) capture and later tear down the new dispatch.
12970
+ if (!await this.#stopSlackWatcher(record.issue)) {
12971
+ // Fail closed. An undrained reply route still holds the retired thread
12972
+ // and would bind it to this dispatch, delivering a stale human reply to
12973
+ // fresh work. Leave the fence up; the next reconcile retries the drain.
12974
+ this.#logger.warn?.('[factory] deferring Slack dispatch thread for reopened work unit; in-flight reply route not drained', { issue: record.issue.key });
12975
+ this.#increment('slackDispatchThreadsDeferredUndrainedReply');
12976
+ return;
12977
+ }
12978
+ }
12244
12979
  const existingThread = await this.#persistedSlackThread(key);
12245
12980
  const watcherStart = this.#slackWatcherStarts.get(key);
12246
12981
  if (existingThread || watcherStart) {
@@ -12323,9 +13058,10 @@ export class FactoryLoop {
12323
13058
  if (existing) {
12324
13059
  const sessionRef = owned?.tracked.sessionRef;
12325
13060
  const agentName = owned ? (owned.tracked.result?.name ?? owned.name) : undefined;
13061
+ let rebound = false;
12326
13062
  if (owned && sessionRef &&
12327
- (agentName !== existing.agent.name || (options.forceAgentRebind === true && sessionRef !== existing.agent.sessionRef))) {
12328
- const rebound = await this.#state.rebindConversationSession(this.#workspaceId, conversationId, {
13063
+ (!existing.agent || agentName !== existing.agent.name || (options.forceAgentRebind === true && sessionRef !== existing.agent.sessionRef))) {
13064
+ rebound = await this.#state.rebindConversationSession(this.#workspaceId, conversationId, {
12329
13065
  name: agentName,
12330
13066
  sessionRef,
12331
13067
  role: owned.tracked.spec.role,
@@ -12339,40 +13075,39 @@ export class FactoryLoop {
12339
13075
  }
12340
13076
  if (existing.pending.length > 0 || existing.delivery) {
12341
13077
  const waiting = await this.#state.getWaitingClarification(this.#workspaceId, issueKey(existing.issue));
12342
- if (!waiting)
13078
+ if (!waiting && (existing.agent || rebound))
12343
13079
  this.#slackConversationTurns.schedule(conversationId);
12344
13080
  }
12345
13081
  return;
12346
13082
  }
12347
13083
  const sessionRef = owned?.tracked.sessionRef;
12348
- if (!owned || !sessionRef) {
12349
- this.#increment('slackConversationSessionsSkippedMissingSession');
12350
- return;
12351
- }
12352
13084
  const channelDir = await this.#slackChannelDir() ?? this.#config.slack?.channel;
12353
13085
  if (!channelDir)
12354
13086
  return;
12355
- const agentName = owned.tracked.result?.name ?? owned.name;
13087
+ const agentName = owned ? (owned.tracked.result?.name ?? owned.name) : undefined;
12356
13088
  const reserved = await this.#state.reserveConversationSession(this.#workspaceId, conversationId, {
12357
13089
  provider: 'slack',
12358
13090
  issue: { ...record.issue },
12359
13091
  externalId: threadId,
12360
13092
  context: { channelDir },
12361
- agent: {
12362
- name: agentName,
12363
- sessionRef,
12364
- role: owned.tracked.spec.role,
12365
- node: owned.tracked.result?.node ?? owned.tracked.spec.node,
12366
- capability: owned.tracked.spec.capability,
12367
- repo: owned.tracked.spec.repo,
12368
- clonePath: owned.tracked.spec.clonePath,
12369
- },
13093
+ ...(owned && sessionRef && agentName ? { agent: {
13094
+ name: agentName,
13095
+ sessionRef,
13096
+ role: owned.tracked.spec.role,
13097
+ node: owned.tracked.result?.node ?? owned.tracked.spec.node,
13098
+ capability: owned.tracked.spec.capability,
13099
+ repo: owned.tracked.spec.repo,
13100
+ clonePath: owned.tracked.spec.clonePath,
13101
+ } } : {}),
12370
13102
  history: [],
12371
13103
  processedMessageIds: [],
13104
+ acknowledgedMessageIds: [],
13105
+ acknowledgementClaims: {},
12372
13106
  pending: [],
12373
13107
  });
12374
- if (reserved)
12375
- this.#increment('slackConversationSessionsOwned');
13108
+ if (reserved) {
13109
+ this.#increment(owned && sessionRef ? 'slackConversationSessionsOwned' : 'slackConversationSessionsReservedUnowned');
13110
+ }
12376
13111
  }
12377
13112
  // Called right after a babysitter is spawned/reattached for an issue's PR so
12378
13113
  // an already-owned Slack conversation session (reserved earlier by the
@@ -12400,11 +13135,15 @@ export class FactoryLoop {
12400
13135
  const claimed = await this.#state.claimConversationTurn(this.#workspaceId, conversationId, this.#slackConversationOwner, claimId, this.#clock.now(), SLACK_CONVERSATION_TURN_LEASE_MS);
12401
13136
  if (!claimed?.delivery) {
12402
13137
  const current = await this.#state.getConversationSession(this.#workspaceId, conversationId);
12403
- if (current && (current.pending.length > 0 || current.delivery)) {
13138
+ if (current?.agent && (current.pending.length > 0 || current.delivery)) {
12404
13139
  this.#slackConversationTurns.schedule(conversationId, SLACK_CONVERSATION_TURN_RETRY_MS);
12405
13140
  }
12406
13141
  return;
12407
13142
  }
13143
+ if (!claimed.agent) {
13144
+ await this.#state.releaseConversationTurn(this.#workspaceId, conversationId, this.#slackConversationOwner, claimId);
13145
+ return;
13146
+ }
12408
13147
  if (!await this.#ownsActiveSlackConversationIssue(claimed.issue)) {
12409
13148
  await this.#state.releaseConversationTurn(this.#workspaceId, conversationId, this.#slackConversationOwner, claimId);
12410
13149
  this.#increment('slackConversationTurnsSuppressedStaleOwner');
@@ -12488,10 +13227,13 @@ export class FactoryLoop {
12488
13227
  }
12489
13228
  }
12490
13229
  async #recordSlackConversationResume(session, result) {
13230
+ const sessionAgent = session.agent;
13231
+ if (!sessionAgent)
13232
+ return;
12491
13233
  const record = (await this.#batch()).getIssue(session.issue);
12492
13234
  if (!record)
12493
13235
  return;
12494
- const entry = [...record.agents.entries()].find(([name, tracked]) => name === session.agent.name || tracked.result?.name === session.agent.name);
13236
+ const entry = [...record.agents.entries()].find(([name, tracked]) => name === sessionAgent.name || tracked.result?.name === sessionAgent.name);
12495
13237
  if (!entry)
12496
13238
  return;
12497
13239
  const [previousName, tracked] = entry;
@@ -12817,7 +13559,14 @@ export class FactoryLoop {
12817
13559
  `Question: ${triageEscalationQuestion(decision, issue)}`,
12818
13560
  ].join('\n'),
12819
13561
  });
12820
- await this.#state.setSlackThread(this.#workspaceId, issueKey(decision.issue), root.threadId);
13562
+ const key = issueKey(decision.issue);
13563
+ await this.#state.setSlackThread(this.#workspaceId, key, root.threadId);
13564
+ await this.#state.setSlackThreadWatch(this.#workspaceId, key, {
13565
+ kind: 'triage',
13566
+ issue: { ...decision.issue },
13567
+ decision: structuredClone(decision),
13568
+ threadId: root.threadId,
13569
+ });
12821
13570
  const replayedResult = await this.#watchSlackThread(escalationWatchRecord(decision), root.threadId);
12822
13571
  this.#recordSlackWritebackSuccess('triage-escalation');
12823
13572
  return replayedResult;
@@ -12883,6 +13632,11 @@ export class FactoryLoop {
12883
13632
  if (!reply || !reply.isThreadReply || reply.threadTs !== threadId || reply.channelDir !== channelDir) {
12884
13633
  return;
12885
13634
  }
13635
+ if (allowPreExisting &&
13636
+ options.replayAfterMs !== undefined &&
13637
+ slackMessageReceivedAtMs(reply.messageTs, Number.MAX_SAFE_INTEGER) < options.replayAfterMs) {
13638
+ return;
13639
+ }
12886
13640
  const replyMessageKey = `${reply.threadTs}:${reply.messageTs}`;
12887
13641
  if (seenReplyMessages.has(replyMessageKey)) {
12888
13642
  this.#logger.debug?.('[factory] suppressed duplicate Slack reply message', { issue: record.issue.key, path });
@@ -13061,6 +13815,47 @@ export class FactoryLoop {
13061
13815
  this.#slackConversationTurns.schedule(conversationId);
13062
13816
  }
13063
13817
  }
13818
+ for (const [key, watch] of await this.#state.listSlackThreadWatches(this.#workspaceId)) {
13819
+ if (this.#slackWatchers.has(key) || this.#slackWatcherStarts.has(key))
13820
+ continue;
13821
+ if (watch.kind === 'terminal-grace' && watch.expiresAtMs <= this.#clock.now()) {
13822
+ await this.#stopSlackWatcher(watch.issue);
13823
+ continue;
13824
+ }
13825
+ await this.#state.setSlackThread(this.#workspaceId, key, watch.threadId);
13826
+ const watchRecord = escalationWatchRecord(watch.decision);
13827
+ if (watch.kind === 'terminal-grace') {
13828
+ const retiredAtMs = terminalSlackWatchRetiredAtMs(watch);
13829
+ if (watch.retiredAtMs !== retiredAtMs) {
13830
+ await this.#state.setSlackThreadWatch(this.#workspaceId, key, { ...watch, retiredAtMs });
13831
+ }
13832
+ this.#terminalSlackWatchIssues.add(key);
13833
+ const conversationId = slackConversationId(watch.threadId);
13834
+ await this.#slackConversationTurns.cancel(conversationId);
13835
+ try {
13836
+ await this.#surfaceUndeliveredSlackConversation(watch.threadId);
13837
+ await this.#state.clearConversationSession(this.#workspaceId, conversationId);
13838
+ }
13839
+ catch (error) {
13840
+ // The undelivered-reply receipt needs Slack writeback, which may be
13841
+ // unavailable at startup. That is retryable state maintenance for this
13842
+ // one thread, not a reason to abandon rehydration: aborting here would
13843
+ // leave every remaining thread watched by nobody. Keep the queued
13844
+ // replies (clearing them now would drop replies nobody was told about)
13845
+ // and carry on re-arming.
13846
+ this.#logger.warn?.('[factory] failed to settle undelivered Slack replies for terminal watch; will retry', { issue: watch.issue.key, error });
13847
+ this.#increment('slackTerminalWatchReceiptsDeferred');
13848
+ this.#scheduleSlackTerminalReceiptRetry(watch.issue, watch.threadId, watch.expiresAtMs);
13849
+ }
13850
+ await this.#rearmSlackWatcher(watchRecord, watch.threadId, {
13851
+ replayConversationReplies: true,
13852
+ replayAfterMs: retiredAtMs,
13853
+ });
13854
+ this.#scheduleSlackTerminalWatchExpiry(watch.issue, watch.expiresAtMs);
13855
+ continue;
13856
+ }
13857
+ await this.#rearmSlackWatcher(watchRecord, watch.threadId, { replayConversationReplies: true });
13858
+ }
13064
13859
  await this.#sweepWaitingClarifications();
13065
13860
  for (const [, waiting] of await this.#state.listWaitingClarifications(this.#workspaceId)) {
13066
13861
  if (!waiting.threadId)
@@ -13206,8 +14001,73 @@ export class FactoryLoop {
13206
14001
  this.#clarificationSweepTimer = timer;
13207
14002
  this.#clarificationSweepDueAtMs = dueAtMs;
13208
14003
  }
14004
+ // The terminal fence is the only thing that makes an in-flight reply route
14005
+ // answer "no active agent" instead of binding the retired thread to whatever
14006
+ // dispatch owns this key. Routes are chained per work unit, so awaiting the
14007
+ // newest one drains every reply queued behind it.
14008
+ async #drainSlackReplyRoutes(key) {
14009
+ // Snapshotting #slackReplyRoutes is not enough on its own. A reply handler
14010
+ // that is still inside its mount read when the drain starts registers its
14011
+ // route *after* the snapshot, so it would run once the fence is gone and
14012
+ // bind the retired thread to the next dispatch — the same escape one level
14013
+ // in. Bar *routing* for this key first (the bar and the registration are
14014
+ // both synchronous, so nothing can slip between them), then drain whatever
14015
+ // is already chained, then prove the set is empty before reporting success.
14016
+ // A barred reply still answers the human, and that writeback registers here
14017
+ // like any other effect, so the extra passes are what pick it up: quiescence
14018
+ // means every effect this work unit started has settled, not merely the ones
14019
+ // that existed when the drain began.
14020
+ const nested = this.#slackReplyRouteDrains.has(key);
14021
+ this.#slackReplyRouteDrains.add(key);
14022
+ try {
14023
+ for (let pass = 0; pass < SLACK_REPLY_ROUTE_DRAIN_PASSES; pass += 1) {
14024
+ const route = this.#slackReplyRoutes.get(key);
14025
+ if (!route)
14026
+ return true;
14027
+ try {
14028
+ await route;
14029
+ }
14030
+ catch (error) {
14031
+ // A route that *rejects* is not drained: the watcher replays it after
14032
+ // SLACK_REPLY_ROUTE_RETRY_MS, and that replay would land on the next
14033
+ // dispatch. Fail closed and let the caller keep the fence up rather
14034
+ // than leak a stale human reply onto fresh work.
14035
+ this.#logger.warn?.('[factory] in-flight Slack reply route did not drain; keeping terminal Slack fence', { issue: key, error });
14036
+ this.#increment('slackReplyRouteDrainsFailed');
14037
+ return false;
14038
+ }
14039
+ // The owner clears its own entry when it settles; retiring it here too
14040
+ // keeps the loop monotonic if that finally has not run yet.
14041
+ if (this.#slackReplyRoutes.get(key) === route)
14042
+ this.#slackReplyRoutes.delete(key);
14043
+ }
14044
+ // Not provably quiescent. Fail closed for the same reason as a rejection.
14045
+ this.#logger.warn?.('[factory] Slack reply routes did not quiesce; keeping terminal Slack fence', { issue: key });
14046
+ this.#increment('slackReplyRouteDrainsFailed');
14047
+ return false;
14048
+ }
14049
+ finally {
14050
+ if (!nested)
14051
+ this.#slackReplyRouteDrains.delete(key);
14052
+ }
14053
+ }
13209
14054
  async #stopSlackWatcher(issue) {
13210
14055
  const key = issueKey(issue);
14056
+ // Drain before clearing the fence. Clearing it first lets a reply that is
14057
+ // already mid-route — or one queued behind it — fall through the fence check
14058
+ // in #routeSlackConversationAnswerUnlocked and rebind the retired thread to
14059
+ // the next dispatch of this work unit.
14060
+ if (!await this.#drainSlackReplyRoutes(key))
14061
+ return false;
14062
+ this.#terminalSlackWatchIssues.delete(key);
14063
+ const expiryTimer = this.#slackTerminalWatchExpiryTimers.get(key);
14064
+ if (expiryTimer)
14065
+ clearTimeout(expiryTimer);
14066
+ this.#slackTerminalWatchExpiryTimers.delete(key);
14067
+ const receiptRetryTimer = this.#slackTerminalReceiptRetryTimers.get(key);
14068
+ if (receiptRetryTimer)
14069
+ clearTimeout(receiptRetryTimer);
14070
+ this.#slackTerminalReceiptRetryTimers.delete(key);
13211
14071
  const watcher = this.#slackWatchers.get(key);
13212
14072
  this.#slackWatchers.delete(key);
13213
14073
  const threadId = await this.#state.getSlackThread(this.#workspaceId, key);
@@ -13218,6 +14078,259 @@ export class FactoryLoop {
13218
14078
  await this.#state.clearConversationSession(this.#workspaceId, conversationId);
13219
14079
  }
13220
14080
  await this.#state.clearSlackThread(this.#workspaceId, key);
14081
+ await this.#state.clearSlackThreadWatch(this.#workspaceId, key);
14082
+ return true;
14083
+ }
14084
+ async #retireSlackWatcher(record) {
14085
+ const key = issueKey(record.issue);
14086
+ const threadId = await this.#state.getSlackThread(this.#workspaceId, key);
14087
+ if (!threadId) {
14088
+ await this.#stopSlackWatcher(record.issue);
14089
+ return;
14090
+ }
14091
+ const existingWatch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
14092
+ .find(([watchKey]) => watchKey === key)?.[1];
14093
+ const retiredAtMs = existingWatch?.kind === 'terminal-grace'
14094
+ ? terminalSlackWatchRetiredAtMs(existingWatch)
14095
+ : this.#clock.now();
14096
+ const expiresAtMs = existingWatch?.kind === 'terminal-grace'
14097
+ ? existingWatch.expiresAtMs
14098
+ : retiredAtMs + SLACK_TERMINAL_THREAD_GRACE_MS;
14099
+ await this.#state.setSlackThreadWatch(this.#workspaceId, key, {
14100
+ kind: 'terminal-grace',
14101
+ issue: { ...record.issue },
14102
+ decision: structuredClone(record.decision),
14103
+ threadId,
14104
+ retiredAtMs,
14105
+ expiresAtMs,
14106
+ });
14107
+ // A terminal thread must never retain a resumable session for an agent that
14108
+ // has already exited. Keep only the exact-thread listener so a late human
14109
+ // reply receives the explicit no-active-agent writeback below.
14110
+ this.#terminalSlackWatchIssues.add(key);
14111
+ await this.#slackReplyRoutes.get(key)?.catch(() => undefined);
14112
+ const conversationId = slackConversationId(threadId);
14113
+ await this.#slackConversationTurns.cancel(conversationId);
14114
+ try {
14115
+ await this.#surfaceUndeliveredSlackConversation(threadId);
14116
+ await this.#state.clearConversationSession(this.#workspaceId, conversationId);
14117
+ }
14118
+ catch (error) {
14119
+ // The receipt fails whenever another handler holds the claim or Slack
14120
+ // writeback is down — neither is a reason to abort retirement. Callers
14121
+ // reach here having already committed the terminal phase and dropped the
14122
+ // pending abandon reason, so a rejection escaping would strand the
14123
+ // registry rewrite, the GitHub watcher stop, and the queued next dispatch
14124
+ // with nothing left to re-run them. Keep the queued replies and let the
14125
+ // retry that owns this receipt settle it inside the grace window.
14126
+ this.#logger.warn?.('[factory] failed to settle undelivered Slack replies while retiring the watcher; will retry', { issue: record.issue.key, error });
14127
+ this.#increment('slackTerminalWatchReceiptsDeferred');
14128
+ this.#scheduleSlackTerminalReceiptRetry(record.issue, threadId, expiresAtMs);
14129
+ }
14130
+ if (!this.#slackWatchers.has(key) && !this.#stopping) {
14131
+ await this.#rearmSlackWatcher(record, threadId);
14132
+ }
14133
+ this.#scheduleSlackTerminalWatchExpiry(record.issue, expiresAtMs);
14134
+ this.#increment('slackTerminalWatchersRetained');
14135
+ }
14136
+ // The durable half of the same record. Every caller here follows the receipt
14137
+ // with a state write (clearing the session), and those two cannot be one
14138
+ // durable step: when the state write fails, the retry that owns it must not
14139
+ // read "replies still queued" as "the human has not been told" and post the
14140
+ // notice again. So the receipt is claimed before the provider write and marked
14141
+ // posted after it, and a retry finds it already settled.
14142
+ async #surfaceUndeliveredSlackConversation(threadId) {
14143
+ const conversationId = slackConversationId(threadId);
14144
+ const session = await this.#state.getConversationSession(this.#workspaceId, conversationId);
14145
+ const pendingCount = session
14146
+ ? session.pending.length + (session.delivery?.messages.length ?? 0)
14147
+ : 0;
14148
+ if (pendingCount === 0)
14149
+ return;
14150
+ if (session?.terminalReceipt?.posted) {
14151
+ this.#increment('slackTerminalReceiptsAlreadySettled');
14152
+ return;
14153
+ }
14154
+ if (!this.#slack)
14155
+ throw new Error(`Slack thread ${threadId} cannot surface undelivered replies without writeback`);
14156
+ const claimId = randomUUID();
14157
+ if (!await this.#state.claimConversationTerminalReceipt(this.#workspaceId, conversationId, claimId, this.#clock.now(), SLACK_TERMINAL_RECEIPT_CLAIM_LEASE_MS)) {
14158
+ const current = await this.#state.getConversationSession(this.#workspaceId, conversationId);
14159
+ if (current?.terminalReceipt?.posted) {
14160
+ this.#increment('slackTerminalReceiptsAlreadySettled');
14161
+ return;
14162
+ }
14163
+ // Another handler is mid-write. Fail closed so the queued replies survive
14164
+ // for whoever settles them rather than racing a second notice onto the
14165
+ // same thread.
14166
+ throw new Error(`Slack thread ${threadId} terminal receipt is claimed by another handler; retrying`);
14167
+ }
14168
+ const noun = pendingCount === 1 ? 'reply' : 'replies';
14169
+ const slack = this.#slack;
14170
+ try {
14171
+ await this.#withRenewedProviderLease('terminal Slack receipt', SLACK_TERMINAL_RECEIPT_CLAIM_LEASE_MS, () => this.#state.renewConversationTerminalReceipt(this.#workspaceId, conversationId, claimId, this.#clock.now()), () => slack.reply(threadId, `Factory could not deliver ${pendingCount} queued ${noun} because this work unit no longer has an active agent. Please continue on the linked issue or pull request.`));
14172
+ }
14173
+ catch (error) {
14174
+ await this.#state.releaseConversationTerminalReceipt(this.#workspaceId, conversationId, claimId);
14175
+ throw error;
14176
+ }
14177
+ if (!await this.#state.completeConversationTerminalReceipt(this.#workspaceId, conversationId, claimId)) {
14178
+ throw new Error(`Slack thread ${threadId} terminal receipt could not be recorded`);
14179
+ }
14180
+ this.#increment('slackConversationRepliesSurfacedTerminal');
14181
+ }
14182
+ // A claim only means something for as long as it outlives the work it covers.
14183
+ // A provider write can legitimately run past a fixed lease, at which point the
14184
+ // claim stops protecting the write it was taken for and a second handler can
14185
+ // post the same thing to the same human. Renewing on a heartbeat scopes the
14186
+ // lease to the work instead of to a guessed duration, and leaves the idle
14187
+ // timeout short enough that a holder that dies mid-write still frees it.
14188
+ //
14189
+ // The heartbeat is bounded on both ends, because a renewal loop that never
14190
+ // stops is a lock with no owner check: a provider write that hangs would hold
14191
+ // the receipt past every retry and past shutdown, and the human whose reply is
14192
+ // queued behind it would be told nothing until the process restarts. So it
14193
+ // stops at the ceiling and it stops when this daemon is stopping, and either
14194
+ // way it says so — from there the claim ages out on its own idle lease and
14195
+ // becomes reclaimable. The write may still land afterwards and duplicate the
14196
+ // notice; a reply nobody can ever reclaim is the worse of the two.
14197
+ async #withRenewedProviderLease(label, leaseMs, renew, run) {
14198
+ const renewUntilMs = this.#clock.now() + SLACK_PROVIDER_LEASE_MAX_RENEWAL_MS;
14199
+ let renewalStopped = false;
14200
+ let renewalInFlight = false;
14201
+ const stopRenewing = (counter, reason) => {
14202
+ renewalStopped = true;
14203
+ this.#increment(counter);
14204
+ this.#logger.warn?.(`[factory] ${label} lease will not be renewed further (${reason}); ` +
14205
+ 'its claim expires and the queued replies return to whoever retries them');
14206
+ };
14207
+ const heartbeat = setInterval(() => {
14208
+ if (renewalInFlight || renewalStopped)
14209
+ return;
14210
+ if (this.#stopping) {
14211
+ stopRenewing('slackProviderReceiptLeaseRenewalsStoppedForShutdown', 'shutting down');
14212
+ return;
14213
+ }
14214
+ if (this.#clock.now() >= renewUntilMs) {
14215
+ stopRenewing('slackProviderReceiptLeaseRenewalsExpired', `provider write exceeded ${SLACK_PROVIDER_LEASE_MAX_RENEWAL_MS}ms`);
14216
+ return;
14217
+ }
14218
+ renewalInFlight = true;
14219
+ void renew()
14220
+ .then((renewed) => {
14221
+ if (renewed)
14222
+ return;
14223
+ // Losing the lease mid-write is not recoverable from in here: the
14224
+ // write may already have landed. Stop renewing and let the caller's
14225
+ // completion check fail closed, which keeps the queued replies for
14226
+ // whoever holds the claim now.
14227
+ renewalStopped = true;
14228
+ this.#increment('slackProviderReceiptLeasesLost');
14229
+ this.#logger.warn?.(`[factory] ${label} lease was lost while its provider write was in flight`);
14230
+ })
14231
+ .catch((error) => this.#logger.warn?.(`[factory] ${label} lease renewal failed`, {
14232
+ error: describeError(error).errorMessage,
14233
+ }))
14234
+ .finally(() => { renewalInFlight = false; });
14235
+ }, Math.max(1_000, Math.floor(leaseMs / 3)));
14236
+ heartbeat.unref?.();
14237
+ try {
14238
+ return await run();
14239
+ }
14240
+ finally {
14241
+ clearInterval(heartbeat);
14242
+ }
14243
+ }
14244
+ // A terminal receipt that could not be written leaves the queued replies
14245
+ // pending with the human who wrote them told nothing. That is retryable
14246
+ // maintenance this daemon owns, not work to leave for the next restart: the
14247
+ // grace watch is the only window in which the receipt can still land on the
14248
+ // retired thread, so keep reattempting inside it and give up when it closes.
14249
+ #scheduleSlackTerminalReceiptRetry(issue, threadId, expiresAtMs, attempt = 0) {
14250
+ if (this.#stopping)
14251
+ return;
14252
+ const key = issueKey(issue);
14253
+ const existing = this.#slackTerminalReceiptRetryTimers.get(key);
14254
+ if (existing)
14255
+ clearTimeout(existing);
14256
+ this.#slackTerminalReceiptRetryTimers.delete(key);
14257
+ const remainingMs = expiresAtMs - this.#clock.now();
14258
+ if (remainingMs <= 0) {
14259
+ this.#increment('slackTerminalWatchReceiptsAbandoned');
14260
+ return;
14261
+ }
14262
+ const backoffMs = Math.min(SLACK_TERMINAL_RECEIPT_RETRY_MAX_MS, SLACK_TERMINAL_RECEIPT_RETRY_MS * 2 ** Math.min(attempt, 16));
14263
+ const timer = setTimeout(() => {
14264
+ this.#slackTerminalReceiptRetryTimers.delete(key);
14265
+ void this.#retrySlackTerminalReceipt(issue, threadId, attempt);
14266
+ }, Math.max(0, Math.min(backoffMs, remainingMs)));
14267
+ timer.unref?.();
14268
+ this.#slackTerminalReceiptRetryTimers.set(key, timer);
14269
+ }
14270
+ async #retrySlackTerminalReceipt(issue, threadId, attempt) {
14271
+ if (this.#stopping)
14272
+ return;
14273
+ const key = issueKey(issue);
14274
+ const watch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
14275
+ .find(([watchKey]) => watchKey === key)?.[1];
14276
+ // The grace watch is gone (expired, or the work unit reopened): the thread
14277
+ // this receipt would settle no longer exists, so there is nothing to say.
14278
+ if (watch?.kind !== 'terminal-grace' || watch.threadId !== threadId)
14279
+ return;
14280
+ try {
14281
+ await this.#surfaceUndeliveredSlackConversation(threadId);
14282
+ await this.#state.clearConversationSession(this.#workspaceId, slackConversationId(threadId));
14283
+ this.#increment('slackTerminalWatchReceiptsRecovered');
14284
+ }
14285
+ catch (error) {
14286
+ this.#logger.warn?.('[factory] terminal Slack receipt retry failed; rescheduling', {
14287
+ issue: issue.key,
14288
+ error,
14289
+ });
14290
+ this.#increment('slackTerminalWatchReceiptRetryFailures');
14291
+ this.#scheduleSlackTerminalReceiptRetry(issue, threadId, watch.expiresAtMs, attempt + 1);
14292
+ }
14293
+ }
14294
+ #scheduleSlackTerminalWatchExpiry(issue, expiresAtMs, retryDelayMs) {
14295
+ if (this.#stopping)
14296
+ return;
14297
+ const key = issueKey(issue);
14298
+ const existing = this.#slackTerminalWatchExpiryTimers.get(key);
14299
+ if (existing)
14300
+ clearTimeout(existing);
14301
+ const timer = setTimeout(() => {
14302
+ this.#slackTerminalWatchExpiryTimers.delete(key);
14303
+ void this.#expireSlackTerminalWatcher(issue, expiresAtMs).catch((error) => {
14304
+ this.#logger.warn?.('[factory] failed to expire terminal Slack reply watcher; retrying', {
14305
+ issue: issue.key,
14306
+ error,
14307
+ });
14308
+ this.#scheduleSlackTerminalWatchExpiry(issue, expiresAtMs, SLACK_REPLY_ROUTE_RETRY_MS);
14309
+ });
14310
+ }, retryDelayMs ?? Math.max(0, expiresAtMs - this.#clock.now()));
14311
+ timer.unref?.();
14312
+ this.#slackTerminalWatchExpiryTimers.set(key, timer);
14313
+ }
14314
+ async #expireSlackTerminalWatcher(issue, expiresAtMs) {
14315
+ const key = issueKey(issue);
14316
+ const watch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
14317
+ .find(([watchKey]) => watchKey === key)?.[1];
14318
+ if (watch?.kind !== 'terminal-grace' || watch.expiresAtMs !== expiresAtMs)
14319
+ return;
14320
+ if (watch.expiresAtMs > this.#clock.now()) {
14321
+ this.#scheduleSlackTerminalWatchExpiry(issue, watch.expiresAtMs);
14322
+ return;
14323
+ }
14324
+ // #stopSlackWatcher fails closed when an in-flight reply route will not
14325
+ // drain, leaving the watch and its terminal fence in place. Counting that as
14326
+ // an expiration retires the watch in the metrics while the real one lives
14327
+ // on unwatched by any expiry timer, so reschedule and count only on success.
14328
+ if (!await this.#stopSlackWatcher(issue)) {
14329
+ this.#increment('slackTerminalWatchExpiriesDeferred');
14330
+ this.#scheduleSlackTerminalWatchExpiry(issue, expiresAtMs, SLACK_REPLY_ROUTE_RETRY_MS);
14331
+ return;
14332
+ }
14333
+ this.#increment('slackTerminalWatchersExpired');
13221
14334
  }
13222
14335
  async #readSlackReply(path) {
13223
14336
  try {
@@ -13282,33 +14395,147 @@ export class FactoryLoop {
13282
14395
  await this.#wakeWaitingClarification(clarificationKey, claimed);
13283
14396
  return;
13284
14397
  }
14398
+ return await this.#routeSlackConversationAnswer(record, reply, text, clarificationKey);
14399
+ }
14400
+ async #routeSlackConversationAnswer(record, reply, text, clarificationKey) {
14401
+ if (this.#slackReplyRouteDrains.has(clarificationKey)) {
14402
+ // The terminal fence for this work unit is being drained right now.
14403
+ // Registering an ordinary route here would put it past the drain's
14404
+ // snapshot and run it once the fence is gone. Answer it the way the fence
14405
+ // would have — but as a tracked effect, because this writeback is still a
14406
+ // side effect of this work unit. Left untracked it is the same escape one
14407
+ // level further in: the drain reports quiescence without it, the watcher
14408
+ // stop clears the retry timer that owns this reply, and a slow or failed
14409
+ // receipt leaves the human told nothing at all.
14410
+ this.#increment('slackReplyRoutesFencedDuringDrain');
14411
+ return await this.#trackSlackWorkUnitEffect(clarificationKey, async () => {
14412
+ await this.#writeUnroutableSlackReply(reply.threadTs);
14413
+ return undefined;
14414
+ });
14415
+ }
14416
+ return await this.#trackSlackWorkUnitEffect(clarificationKey, () => this.#routeSlackConversationAnswerUnlocked(record, reply, text, clarificationKey));
14417
+ }
14418
+ // Every Slack side effect a work unit makes on its own behalf runs through
14419
+ // here, so #slackReplyRoutes stays the single record the terminal drain
14420
+ // consults. Effects are chained per work unit: awaiting the newest one drains
14421
+ // everything queued behind it, and a rejection propagates to the drain, which
14422
+ // fails closed rather than tearing the effect's retry path down.
14423
+ async #trackSlackWorkUnitEffect(key, run) {
14424
+ const preceding = this.#slackReplyRoutes.get(key);
14425
+ const effect = (async () => {
14426
+ await preceding?.catch(() => undefined);
14427
+ return await run();
14428
+ })();
14429
+ this.#slackReplyRoutes.set(key, effect);
14430
+ try {
14431
+ return await effect;
14432
+ }
14433
+ finally {
14434
+ if (this.#slackReplyRoutes.get(key) === effect)
14435
+ this.#slackReplyRoutes.delete(key);
14436
+ }
14437
+ }
14438
+ async #routeSlackConversationAnswerUnlocked(record, reply, text, clarificationKey) {
14439
+ if (this.#terminalSlackWatchIssues.has(clarificationKey)) {
14440
+ await this.#writeUnroutableSlackReply(reply.threadTs);
14441
+ return;
14442
+ }
13285
14443
  const conversationId = slackConversationId(reply.threadTs);
13286
- const conversation = await this.#state.getConversationSession(this.#workspaceId, conversationId);
14444
+ let conversation = await this.#state.getConversationSession(this.#workspaceId, conversationId);
14445
+ let liveRecord;
14446
+ if (!conversation) {
14447
+ liveRecord = (await this.#batch()).getIssue(record.issue);
14448
+ if (liveRecord && !liveRecord.dryRun) {
14449
+ await this.#ensureSlackConversationSession(liveRecord, reply.threadTs);
14450
+ conversation = await this.#state.getConversationSession(this.#workspaceId, conversationId);
14451
+ }
14452
+ }
13287
14453
  if (conversation && issueKey(conversation.issue) === clarificationKey) {
14454
+ const replyId = `${reply.threadTs}:${reply.messageTs}`;
13288
14455
  const queued = await this.#state.appendConversationMessage(this.#workspaceId, conversationId, {
13289
- id: `${reply.threadTs}:${reply.messageTs}`,
14456
+ id: replyId,
13290
14457
  text,
13291
14458
  receivedAtMs: slackMessageReceivedAtMs(reply.messageTs, this.#clock.now()),
13292
14459
  providerSequence: reply.messageTs,
13293
14460
  author: reply.author,
13294
14461
  });
13295
- if (!queued) {
14462
+ const durable = queued ?? await this.#state.getConversationSession(this.#workspaceId, conversationId);
14463
+ if (!durable || !durable.processedMessageIds.includes(replyId)) {
14464
+ throw new Error(`Slack reply ${replyId} was not durably queued`);
14465
+ }
14466
+ if (!(durable.acknowledgedMessageIds ?? []).includes(replyId)) {
14467
+ const acknowledgementClaimId = randomUUID();
14468
+ const acknowledgementClaimed = await this.#state.claimConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId, this.#clock.now(), SLACK_REPLY_ACKNOWLEDGEMENT_LEASE_MS);
14469
+ if (acknowledgementClaimed) {
14470
+ try {
14471
+ if (!this.#slack)
14472
+ throw new Error(`Slack reply ${replyId} cannot be acknowledged without writeback`);
14473
+ const owner = durable.agent?.role === 'babysitter'
14474
+ ? 'the PR babysitter'
14475
+ : durable.agent
14476
+ ? 'the issue implementer'
14477
+ : 'an issue agent';
14478
+ const receipt = durable.agent
14479
+ ? `Factory received this reply and durably queued it for ${owner}.`
14480
+ : 'Factory received and durably stored this reply; it will route when an issue agent is resumable.';
14481
+ const slack = this.#slack;
14482
+ // Same lease scope as the terminal receipt: this claim covers a
14483
+ // provider write that can outrun any fixed duration, so it is
14484
+ // renewed for as long as that write is actually running.
14485
+ await this.#withRenewedProviderLease('Slack reply acknowledgement', SLACK_REPLY_ACKNOWLEDGEMENT_LEASE_MS, () => this.#state.renewConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId, this.#clock.now()), () => slack.reply(reply.threadTs, receipt));
14486
+ if (!await this.#state.completeConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId)) {
14487
+ throw new Error(`Slack reply ${replyId} receipt could not be recorded`);
14488
+ }
14489
+ this.#increment('slackConversationRepliesAcknowledged');
14490
+ }
14491
+ catch (error) {
14492
+ await this.#state.releaseConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId);
14493
+ throw error;
14494
+ }
14495
+ }
14496
+ else {
14497
+ const acknowledgementState = await this.#state.getConversationSession(this.#workspaceId, conversationId);
14498
+ if (!(acknowledgementState?.acknowledgedMessageIds ?? []).includes(replyId)) {
14499
+ throw new Error(`Slack reply ${replyId} receipt is claimed by another handler; retrying`);
14500
+ }
14501
+ }
14502
+ }
14503
+ if (queued) {
14504
+ this.#increment('slackConversationRepliesQueued');
14505
+ }
14506
+ else {
13296
14507
  this.#increment('slackConversationDuplicateRepliesSuppressed');
13297
- return;
13298
14508
  }
13299
- this.#increment('slackConversationRepliesQueued');
13300
- this.#slackConversationTurns.schedule(conversationId);
14509
+ const pending = durable.pending.some((message) => message.id === replyId) ||
14510
+ Boolean(durable.delivery?.messages.some((message) => message.id === replyId));
14511
+ if (pending && durable.agent) {
14512
+ this.#slackConversationTurns.schedule(conversationId);
14513
+ }
14514
+ else if (pending) {
14515
+ this.#increment('slackConversationRepliesWaitingForOwner');
14516
+ }
13301
14517
  return;
13302
14518
  }
13303
- const liveRecord = (await this.#batch()).getIssue(record.issue);
14519
+ liveRecord ??= (await this.#batch()).getIssue(record.issue);
13304
14520
  if (!liveRecord || liveRecord.dryRun) {
13305
14521
  if (isTriageEscalationWatchRecord(record)) {
13306
14522
  return await this.#handleTriageEscalationSlackAnswer(record, text);
13307
14523
  }
13308
- this.#increment('slackAnswersIgnoredNoInFlight');
14524
+ await this.#writeUnroutableSlackReply(reply.threadTs);
13309
14525
  return;
13310
14526
  }
13311
14527
  this.#increment('slackAnswersIgnoredNoConversationSession');
14528
+ if (this.#slack) {
14529
+ await this.#slack.reply(reply.threadTs, 'Factory received this reply but could not create a durable agent route. It will remain replayable; please also continue on the linked issue or pull request.');
14530
+ this.#increment('slackAnswersUnroutableVisible');
14531
+ }
14532
+ }
14533
+ async #writeUnroutableSlackReply(threadId) {
14534
+ this.#increment('slackAnswersIgnoredNoInFlight');
14535
+ if (!this.#slack)
14536
+ return;
14537
+ await this.#slack.reply(threadId, 'Factory received this reply but could not route it because this work unit no longer has an active agent. Please continue on the linked issue or pull request.');
14538
+ this.#increment('slackAnswersUnroutableVisible');
13312
14539
  }
13313
14540
  async #wakeWaitingClarification(key, waiting) {
13314
14541
  const existing = this.#clarificationWakeInFlight.get(key);
@@ -13680,6 +14907,7 @@ export class FactoryLoop {
13680
14907
  const batch = await this.#batch();
13681
14908
  if (batch.isInFlight(record.issue) || batch.isQueued(record.issue)) {
13682
14909
  this.#increment('slackTriageAnswersIgnoredAlreadyActive');
14910
+ await this.#state.clearSlackThreadWatch(this.#workspaceId, issueKey(record.issue));
13683
14911
  return;
13684
14912
  }
13685
14913
  if (await this.#dispatchBlockReason(record.issue)) {
@@ -13696,6 +14924,10 @@ export class FactoryLoop {
13696
14924
  if (hasDispatchableRoute(decision)) {
13697
14925
  this.#pendingSlackClarifications.set(issueKey(decision.issue), text);
13698
14926
  const result = await this.#startOrQueueSlackClarifiedDecision(dispatchAfterSlackClarification(decision, escalationReason));
14927
+ const active = await this.#batch();
14928
+ if (result || active.isInFlight(decision.issue) || active.isQueued(decision.issue)) {
14929
+ await this.#state.clearSlackThreadWatch(this.#workspaceId, issueKey(record.issue));
14930
+ }
13699
14931
  this.#increment('slackTriageAnswersDispatchedWithRemainingEscalation');
13700
14932
  return result;
13701
14933
  }
@@ -13708,6 +14940,10 @@ export class FactoryLoop {
13708
14940
  }
13709
14941
  this.#pendingSlackClarifications.set(issueKey(decision.issue), text);
13710
14942
  const result = await this.#startOrQueueSlackClarifiedDecision(decision);
14943
+ const active = await this.#batch();
14944
+ if (result || active.isInFlight(decision.issue) || active.isQueued(decision.issue)) {
14945
+ await this.#state.clearSlackThreadWatch(this.#workspaceId, issueKey(record.issue));
14946
+ }
13711
14947
  this.#increment('slackTriageAnswersDispatched');
13712
14948
  return result;
13713
14949
  }
@@ -14237,6 +15473,15 @@ const githubIssueAuthor = (issue) => {
14237
15473
  }
14238
15474
  return source ? undefined : githubAuthorLogin(payload)?.trim() || undefined;
14239
15475
  };
15476
+ /**
15477
+ * An `IssueRef` for a path whose issue body could not be read at all — the
15478
+ * shape a relayfile-shed ready-issue read leaves behind (#297). The key is
15479
+ * what an operator needs to correlate the skip; the uuid falls back to it.
15480
+ */
15481
+ const issueRefFromPath = (path) => {
15482
+ const key = keyFromPath(path);
15483
+ return { uuid: uuidFromPath(path) ?? key, key, path };
15484
+ };
14240
15485
  const issueRef = (issue) => ({ uuid: issue.uuid, key: issue.key, path: issue.path });
14241
15486
  // Preserve the historical Linear state namespace while keeping GitHub-native
14242
15487
  // issue numbers independent across repositories in the same workspace.
@@ -16150,10 +17395,54 @@ const relayfileOverload = (error) => {
16150
17395
  stringValue(flat.code) ?? stringValue(data.code) ?? 'rate_limited';
16151
17396
  return { status, reason, ...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }) };
16152
17397
  };
17398
+ /**
17399
+ * relayfile's overload reason codes, allowlisted.
17400
+ *
17401
+ * `IterationReport.skipped[].reason` is serialized to stdout by `factory
17402
+ * run-once`, so it stays a fixed classification plus a known code — the same
17403
+ * public-surface rule #293 applied to error class names — rather than
17404
+ * whatever string the dependency happened to send.
17405
+ */
17406
+ const RELAYFILE_OVERLOAD_REASONS = new Set([
17407
+ // Admission gate inside the workspace durable object.
17408
+ 'inflight_limit',
17409
+ 'oldest_inflight_age',
17410
+ 'write_admission_limit',
17411
+ // Worker-side backpressure, per isolate rather than per workspace.
17412
+ 'router_inflight_limit',
17413
+ // The Cloudflare runtime shed the object; relayfile only relabels it.
17414
+ 'durable_object_overloaded',
17415
+ // relayfileOverload()'s fallback when the body carried no reason at all.
17416
+ 'rate_limited',
17417
+ ]);
17418
+ /**
17419
+ * Both the run-report reason and the per-reason counter key are built from
17420
+ * this, so an unknown code from the dependency can neither leak into stdout
17421
+ * nor open an unbounded counter namespace.
17422
+ */
17423
+ const relayfileOverloadReasonLabel = (reason) => RELAYFILE_OVERLOAD_REASONS.has(reason) ? reason : 'unrecognized';
17424
+ /**
17425
+ * How long to wait before the next discovery sweep after relayfile shed this
17426
+ * one.
17427
+ *
17428
+ * The advertised `Retry-After` is authoritative in BOTH directions (#297):
17429
+ * it is the first rung, so we never retry sooner than the dependency allows,
17430
+ * and it bounds the ceiling, so we never sleep for minutes because of a
17431
+ * request to wait seconds. Without an advertised delay there is nothing to
17432
+ * respect and the original five-minute ladder governs unchanged.
17433
+ */
16153
17434
  const discoveryOverloadBackoffMs = (retryAfterSeconds, consecutiveOverloads) => {
16154
- const retryAfterMs = Math.max(0, Math.ceil((retryAfterSeconds ?? 0) * 1_000));
16155
- const exponentialMs = Math.min(DISCOVERY_OVERLOAD_BACKOFF_MAX_MS, 5_000 * (2 ** Math.min(10, Math.max(0, consecutiveOverloads - 1))));
16156
- return Math.max(retryAfterMs, exponentialMs);
17435
+ const advertisedMs = retryAfterSeconds === undefined
17436
+ ? undefined
17437
+ : Math.max(DISCOVERY_OVERLOAD_BACKOFF_MIN_MS, Math.ceil(retryAfterSeconds * 1_000));
17438
+ const baseMs = advertisedMs ?? DISCOVERY_OVERLOAD_BACKOFF_BASE_MS;
17439
+ // A dependency that asks for longer than the advertised ceiling still gets
17440
+ // what it asked for; the ceiling only stops the ladder from overshooting it.
17441
+ const ceilingMs = advertisedMs === undefined
17442
+ ? DISCOVERY_OVERLOAD_BACKOFF_MAX_MS
17443
+ : Math.max(advertisedMs, DISCOVERY_OVERLOAD_ADVERTISED_BACKOFF_MAX_MS);
17444
+ const steps = Math.min(10, Math.max(0, consecutiveOverloads - 1));
17445
+ return Math.min(ceilingMs, baseMs * (2 ** steps));
16157
17446
  };
16158
17447
  const eventSequenceNumber = (eventId) => {
16159
17448
  const whole = Number(eventId);
@@ -16218,8 +17507,8 @@ const normalizeGithubRepo = (repo, defaultOwner) => {
16218
17507
  }
16219
17508
  return `${owner}/${repo}`;
16220
17509
  };
16221
- const githubPullRequestBody = (issue, preview) => [
16222
- issue.description,
17510
+ const githubPullRequestBody = (issue, preview, sessionRef) => [
17511
+ stripTrajectoryPointers(issue.description),
16223
17512
  '',
16224
17513
  isGithubIssue(issue) && /^\d+$/u.test(issue.key)
16225
17514
  ? `Fixes #${issue.key}`
@@ -16229,7 +17518,25 @@ const githubPullRequestBody = (issue, preview) => [
16229
17518
  `Live preview: ${preview.url}`,
16230
17519
  'Access: Tailscale tailnet membership and the tailnet grants/ACLs are required; this URL is not public.',
16231
17520
  ] : []),
17521
+ '',
17522
+ renderTrajectoryPointer({
17523
+ ...trajectoryWorkUnitForIssue(issue),
17524
+ sessionRef,
17525
+ }),
16232
17526
  ].join('\n').trim();
17527
+ const trajectoryWorkUnitForIssue = (issue) => {
17528
+ const github = githubIssueSourceRef(issue);
17529
+ if (github) {
17530
+ return {
17531
+ workUnitId: `${github.owner}/${github.repo}#${github.number}`,
17532
+ workUnitSurface: 'github',
17533
+ };
17534
+ }
17535
+ if (isRealLinearIssue(issue)) {
17536
+ return { workUnitId: issue.key, workUnitSurface: 'linear' };
17537
+ }
17538
+ return { workUnitId: `factory:${issue.uuid}`, workUnitSurface: 'factory' };
17539
+ };
16233
17540
  // The broker rejects re-registering a name it never released on exit
16234
17541
  // (relay#1116-family) with a 500 "agent '<name>' already exists". Detect it from
16235
17542
  // the structured payload or the message so resume can treat it as terminal
@@ -16258,6 +17565,9 @@ const slackMessageReceivedAtMs = (messageTs, fallback) => {
16258
17565
  const seconds = Number(messageTs);
16259
17566
  return Number.isFinite(seconds) && seconds > 0 ? Math.floor(seconds * 1_000) : fallback;
16260
17567
  };
17568
+ const terminalSlackWatchRetiredAtMs = (watch) => typeof watch.retiredAtMs === 'number' && Number.isFinite(watch.retiredAtMs)
17569
+ ? watch.retiredAtMs
17570
+ : Math.max(0, watch.expiresAtMs - SLACK_TERMINAL_THREAD_GRACE_MS);
16261
17571
  const eventIdentity = (event) => {
16262
17572
  const record = event;
16263
17573
  const rawId = record.id ?? record.event_id ?? record.seq;
@@ -16361,10 +17671,6 @@ const telemetryCategory = (value) => {
16361
17671
  const normalized = value.trim().toLowerCase().replace(/[^a-z0-9._:/-]+/gu, '-');
16362
17672
  return normalized.slice(0, 120) || undefined;
16363
17673
  };
16364
- const telemetryErrorClass = (error) => {
16365
- const name = error instanceof Error ? error.name : '';
16366
- return /^[A-Za-z][A-Za-z0-9]{0,63}(?:Error|Exception)$/u.test(name) ? name : 'Error';
16367
- };
16368
17674
  const isTimeoutError = (error) => error instanceof Error && (error.name === 'TimeoutError' || error.name === 'AbortError');
16369
17675
  const retryOnTimeout = async (fn, opts) => {
16370
17676
  let lastError;
@@ -16569,6 +17875,77 @@ export class LiveDispatchStateChangedError extends Error {
16569
17875
  export function isLiveDispatchStateChangedError(error) {
16570
17876
  return error instanceof LiveDispatchStateChangedError;
16571
17877
  }
17878
+ /** How deep to follow `cause` when classifying a wrapped failure. */
17879
+ const PASS_FATAL_CAUSE_DEPTH = 4;
17880
+ /**
17881
+ * Whether `error`, or anything it wraps, is an instance of `type`.
17882
+ * `contextualError` and the fleet control-plane guard both rethrow wrapped, so
17883
+ * classification has to follow the cause chain rather than trust the outermost
17884
+ * type.
17885
+ */
17886
+ const wrapsErrorOfType = (error, type, depth = 0) => {
17887
+ if (depth > PASS_FATAL_CAUSE_DEPTH || !(error instanceof Error))
17888
+ return false;
17889
+ if (error instanceof type)
17890
+ return true;
17891
+ return wrapsErrorOfType(error.cause, type, depth + 1);
17892
+ };
17893
+ /**
17894
+ * How many *unclassified* per-item failures without an intervening successful
17895
+ * dispatch end the pass. Named per-item conditions (a lifecycle claim refusal,
17896
+ * a live-state race) never count toward it and never reset it: those
17897
+ * legitimately affect many units at once and are exactly the benign case #292
17898
+ * asks the loop to survive, so they are neither evidence of a pass-wide fault
17899
+ * nor evidence against one.
17900
+ */
17901
+ const UNCLASSIFIED_DISPATCH_FAILURE_LIMIT = 5;
17902
+ /**
17903
+ * Failures the loop recognizes as belonging to one work unit. They are always
17904
+ * skippable and are exempt from the consecutive-failure fuse.
17905
+ */
17906
+ const isClassifiedPerItemDispatchFailure = (error) => error instanceof LiveDispatchStateChangedError ||
17907
+ error instanceof DispatchLifecycleClaimRefusedError ||
17908
+ // Relayfile shedding one operation is a state of the dependency, not an
17909
+ // unexplained fault, and it has its own fuse — see #297 and
17910
+ // DISCOVERY_OVERLOAD_PER_SWEEP_LIMIT.
17911
+ relayfileOverload(error) !== undefined;
17912
+ /**
17913
+ * Whether a per-item failure could have left half-spawned agents behind.
17914
+ *
17915
+ * The two lifecycle refusals are decided before `#dispatchUnlocked` spawns
17916
+ * anything, so there is nothing to reap for them. Everything else can fail
17917
+ * *after* a spawn — including relayfile shedding a post-spawn read — having
17918
+ * persisted the agents as failure handoffs on the way out.
17919
+ *
17920
+ * Deliberately a denylist rather than an allowlist: a new failure mode that
17921
+ * nobody classified should default to "reap it", because the cost of a
17922
+ * needless reap is one no-op pass over an empty handoff list, while the cost
17923
+ * of a missed one is leaked agents and duplicate workers on the next retry.
17924
+ */
17925
+ const mayHaveSpawnedBeforeFailing = (error) => !(error instanceof LiveDispatchStateChangedError) &&
17926
+ !(error instanceof DispatchLifecycleClaimRefusedError);
17927
+ /**
17928
+ * The run-report reason recorded for a work unit the pass could not dispatch.
17929
+ *
17930
+ * `factory run-once` serializes the whole report to stdout, so this string is
17931
+ * a public surface: it stays a fixed classification plus an allowlisted error
17932
+ * class name, never raw provider text or filesystem paths. The full message
17933
+ * goes to the operator log instead, the same split
17934
+ * `describeControlPlaneError` makes for circuit state.
17935
+ */
17936
+ const perItemDispatchSkipReason = (error) => {
17937
+ const overload = relayfileOverload(error);
17938
+ if (overload)
17939
+ return `relayfile overloaded (${relayfileOverloadReasonLabel(overload.reason)})`;
17940
+ if (error instanceof LiveDispatchStateChangedError)
17941
+ return 'live state changed during dispatch';
17942
+ if (error instanceof DispatchLifecycleClaimRefusedError) {
17943
+ return error.refusal === 'terminal'
17944
+ ? 'dispatch lifecycle already terminal'
17945
+ : 'dispatch lifecycle owned by another publisher';
17946
+ }
17947
+ return `dispatch failed (${telemetryErrorClass(error)})`;
17948
+ };
16572
17949
  const triageEscalationQuestion = (decision, issue) => {
16573
17950
  const routedRepos = decision.routes.map((route) => route.repo).filter(Boolean);
16574
17951
  const subject = issue?.title?.trim() || decision.issue.key;