@agent-relay/factory 0.1.62 → 0.1.64

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. package/README.md +117 -15
  2. package/dist/cli/exit-codes.d.ts +81 -0
  3. package/dist/cli/exit-codes.d.ts.map +1 -0
  4. package/dist/cli/exit-codes.js +122 -0
  5. package/dist/cli/exit-codes.js.map +1 -0
  6. package/dist/cli/fleet.d.ts +26 -2
  7. package/dist/cli/fleet.d.ts.map +1 -1
  8. package/dist/cli/fleet.js +183 -40
  9. package/dist/cli/fleet.js.map +1 -1
  10. package/dist/cli/index.d.ts +3 -0
  11. package/dist/cli/index.d.ts.map +1 -0
  12. package/dist/cli/index.js +2 -0
  13. package/dist/cli/index.js.map +1 -0
  14. package/dist/config/schema.d.ts +156 -52
  15. package/dist/config/schema.d.ts.map +1 -1
  16. package/dist/config/schema.js +17 -5
  17. package/dist/config/schema.js.map +1 -1
  18. package/dist/dispatch/templates.d.ts +7 -0
  19. package/dist/dispatch/templates.d.ts.map +1 -1
  20. package/dist/dispatch/templates.js +64 -12
  21. package/dist/dispatch/templates.js.map +1 -1
  22. package/dist/dispatch/work-unit-identity.d.ts +19 -0
  23. package/dist/dispatch/work-unit-identity.d.ts.map +1 -0
  24. package/dist/dispatch/work-unit-identity.js +40 -0
  25. package/dist/dispatch/work-unit-identity.js.map +1 -0
  26. package/dist/environments/stack-descriptor.d.ts +3 -3
  27. package/dist/fleet/control-plane-circuit.d.ts +57 -0
  28. package/dist/fleet/control-plane-circuit.d.ts.map +1 -0
  29. package/dist/fleet/control-plane-circuit.js +231 -0
  30. package/dist/fleet/control-plane-circuit.js.map +1 -0
  31. package/dist/fleet/internal-fleet-client.d.ts +2 -1
  32. package/dist/fleet/internal-fleet-client.d.ts.map +1 -1
  33. package/dist/fleet/internal-fleet-client.js +11 -7
  34. package/dist/fleet/internal-fleet-client.js.map +1 -1
  35. package/dist/fleet/relay-fleet-client.d.ts +3 -0
  36. package/dist/fleet/relay-fleet-client.d.ts.map +1 -1
  37. package/dist/fleet/relay-fleet-client.js +97 -18
  38. package/dist/fleet/relay-fleet-client.js.map +1 -1
  39. package/dist/github/writeback-paths.d.ts +7 -0
  40. package/dist/github/writeback-paths.d.ts.map +1 -0
  41. package/dist/github/writeback-paths.js +28 -0
  42. package/dist/github/writeback-paths.js.map +1 -0
  43. package/dist/hosted/orchestrator.d.ts.map +1 -1
  44. package/dist/hosted/orchestrator.js +2 -0
  45. package/dist/hosted/orchestrator.js.map +1 -1
  46. package/dist/index.d.ts +10 -4
  47. package/dist/index.d.ts.map +1 -1
  48. package/dist/index.js +6 -3
  49. package/dist/index.js.map +1 -1
  50. package/dist/intake/index.d.ts +2 -1
  51. package/dist/intake/index.d.ts.map +1 -1
  52. package/dist/intake/index.js +2 -1
  53. package/dist/intake/index.js.map +1 -1
  54. package/dist/intake/notion-manifest.d.ts +48 -0
  55. package/dist/intake/notion-manifest.d.ts.map +1 -0
  56. package/dist/intake/notion-manifest.js +258 -0
  57. package/dist/intake/notion-manifest.js.map +1 -0
  58. package/dist/intake/notion-relay-claim.d.ts +5 -4
  59. package/dist/intake/notion-relay-claim.d.ts.map +1 -1
  60. package/dist/intake/notion-relay-claim.js +48 -7
  61. package/dist/intake/notion-relay-claim.js.map +1 -1
  62. package/dist/intake/notion.d.ts +6 -3
  63. package/dist/intake/notion.d.ts.map +1 -1
  64. package/dist/intake/notion.js +63 -13
  65. package/dist/intake/notion.js.map +1 -1
  66. package/dist/issue-key-match.d.ts +6 -0
  67. package/dist/issue-key-match.d.ts.map +1 -1
  68. package/dist/issue-key-match.js +6 -1
  69. package/dist/issue-key-match.js.map +1 -1
  70. package/dist/mount/local-mount-preflight.d.ts.map +1 -1
  71. package/dist/mount/local-mount-preflight.js +6 -2
  72. package/dist/mount/local-mount-preflight.js.map +1 -1
  73. package/dist/mount/mount-auth-error.d.ts +16 -3
  74. package/dist/mount/mount-auth-error.d.ts.map +1 -1
  75. package/dist/mount/mount-auth-error.js +34 -7
  76. package/dist/mount/mount-auth-error.js.map +1 -1
  77. package/dist/mount/relayfile-cloud-mount-client.d.ts +36 -1
  78. package/dist/mount/relayfile-cloud-mount-client.d.ts.map +1 -1
  79. package/dist/mount/relayfile-cloud-mount-client.js +136 -6
  80. package/dist/mount/relayfile-cloud-mount-client.js.map +1 -1
  81. package/dist/mount/relayfile-github-connection-write.d.ts +24 -2
  82. package/dist/mount/relayfile-github-connection-write.d.ts.map +1 -1
  83. package/dist/mount/relayfile-github-connection-write.js +90 -1
  84. package/dist/mount/relayfile-github-connection-write.js.map +1 -1
  85. package/dist/node/factory-node.d.ts.map +1 -1
  86. package/dist/node/factory-node.js +18 -4
  87. package/dist/node/factory-node.js.map +1 -1
  88. package/dist/observability/cloud-reporter.d.ts +1 -0
  89. package/dist/observability/cloud-reporter.d.ts.map +1 -1
  90. package/dist/observability/cloud-reporter.js +182 -50
  91. package/dist/observability/cloud-reporter.js.map +1 -1
  92. package/dist/observability/events.d.ts +52 -52
  93. package/dist/orchestrator/batch-tracker.d.ts +25 -0
  94. package/dist/orchestrator/batch-tracker.d.ts.map +1 -1
  95. package/dist/orchestrator/batch-tracker.js +48 -2
  96. package/dist/orchestrator/batch-tracker.js.map +1 -1
  97. package/dist/orchestrator/factory.d.ts +36 -2
  98. package/dist/orchestrator/factory.d.ts.map +1 -1
  99. package/dist/orchestrator/factory.js +1208 -82
  100. package/dist/orchestrator/factory.js.map +1 -1
  101. package/dist/orchestrator/index.d.ts +1 -1
  102. package/dist/orchestrator/index.d.ts.map +1 -1
  103. package/dist/orchestrator/index.js +1 -1
  104. package/dist/orchestrator/index.js.map +1 -1
  105. package/dist/ports/fleet.d.ts +6 -0
  106. package/dist/ports/fleet.d.ts.map +1 -1
  107. package/dist/ports/index.d.ts +1 -1
  108. package/dist/ports/index.d.ts.map +1 -1
  109. package/dist/ports/mount.d.ts +39 -0
  110. package/dist/ports/mount.d.ts.map +1 -1
  111. package/dist/ports/state.d.ts +75 -4
  112. package/dist/ports/state.d.ts.map +1 -1
  113. package/dist/state/document-store.d.ts +33 -0
  114. package/dist/state/document-store.d.ts.map +1 -0
  115. package/dist/state/document-store.js +7 -0
  116. package/dist/state/document-store.js.map +1 -0
  117. package/dist/state/file-state-store.d.ts +32 -7
  118. package/dist/state/file-state-store.d.ts.map +1 -1
  119. package/dist/state/file-state-store.js +169 -250
  120. package/dist/state/file-state-store.js.map +1 -1
  121. package/dist/state/in-memory-state-store.d.ts +13 -2
  122. package/dist/state/in-memory-state-store.d.ts.map +1 -1
  123. package/dist/state/in-memory-state-store.js +87 -4
  124. package/dist/state/in-memory-state-store.js.map +1 -1
  125. package/dist/state/watch-state-document.d.ts +5 -0
  126. package/dist/state/watch-state-document.d.ts.map +1 -0
  127. package/dist/state/watch-state-document.js +486 -0
  128. package/dist/state/watch-state-document.js.map +1 -0
  129. package/dist/testing/deadline.d.ts +11 -0
  130. package/dist/testing/deadline.d.ts.map +1 -0
  131. package/dist/testing/deadline.js +23 -0
  132. package/dist/testing/deadline.js.map +1 -0
  133. package/dist/testing/fakes.d.ts +2 -0
  134. package/dist/testing/fakes.d.ts.map +1 -1
  135. package/dist/testing/fakes.js.map +1 -1
  136. package/dist/testing/index.d.ts +1 -0
  137. package/dist/testing/index.d.ts.map +1 -1
  138. package/dist/testing/index.js +1 -0
  139. package/dist/testing/index.js.map +1 -1
  140. package/dist/trajectory.d.ts +25 -0
  141. package/dist/trajectory.d.ts.map +1 -0
  142. package/dist/trajectory.js +51 -0
  143. package/dist/trajectory.js.map +1 -0
  144. package/dist/triage/heuristic.d.ts +5 -0
  145. package/dist/triage/heuristic.d.ts.map +1 -1
  146. package/dist/triage/heuristic.js +42 -3
  147. package/dist/triage/heuristic.js.map +1 -1
  148. package/dist/triage/index.d.ts +1 -1
  149. package/dist/triage/index.d.ts.map +1 -1
  150. package/dist/triage/index.js +1 -1
  151. package/dist/triage/index.js.map +1 -1
  152. package/dist/triage/schema.d.ts +73 -55
  153. package/dist/triage/schema.d.ts.map +1 -1
  154. package/dist/triage/schema.js +2 -1
  155. package/dist/triage/schema.js.map +1 -1
  156. package/dist/types.d.ts +19 -3
  157. package/dist/types.d.ts.map +1 -1
  158. package/dist/writeback/github.d.ts +22 -7
  159. package/dist/writeback/github.d.ts.map +1 -1
  160. package/dist/writeback/github.js +86 -16
  161. package/dist/writeback/github.js.map +1 -1
  162. package/dist/writeback/index.d.ts +1 -1
  163. package/dist/writeback/index.d.ts.map +1 -1
  164. package/dist/writeback/index.js +1 -1
  165. package/dist/writeback/index.js.map +1 -1
  166. package/package.json +13 -8
@@ -5,6 +5,7 @@ import { FactoryConfigSchema } from '../config/schema.js';
5
5
  import { linearByStatePath, linearByIdPath, linearByUuidPath } from '../constants/linear.js';
6
6
  import { stateResolutionFromIds } from '../linear/state-resolver.js';
7
7
  import { GithubMergeGate, closeProbePr } from '../github/index.js';
8
+ import { factoryGithubIssueCommentDraftName, isFactoryGithubIssueCommentDraftName, isFactoryGithubOperationDraftName, } from '../github/writeback-paths.js';
8
9
  import { VerificationPipeline } from '../environments/verification-pipeline.js';
9
10
  import { factoryWorktreeIssueSlug, factoryWorktreePath } from '../git/agent-worktree.js';
10
11
  import { InMemoryStateStore } from '../state/in-memory-state-store.js';
@@ -12,13 +13,14 @@ import { containsExplicitIssueReference, containsIssueKey, factoryBranchBelongsT
12
13
  import { normalizeLogger, normalizeLogValue, setSafeErrorStack, stringifyLogValue } from '../logging.js';
13
14
  import { isInFactoryScope } from '../safety/factory-scope.js';
14
15
  import { dispatchRelayflowForChangeEvent } from '../dispatch/relayflow-registry.js';
16
+ import { dispatchAgentIdentityKey } from '../dispatch/work-unit-identity.js';
15
17
  import { deriveDescriptorsFromMount, prescriptiveInstructions, } from '@agent-relay/integration-prompts';
16
18
  import { parseGithubHumanInputRequest, renderAgentTask, } from '../dispatch/templates.js';
17
19
  import { resolveTestGuidance } from '../dispatch/test-guidance.js';
18
- import { HeuristicTriage, TieredTriage, babysitterSpec, isShapeLabel, scopeFromLabels } from '../triage/index.js';
20
+ import { HeuristicTriage, TieredTriage, babysitterSpec, isShapeLabel, scopeFromLabels, swarmChannel, swarmMemberSlugs, swarmTaskFor } from '../triage/index.js';
19
21
  import { agentNameForRole, sanitizeAgentSlug } from '../triage/agent-names.js';
20
22
  import { isResourceSubscriptionsUnavailable } from '../subscriptions/index.js';
21
- import { GhCliGithubWriteback, MountGithubRead, MountLinearWriteback, MountSlackWriteback, slackChannelAliases, slackChannelSegment } from '../writeback/index.js';
23
+ import { AppGithubWriteback, FACTORY_GITHUB_STATUS_LABELS, GhCliGithubWriteback, MountGithubRead, MountLinearWriteback, MountSlackWriteback, slackChannelAliases, slackChannelSegment } from '../writeback/index.js';
22
24
  import { parseSlackThreadReply, slackThreadReplyGlob } from '../subscriptions/slack-filter.js';
23
25
  import { asRecord, parseJsonContent, stableHash, wrappedPayload } from '../writeback/shared.js';
24
26
  import { issueKey, } from './batch-tracker.js';
@@ -29,6 +31,8 @@ import { readFactoryInFlightRegistry, terminatePids } from './reaper.js';
29
31
  import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../observability/events.js';
30
32
  import { boundedRunCostTotal, CostLedger } from '../cost/ledger.js';
31
33
  import { createTicketDispatchDelivery } from '../delivery/ticket-dispatch.js';
34
+ import { canonicalTrajectorySessionRef, renderTrajectoryPointer, stripTrajectoryPointers, } from '../trajectory.js';
35
+ import { FleetControlPlaneCircuit, FleetControlPlaneCircuitOpenError, guardFleetControlPlane, } from '../fleet/control-plane-circuit.js';
32
36
  class ClarificationWakeLeaseLostError extends Error {
33
37
  }
34
38
  class ClarificationQuestionDeliveryLeaseLostError extends Error {
@@ -143,8 +147,32 @@ const STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS = 30_000;
143
147
  const RECONCILED_AGENT_EXIT_CONCURRENCY = 4;
144
148
  const SLACK_EVENT_WATERMARK_CACHE_MS = 60_000;
145
149
  const SLACK_CONVERSATION_TURN_LEASE_MS = 60_000;
150
+ // Both receipt leases guard an in-flight Slack writeback, and no fixed lease can
151
+ // cover one: MountSlackWriteback budgets 90s for the confirm alone, on top of an
152
+ // unbounded writeFile. Sizing them past that worst case would only trade a stolen
153
+ // claim for a stranded one — a lease long enough to survive the slowest write is
154
+ // equally long enough to hold the receipt hostage to a dead holder. So these
155
+ // bound the *idle* claim and #withRenewedProviderLease extends them for exactly
156
+ // as long as the write they cover is still running.
157
+ const SLACK_REPLY_ACKNOWLEDGEMENT_LEASE_MS = 60_000;
158
+ const SLACK_TERMINAL_RECEIPT_CLAIM_LEASE_MS = 60_000;
159
+ // Renewal without a ceiling is the same defect from the other side: a heartbeat
160
+ // that extends the claim for as long as the write runs also extends it forever
161
+ // when the write never returns, and nothing else can reclaim the receipt short
162
+ // of a restart. So renewal is bounded past the slowest write this daemon budgets
163
+ // for — MountSlackWriteback's 90s confirm on top of its writeFile — and beyond
164
+ // that the write is not slow, it is wedged: the heartbeat stops, the idle lease
165
+ // runs out, and the retry that owns the queued replies can take them back.
166
+ const SLACK_PROVIDER_LEASE_MAX_RENEWAL_MS = 5 * 60_000;
146
167
  const SLACK_CONVERSATION_TURN_RETRY_MS = 1_000;
147
168
  const SLACK_REPLY_ROUTE_RETRY_MS = 1_000;
169
+ // One pass drains the whole chain (#slackReplyRoutes holds only the newest
170
+ // route per key and every route awaits its predecessor). The extra passes only
171
+ // exist so the drain can prove quiescence rather than assume it.
172
+ const SLACK_REPLY_ROUTE_DRAIN_PASSES = 8;
173
+ const SLACK_TERMINAL_THREAD_GRACE_MS = 24 * 60 * 60_000;
174
+ const SLACK_TERMINAL_RECEIPT_RETRY_MS = 1_000;
175
+ const SLACK_TERMINAL_RECEIPT_RETRY_MAX_MS = 5 * 60_000;
148
176
  const MERGE_GATE_MAX_ATTEMPTS = 12;
149
177
  const MERGE_GATE_POLL_DELAY_MS = 10_000;
150
178
  const MAX_LABEL_IMPLEMENTERS = 4;
@@ -212,6 +240,25 @@ class DispatchLifecycleOwnedElsewhereError extends Error {
212
240
  this.leaseUntilMs = leaseUntilMs;
213
241
  }
214
242
  }
243
+ /**
244
+ * The durable dispatch-lifecycle claim was refused for one work unit: its
245
+ * record is already terminal, or another publisher currently holds the lease.
246
+ * Both are facts about that single unit — the rest of the pass is unaffected —
247
+ * so the readiness loop skips it and keeps going (#292).
248
+ *
249
+ * Typed rather than left as a plain `Error` so the loop can classify it by
250
+ * construction instead of by matching on `Refusing to dispatch ...` text.
251
+ */
252
+ class DispatchLifecycleClaimRefusedError extends Error {
253
+ issueKey;
254
+ refusal;
255
+ constructor(issueKey, refusal, message) {
256
+ super(message);
257
+ this.issueKey = issueKey;
258
+ this.refusal = refusal;
259
+ this.name = 'DispatchLifecycleClaimRefusedError';
260
+ }
261
+ }
215
262
  const realClock = {
216
263
  now: () => Date.now(),
217
264
  sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
@@ -224,6 +271,7 @@ export class FactoryLoop {
224
271
  #mount;
225
272
  #states;
226
273
  #fleet;
274
+ #fleetControlPlane;
227
275
  #ticketDispatchDelivery;
228
276
  #triage;
229
277
  #linear;
@@ -262,6 +310,19 @@ export class FactoryLoop {
262
310
  #dispatchInFlight = new Map();
263
311
  #slackWatchers = new Map();
264
312
  #slackWatcherStarts = new Map();
313
+ #slackTerminalWatchExpiryTimers = new Map();
314
+ #slackTerminalReceiptRetryTimers = new Map();
315
+ #terminalSlackWatchIssues = new Set();
316
+ /**
317
+ * The one in-memory record of "this work unit has an in-flight Slack side
318
+ * effect". Both ordinary reply routes and the writebacks the terminal fence
319
+ * issues on their behalf register here, because this map is what the terminal
320
+ * drain waits on: anything that touches Slack for a work unit without
321
+ * registering is invisible to the drain, and the watcher teardown that follows
322
+ * a successful drain then pulls that effect's retry timer out from under it.
323
+ */
324
+ #slackReplyRoutes = new Map();
325
+ #slackReplyRouteDrains = new Set();
265
326
  #slackConversationTurns;
266
327
  #slackConversationOwner = `${process.pid}:${randomUUID()}`;
267
328
  #githubIssueCommentWatchers = new Map();
@@ -435,6 +496,11 @@ export class FactoryLoop {
435
496
  #discoverySweepRenewTimer;
436
497
  #discoverySweepRenewalInFlight;
437
498
  #discoverySweepLeaseLost = false;
499
+ // Registry/heartbeat paths the in-flight runLoop iteration would use. A
500
+ // per-item dispatch failure now skips instead of aborting the pass (#292),
501
+ // so the loop's catch no longer runs the failure-handoff reaper for it; the
502
+ // pass reaps inline and must write to the same paths runLoop would.
503
+ #loopReapPaths;
438
504
  #discoveryOverloadError;
439
505
  #resolvedIssueSource;
440
506
  #integrationInstructions;
@@ -452,7 +518,6 @@ export class FactoryLoop {
452
518
  // synced records (state.name but no state.id) without the states catalog.
453
519
  this.#states = ports.stateResolution ?? stateResolutionFromIds(config.stateIds, config.linear.states);
454
520
  installFactoryDraftPredicate(this.#mount, config);
455
- this.#fleet = ports.fleet;
456
521
  this.#ticketDispatchDelivery = ports.ticketDispatchDelivery ?? createTicketDispatchDelivery({
457
522
  mountRoot: config.localMountRoot,
458
523
  });
@@ -461,7 +526,7 @@ export class FactoryLoop {
461
526
  safety: config.safety,
462
527
  });
463
528
  this.#githubWritebackProvided = Boolean(ports.githubWriteback);
464
- this.#githubWriteback = ports.githubWriteback ?? new GhCliGithubWriteback();
529
+ this.#githubWriteback = ports.githubWriteback ?? defaultGithubWriteback(config, ports.mount);
465
530
  this.#slack = config.slack ? MountSlackWriteback(ports.mount, config.slack) : ports.slack;
466
531
  this.#github = ports.github ?? MountGithubRead(ports.mount);
467
532
  this.#mergeGate = ports.mergeGate ?? new GithubMergeGate();
@@ -483,6 +548,13 @@ export class FactoryLoop {
483
548
  this.#probePrResolver = ports.probePrResolver ?? ((issue) => this.#resolveIssuePr(issue));
484
549
  this.#logger = normalizeLogger(ports.logger ?? console);
485
550
  this.#clock = ports.clock ?? realClock;
551
+ this.#fleetControlPlane = new FleetControlPlaneCircuit({
552
+ timeoutMs: config.fleetHealth.rosterTimeoutMs,
553
+ failureThreshold: config.fleetHealth.failureThreshold,
554
+ resetTimeoutMs: config.fleetHealth.resetTimeoutMs,
555
+ now: () => this.#clock.now(),
556
+ });
557
+ this.#fleet = guardFleetControlPlane(ports.fleet, this.#fleetControlPlane);
486
558
  this.#processIdentityReader = ports.processIdentityReader ?? readProcessIdentity;
487
559
  this.#processFinder = ports.processFinder ?? ((agentName, opts) => findAgentProcessByName(agentName, {
488
560
  readProcessIdentity: this.#processIdentityReader,
@@ -590,11 +662,32 @@ export class FactoryLoop {
590
662
  this.#batchView = batch;
591
663
  return batch;
592
664
  }
665
+ /**
666
+ * Resolves once this issue's durable dispatch row reaches a terminal phase,
667
+ * and reports which one. A caller that turns the run into an exit code needs
668
+ * the phase: a dispatch that hit capacity returns an empty hold result and
669
+ * schedules a durable retry, so the pre-wait result says nothing about how
670
+ * the run actually ended.
671
+ *
672
+ * `undefined` means no terminal phase was observed — there was no lifecycle
673
+ * row to wait on, or the wait ended because Factory is stopping.
674
+ */
593
675
  async waitForDispatchTerminal(issue) {
594
676
  const key = issueKey(issue);
595
677
  const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
596
- if (lifecycle && isTerminalDispatchLifecycle(lifecycle))
597
- return;
678
+ // No durable row means this dispatch never claimed a lifecycle: a
679
+ // dependency park, a triage escalation, and a label refusal all return
680
+ // before the claim. Nothing can ever become terminal, so polling would
681
+ // never stop and the caller would never produce an exit code at all.
682
+ if (!lifecycle)
683
+ return undefined;
684
+ if (isTerminalDispatchLifecycle(lifecycle))
685
+ return lifecycle.phase;
686
+ // Capture the phase at the moment this waiter observes it. The waiters are
687
+ // shared across callers of one row and carry no payload, so a re-read after
688
+ // the fact can race lifecycle cleanup or a reopened dispatch for the same
689
+ // issue and report a different run — or none.
690
+ let observedPhase;
598
691
  await new Promise((resolve) => {
599
692
  let settled = false;
600
693
  let timer;
@@ -603,10 +696,11 @@ export class FactoryLoop {
603
696
  waiters = new Set();
604
697
  this.#dispatchTerminalWaiters.set(key, waiters);
605
698
  }
606
- const finish = () => {
699
+ const finish = (phase) => {
607
700
  if (settled)
608
701
  return;
609
702
  settled = true;
703
+ observedPhase = phase;
610
704
  if (timer)
611
705
  clearTimeout(timer);
612
706
  const current = this.#dispatchTerminalWaiters.get(key);
@@ -629,7 +723,7 @@ export class FactoryLoop {
629
723
  try {
630
724
  const latest = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
631
725
  if (latest && isTerminalDispatchLifecycle(latest)) {
632
- this.#resolveDispatchTerminalWaiters(issue);
726
+ this.#resolveDispatchTerminalWaiters(issue, latest.phase);
633
727
  return;
634
728
  }
635
729
  if (latest?.phase === 'waiting-for-human' && this.#startMode === 'dispatch-owner') {
@@ -659,6 +753,10 @@ export class FactoryLoop {
659
753
  };
660
754
  void poll();
661
755
  });
756
+ // `observedPhase` is whatever resolved THIS waiter. It stays undefined only
757
+ // when the wait ended without a terminal resolution at all — Factory is
758
+ // stopping — which is exactly what `undefined` reports.
759
+ return observedPhase;
662
760
  }
663
761
  async start(opts = {}) {
664
762
  if (this.#started) {
@@ -871,6 +969,14 @@ export class FactoryLoop {
871
969
  await this.#boundedStopTeardown('factory subscription unsubscribe', () => subscription?.unsubscribe());
872
970
  await Promise.all([...this.#slackWatchers.values()].map((watcher) => watcher.stop()));
873
971
  this.#slackWatchers.clear();
972
+ for (const timer of this.#slackTerminalWatchExpiryTimers.values())
973
+ clearTimeout(timer);
974
+ this.#slackTerminalWatchExpiryTimers.clear();
975
+ for (const timer of this.#slackTerminalReceiptRetryTimers.values())
976
+ clearTimeout(timer);
977
+ this.#slackTerminalReceiptRetryTimers.clear();
978
+ this.#terminalSlackWatchIssues.clear();
979
+ this.#slackReplyRouteDrains.clear();
874
980
  await Promise.all([...this.#githubIssueCommentWatchers.values()].map((watcher) => watcher.stop()));
875
981
  this.#githubIssueCommentWatchers.clear();
876
982
  this.#githubIssueCommentWatchStates.clear();
@@ -1831,8 +1937,30 @@ export class FactoryLoop {
1831
1937
  }
1832
1938
  }
1833
1939
  }
1940
+ async #assertFleetControlPlaneAvailable() {
1941
+ try {
1942
+ await this.#fleet.roster();
1943
+ this.#increment('fleetControlPlaneProbeSuccesses');
1944
+ }
1945
+ catch (error) {
1946
+ const health = this.#fleetControlPlane.status();
1947
+ this.#increment('fleetControlPlaneProbeFailures');
1948
+ if (health.state === 'open')
1949
+ this.#increment('fleetControlPlaneCircuitOpen');
1950
+ this.#logger.error?.('[factory] fleet control plane unavailable; dispatch paused', {
1951
+ state: health.state,
1952
+ consecutiveFailures: health.consecutiveFailures,
1953
+ retryAtMs: health.retryAtMs,
1954
+ error: health.lastError ?? 'unknown control-plane failure',
1955
+ });
1956
+ throw contextualError('Factory dispatch paused because the fleet control plane is unavailable', error);
1957
+ }
1958
+ }
1834
1959
  async #runOnceWithDiscoveryFence(opts) {
1835
1960
  const sweepStartedAtMs = this.#clock.now();
1961
+ if (!(opts.dryRun ?? this.#config.dryRun)) {
1962
+ await this.#assertFleetControlPlaneAvailable();
1963
+ }
1836
1964
  let claim = await this.#state.claimDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, this.#clock.now(), DISCOVERY_SWEEP_LEASE_MS);
1837
1965
  if (!claim.acquired && claim.reason === 'backoff') {
1838
1966
  const delayMs = Math.max(0, claim.state.backoffUntilMs - this.#clock.now());
@@ -1989,6 +2117,10 @@ export class FactoryLoop {
1989
2117
  reason: entry.reason,
1990
2118
  });
1991
2119
  };
2120
+ // Backstop for the skip-by-default catch below: see #292. Reset only by
2121
+ // a completed dispatch, so the name says "since a dispatch" rather than
2122
+ // "consecutive" — a benign classified skip in between does not clear it.
2123
+ let unclassifiedFailuresSinceDispatch = 0;
1992
2124
  let lastReadyReadProgressAtMs = this.#clock.now();
1993
2125
  let readyIssueReads = 0;
1994
2126
  const issueEntries = [];
@@ -2086,19 +2218,10 @@ export class FactoryLoop {
2086
2218
  }
2087
2219
  const decision = await this.triageIssue(issue);
2088
2220
  triaged.push(decision);
2089
- let result;
2090
- try {
2091
- result = await this.dispatch(decision, { dryRun });
2092
- }
2093
- catch (error) {
2094
- if (!(error instanceof LiveDispatchStateChangedError))
2095
- throw error;
2096
- recordSkip({ issue: decision.issue, reason: 'live state changed during dispatch' });
2097
- this.#logger.info?.('[factory] skipped issue whose live state changed during dispatch', {
2098
- issue: decision.issue.key,
2099
- });
2100
- continue;
2101
- }
2221
+ const result = await this.dispatch(decision, { dryRun });
2222
+ // A completed dispatch — even one that parks or escalates the issue —
2223
+ // proves the pipeline still works, so the fuse below starts over.
2224
+ unclassifiedFailuresSinceDispatch = 0;
2102
2225
  if (result.agents.length === 0 && !dryRun) {
2103
2226
  const reason = result.hold?.kind === 'dependency-cycle'
2104
2227
  ? `dependency cycle detected: ${result.hold.cycle?.join(' -> ') ?? 'unknown cycle'}`
@@ -2111,6 +2234,52 @@ export class FactoryLoop {
2111
2234
  dispatched.push(result);
2112
2235
  }
2113
2236
  }
2237
+ catch (error) {
2238
+ // #292: issues in a pass are independent work units, so a failure
2239
+ // that is about ONE unit costs that unit and nothing else. Only the
2240
+ // conditions named in `#isPassFatalFailure` — the ones where
2241
+ // continuing the pass is meaningless — abort the whole sweep.
2242
+ if (this.#isPassFatalFailure(error, dryRun))
2243
+ throw error;
2244
+ if (!isClassifiedPerItemDispatchFailure(error)) {
2245
+ unclassifiedFailuresSinceDispatch += 1;
2246
+ // A pass-wide fault can arrive disguised as a run of per-item
2247
+ // faults. Skipping every unit would then hand back a green report
2248
+ // that dispatched nothing, which is the same silent wedge #292 is
2249
+ // about, wearing the opposite costume. Fail the pass loudly so
2250
+ // `readinessReconcile.lastError` carries the cause.
2251
+ if (unclassifiedFailuresSinceDispatch >= UNCLASSIFIED_DISPATCH_FAILURE_LIMIT) {
2252
+ throw contextualError(`Aborting readiness pass after ${unclassifiedFailuresSinceDispatch} unclassified dispatch failures without a successful dispatch`, error);
2253
+ }
2254
+ this.#increment('dispatchItemFailuresSkipped');
2255
+ // The raw message is operator-facing only; the run report carries
2256
+ // the sanitized classification from `perItemDispatchSkipReason`.
2257
+ this.#logger.warn?.('[factory] skipped a work unit whose dispatch failed; continuing the pass', {
2258
+ issue: issueRef(issue).key,
2259
+ unclassifiedFailuresSinceDispatch,
2260
+ error: describeError(error).errorMessage,
2261
+ });
2262
+ this.#error(error, issueRef(issue));
2263
+ // The failure may have left half-spawned agents behind. runLoop's
2264
+ // catch used to reap them because this error aborted the pass;
2265
+ // now that the pass survives, the reap has to happen here or the
2266
+ // agents leak until the next failed iteration.
2267
+ await this.#reapDispatchFailureHandoffsNow();
2268
+ }
2269
+ else {
2270
+ // Not an error — the unit simply cannot be dispatched right now —
2271
+ // so this stays out of `counters.errors` and gets its own counter
2272
+ // instead, or a terminal-lifecycle backlog would be invisible to
2273
+ // anyone watching only `dispatchItemFailuresSkipped`.
2274
+ this.#increment('dispatchItemsSkippedUndispatchable');
2275
+ this.#logger.info?.('[factory] skipped a work unit that cannot be dispatched right now', {
2276
+ issue: issueRef(issue).key,
2277
+ error: describeError(error).errorMessage,
2278
+ });
2279
+ }
2280
+ recordSkip({ issue: issueRef(issue), reason: perItemDispatchSkipReason(error) });
2281
+ continue;
2282
+ }
2114
2283
  finally {
2115
2284
  if (recoveredIdentity)
2116
2285
  this.#reconciledGithubInProgress.delete(recoveredIdentity);
@@ -2144,6 +2313,79 @@ export class FactoryLoop {
2144
2313
  }
2145
2314
  }
2146
2315
  }
2316
+ /**
2317
+ * Whether a failure raised while processing ONE work unit must abort the
2318
+ * whole readiness pass instead of skipping that unit.
2319
+ *
2320
+ * The default is the opposite, and that inversion is the fix for #292.
2321
+ * Issues in a pass are independent work units: a failure that is *about one
2322
+ * unit* — its dispatch-lifecycle record, its live state, a provider fault on
2323
+ * its own writeback — costs that unit and nothing else. Before this, every
2324
+ * error except `LiveDispatchStateChangedError` escaped the `for` loop, so a
2325
+ * single issue whose lifecycle record had gone terminal stopped all dispatch
2326
+ * indefinitely, every pass.
2327
+ *
2328
+ * A condition belongs here only when continuing the pass is meaningless or
2329
+ * actively harmful — when the failure is about the *pass*, not the item:
2330
+ *
2331
+ * - The discovery sweep lease is gone. Another process now owns this
2332
+ * workspace's sweep, so every remaining read throws the same way and each
2333
+ * one would be recorded as an ordinary per-issue skip. The run report
2334
+ * would then claim a clean pass over work this process no longer has the
2335
+ * right to touch.
2336
+ * - Relayfile signalled overload for this sweep. The backend is shedding
2337
+ * load; grinding through the remaining units makes it worse, and
2338
+ * `#runOnceWithDiscoveryFence` is going to rethrow this at the fence
2339
+ * anyway.
2340
+ * - The factory is stopping. Teardown is in progress and dispatching more
2341
+ * agents now leaks them past the shutdown deadline.
2342
+ * - The fleet control-plane circuit is no longer closed, **on a live pass**.
2343
+ * Dispatch is globally paused — the same condition
2344
+ * `#assertFleetControlPlaneAvailable` refuses to *start* a live pass on,
2345
+ * so it must also stop one already in flight. A dry run is exempt: it
2346
+ * never calls that admission gate and never spawns, so a paused control
2347
+ * plane is irrelevant to it rather than fatal to it. Without the
2348
+ * exemption, one live pass that trips the circuit would poison every
2349
+ * later dry run — including the boot gate's own `run-once --dry-run`
2350
+ * probe, turning a recoverable circuit-open condition into a failed boot.
2351
+ * That is a nastier version of the wedge this whole change removes.
2352
+ *
2353
+ * Deliberately NOT here: JavaScript builtin error types. Classifying
2354
+ * "programmer faults" such as `TypeError` as fatal is the obvious next rule
2355
+ * and it is a trap — Node reports a failed `fetch` as `TypeError: fetch
2356
+ * failed`, which is precisely the transient per-item roster lookup that
2357
+ * wedged the second instance (#291). A rule keyed on builtin types would
2358
+ * have preserved that outage verbatim.
2359
+ *
2360
+ * Everything else — a refused lifecycle claim, a terminal lifecycle record,
2361
+ * a transient network fault on one issue — is per-item: record a skip and
2362
+ * keep going. The unclassified-failure fuse in `#performRunOnce` is the
2363
+ * backstop for a pass-wide fault that does not announce itself as one.
2364
+ */
2365
+ #isPassFatalFailure(error, dryRun) {
2366
+ // Sweep-scoped: these are about this process's right or ability to run the
2367
+ // pass at all, so they hold for a dry run exactly as for a live one.
2368
+ if (this.#discoverySweepLeaseLost || this.#discoveryOverloadError !== undefined || this.#stopping) {
2369
+ return true;
2370
+ }
2371
+ // Fleet-scoped, and therefore live-only. See the doc comment above.
2372
+ return !dryRun && this.#isFleetControlPlaneHalted(error);
2373
+ }
2374
+ /**
2375
+ * Whether dispatch is globally paused by the fleet control-plane circuit.
2376
+ *
2377
+ * Two reads, because the circuit announces itself two different ways. The
2378
+ * state read covers `guardedMutation`, which records a mutation's own
2379
+ * transport failure and rethrows the *original* error rather than the
2380
+ * circuit-open type — converting it there would be wrong, since the mutation
2381
+ * may already have reached the broker and callers key spawn-failure handling
2382
+ * off that original error. The type check covers a rejection raised without
2383
+ * any state transition, such as an already-open circuit refusing admission.
2384
+ */
2385
+ #isFleetControlPlaneHalted(error) {
2386
+ return this.#fleetControlPlane.status().state !== 'closed' ||
2387
+ wrapsErrorOfType(error, FleetControlPlaneCircuitOpenError);
2388
+ }
2147
2389
  #startDiscoverySweepRenewal(epoch) {
2148
2390
  this.#discoverySweepRenewTimer = setInterval(() => {
2149
2391
  if (this.#discoverySweepRenewalInFlight || this.#discoverySweepLeaseLost)
@@ -2699,7 +2941,7 @@ export class FactoryLoop {
2699
2941
  await this.#state.clearBabysitterSession(this.#workspaceId, issueKey(lifecycle.issue));
2700
2942
  this.#dispatchLifecycleEpochs.delete(key);
2701
2943
  this.#abandonedDispatchReasons.delete(key);
2702
- this.#resolveDispatchTerminalWaiters(lifecycle.issue);
2944
+ this.#resolveDispatchTerminalWaiters(lifecycle.issue, 'abandoned');
2703
2945
  await this.#writeInFlightRegistry().catch((error) => {
2704
2946
  this.#logger.warn?.('[factory] failed to rewrite registry after orphaned claim release', {
2705
2947
  issue: issue.key,
@@ -3027,6 +3269,7 @@ export class FactoryLoop {
3027
3269
  const maxConsecutiveFailures = Math.min(5, Math.max(1, Math.trunc(opts.maxConsecutiveFailures ?? this.#config.loop.maxConsecutiveFailures)));
3028
3270
  const heartbeatPath = opts.heartbeatPath ?? this.#config.loop.heartbeatPath;
3029
3271
  const registryPath = opts.registryPath ?? this.#config.loop.registryPath;
3272
+ this.#loopReapPaths = { heartbeatPath, registryPath };
3030
3273
  const reports = [];
3031
3274
  let consecutiveFailures = 0;
3032
3275
  let completed = false;
@@ -3054,6 +3297,16 @@ export class FactoryLoop {
3054
3297
  reports.push(failedIterationReport(error, opts.dryRun ?? this.#config.dryRun));
3055
3298
  await this.#reapDispatchFailureHandoffsNow(heartbeatPath, registryPath);
3056
3299
  await this.#writeLoopHeartbeat(heartbeatPath, registryPath, 'running', iteration + 1, maxIterations);
3300
+ const fleetControlPlane = this.#fleetControlPlane.status();
3301
+ if (fleetControlPlane.state !== 'closed') {
3302
+ this.#increment('loopCircuitBreaks');
3303
+ this.#logger.error?.('[factory] stopping loop because dispatch is paused by the fleet control-plane circuit', {
3304
+ state: fleetControlPlane.state,
3305
+ consecutiveFailures: fleetControlPlane.consecutiveFailures,
3306
+ retryAtMs: fleetControlPlane.retryAtMs,
3307
+ });
3308
+ throw new FleetControlPlaneCircuitOpenError(fleetControlPlane.retryAtMs ?? this.#clock.now(), fleetControlPlane.state);
3309
+ }
3057
3310
  if (consecutiveFailures >= maxConsecutiveFailures) {
3058
3311
  this.#increment('loopCircuitBreaks');
3059
3312
  this.#logger.error?.('[factory] stopping loop after consecutive iteration failures', {
@@ -3072,6 +3325,7 @@ export class FactoryLoop {
3072
3325
  return reports;
3073
3326
  }
3074
3327
  finally {
3328
+ this.#loopReapPaths = undefined;
3075
3329
  if (!completed) {
3076
3330
  await this.#writeLoopHeartbeat(heartbeatPath, registryPath, 'stopping', reports.length, maxIterations);
3077
3331
  }
@@ -3209,6 +3463,13 @@ export class FactoryLoop {
3209
3463
  };
3210
3464
  }
3211
3465
  this.#clearDependencyPark(batch, dispatchDecision.issue);
3466
+ // Event-driven and direct dispatches do not necessarily pass through issue
3467
+ // discovery. Admit them before creating previews, claiming a lifecycle, or
3468
+ // consuming a dispatch attempt. The mutation proxy probes again at the
3469
+ // actual spawn/resume boundary so a later control-plane fault still fails
3470
+ // closed.
3471
+ if (!dryRun)
3472
+ await this.#assertFleetControlPlaneAvailable();
3212
3473
  const durableDispatch = !dryRun && this.#usesDurableDispatchLifecycle();
3213
3474
  // Local dispatches need the same deterministic branch identity as remote
3214
3475
  // ones. Without it, every worker starts in the configured shared checkout
@@ -3512,6 +3773,7 @@ export class FactoryLoop {
3512
3773
  capacityBlocked: parked.capacityBlocked,
3513
3774
  })) ?? [],
3514
3775
  counters: { ...this.#counters },
3776
+ fleetControlPlane: this.#fleetControlPlane.status(),
3515
3777
  slackDegraded: this.#slackDegraded,
3516
3778
  slackDegradedReason: this.#slackDegradedReason,
3517
3779
  eventListener: this.#eventListenerStatus(),
@@ -4066,10 +4328,11 @@ export class FactoryLoop {
4066
4328
  seed.decision = decisionWithLifecycleBranches(seed.decision, seed.runId);
4067
4329
  const claim = await this.#state.claimDispatchLifecycle(this.#workspaceId, key, seed, this.#dispatchLifecycleOwner, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS);
4068
4330
  if (!claim.acquired || !claim.lease) {
4069
- const reason = isTerminalDispatchLifecycle(claim.lifecycle)
4331
+ const terminal = isTerminalDispatchLifecycle(claim.lifecycle);
4332
+ const reason = terminal
4070
4333
  ? 'dispatch lifecycle is already terminal'
4071
4334
  : `dispatch lifecycle is owned by ${claim.lifecycle.lease?.owner ?? 'another publisher'}`;
4072
- throw new Error(`Refusing to dispatch ${decision.issue.key}: ${reason}`);
4335
+ throw new DispatchLifecycleClaimRefusedError(decision.issue.key, terminal ? 'terminal' : 'owned-elsewhere', `Refusing to dispatch ${decision.issue.key}: ${reason}`);
4073
4336
  }
4074
4337
  this.#dispatchLifecycleEpochs.set(claim.key ?? key, claim.lease.epoch);
4075
4338
  this.#hydrateCostLedger(claim.lifecycle);
@@ -4415,12 +4678,25 @@ export class FactoryLoop {
4415
4678
  }
4416
4679
  for (const agent of lifecycle.agents) {
4417
4680
  const previousAgent = previous?.agents.find((candidate) => candidate.name === agent.name);
4418
- if (previousAgent?.releasedAtMs !== undefined)
4419
- agent.releasedAtMs = previousAgent.releasedAtMs;
4681
+ // The in-flight record is authoritative for release state: the stamp
4682
+ // now lives on the tracked agent and round-trips through the lifecycle,
4683
+ // so a still-released agent brings its own stamp here. Inheriting the
4684
+ // previous row's stamp instead would refile a respawned worker as
4685
+ // released — a respawn reuses both the agent name and the deterministic
4686
+ // invocation id, so neither can tell the generations apart — and every
4687
+ // consumer that filters on `releasedAtMs` would then treat a live
4688
+ // worker as gone.
4420
4689
  if (previousAgent?.costUsage)
4421
4690
  agent.costUsage = structuredClone(previousAgent.costUsage);
4422
- if (releasedAgentNames.has(agent.name))
4423
- agent.releasedAtMs ??= this.#clock.now();
4691
+ if (releasedAgentNames.has(agent.name)) {
4692
+ const releasedAtMs = agent.releasedAtMs ?? this.#clock.now();
4693
+ agent.releasedAtMs = releasedAtMs;
4694
+ // Keep the record the single source of the stamp, so the next save
4695
+ // carries it without having to read it back off the durable row.
4696
+ const tracked = record.agents.get(agent.name);
4697
+ if (tracked)
4698
+ tracked.releasedAtMs ??= releasedAtMs;
4699
+ }
4424
4700
  }
4425
4701
  const saved = await this.#state.saveDispatchLifecycle(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch, this.#clock.now(), lifecycle);
4426
4702
  if (!saved) {
@@ -4448,10 +4724,14 @@ export class FactoryLoop {
4448
4724
  return true;
4449
4725
  });
4450
4726
  }
4451
- #resolveDispatchTerminalWaiters(issue) {
4727
+ // Every caller of this knows the phase the row settled in, and each waiter
4728
+ // must be handed it directly. A waiter that instead re-read the shared row
4729
+ // after release could see it cleared, or see the next dispatch for the same
4730
+ // issue, and classify the wrong run.
4731
+ #resolveDispatchTerminalWaiters(issue, phase) {
4452
4732
  const key = issueKey(issue);
4453
4733
  for (const resolve of this.#dispatchTerminalWaiters.get(key) ?? [])
4454
- resolve();
4734
+ resolve(phase);
4455
4735
  this.#dispatchTerminalWaiters.delete(key);
4456
4736
  }
4457
4737
  #scheduleDispatchLifecycleRetry(record) {
@@ -4538,7 +4818,7 @@ export class FactoryLoop {
4538
4818
  if (!lifecycle)
4539
4819
  return;
4540
4820
  if (isTerminalDispatchLifecycle(lifecycle)) {
4541
- this.#resolveDispatchTerminalWaiters(lifecycle.issue);
4821
+ this.#resolveDispatchTerminalWaiters(lifecycle.issue, lifecycle.phase);
4542
4822
  return;
4543
4823
  }
4544
4824
  if (lifecycle.phase === 'waiting-for-human')
@@ -4915,11 +5195,11 @@ export class FactoryLoop {
4915
5195
  for (const [name] of record.agents) {
4916
5196
  this.#fleet.markAgentTerminal?.(name, 'durable-dispatch-abandoned');
4917
5197
  }
4918
- await this.#stopSlackWatcher(record.issue);
5198
+ await this.#retireSlackWatcher(record);
4919
5199
  await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
4920
5200
  await this.#writeInFlightRegistry();
4921
5201
  this.#increment('dispatchLifecycleStaleIssuesAbandoned');
4922
- this.#resolveDispatchTerminalWaiters(record.issue);
5202
+ this.#resolveDispatchTerminalWaiters(record.issue, 'abandoned');
4923
5203
  this.#logger.info?.('[factory] abandoned durable dispatch whose live issue is no longer ready', {
4924
5204
  issue: record.issue.key,
4925
5205
  reason,
@@ -4995,7 +5275,7 @@ export class FactoryLoop {
4995
5275
  this.#increment(releaseReason === 'issue-human-review' ? 'humanReview' : 'done');
4996
5276
  this.#emit('issue-done', { issue: record.issue });
4997
5277
  await this.#writeInFlightRegistry();
4998
- this.#resolveDispatchTerminalWaiters(record.issue);
5278
+ this.#resolveDispatchTerminalWaiters(record.issue, 'complete');
4999
5279
  return true;
5000
5280
  }
5001
5281
  async #assertDispatchLifecycleOwner(record) {
@@ -6019,6 +6299,7 @@ export class FactoryLoop {
6019
6299
  registryPath,
6020
6300
  eventListener: this.#eventListenerStatus(),
6021
6301
  readinessReconcile: this.#readinessReconcileStatus(),
6302
+ fleetControlPlane: this.#fleetControlPlane.status(),
6022
6303
  };
6023
6304
  await mkdir(dirname(path), { recursive: true });
6024
6305
  await writeFile(path, `${JSON.stringify(heartbeat, null, 2)}\n`, 'utf8');
@@ -6037,7 +6318,7 @@ export class FactoryLoop {
6037
6318
  },
6038
6319
  });
6039
6320
  }
6040
- async #reapDispatchFailureHandoffsNow(heartbeatPath, registryPath) {
6321
+ async #reapDispatchFailureHandoffsNow(heartbeatPath = this.#loopReapPaths?.heartbeatPath ?? this.#config.loop.heartbeatPath, registryPath = this.#loopReapPaths?.registryPath ?? this.#config.loop.registryPath) {
6041
6322
  const handoffs = await this.#state.listFailureHandoffs(this.#workspaceId);
6042
6323
  if (handoffs.length === 0) {
6043
6324
  return;
@@ -6277,6 +6558,22 @@ export class FactoryLoop {
6277
6558
  }
6278
6559
  try {
6279
6560
  await this.#fleet.release(agentName, reason);
6561
+ // Invalidate the spawn memory the moment the release is confirmed, so a
6562
+ // retry for the same deterministic invocation spawns a real worker
6563
+ // instead of inheriting this one's claim. Shutdown is excluded on
6564
+ // purpose: nothing is being retried there, and a takeover must still be
6565
+ // free to adopt agents this process merely stopped supervising.
6566
+ if (record && batch && context !== 'stop') {
6567
+ const releasedInvocationId = batch.recordRelease(record, agentName, this.#clock.now());
6568
+ if (releasedInvocationId) {
6569
+ this.#logger.debug?.('[factory] released agent invocation is no longer dispatchable', {
6570
+ issue: record.issue.key,
6571
+ agentName,
6572
+ reason,
6573
+ invocationId: releasedInvocationId,
6574
+ });
6575
+ }
6576
+ }
6280
6577
  if (record)
6281
6578
  await this.#reportAgent(record, tracked, 'agent.released', { releaseReason: reason });
6282
6579
  }
@@ -6506,7 +6803,10 @@ export class FactoryLoop {
6506
6803
  const batch = await this.#batch();
6507
6804
  const invocationId = batch.invocationIdFor(record.issue, spec);
6508
6805
  const existing = record.agents.get(spec.name);
6509
- if (existing?.result) {
6806
+ // A released placement keeps its bookkeeping entry, but it is not a worker.
6807
+ // Answering with its old spawn result here would report a synthetic success
6808
+ // for a process that no longer exists.
6809
+ if (existing?.result && existing.releasedAtMs === undefined) {
6510
6810
  this.#scheduleHeldAgentDeadline(record);
6511
6811
  return { name: existing.result?.name ?? spec.name };
6512
6812
  }
@@ -6556,6 +6856,7 @@ export class FactoryLoop {
6556
6856
  result = await this.#fleet.spawn({
6557
6857
  name: spec.name,
6558
6858
  capability: spec.capability,
6859
+ identityKey: dispatchAgentIdentityKey(record.issue, spec.role),
6559
6860
  node: spec.node ?? 'self',
6560
6861
  repo: spec.repo,
6561
6862
  task: spec.task,
@@ -6657,6 +6958,16 @@ export class FactoryLoop {
6657
6958
  return;
6658
6959
  }
6659
6960
  }
6961
+ // Swarm workers share the lead's checkout and lifecycle branch. If a worker
6962
+ // exit reached the publication/completion paths below, whichever worker
6963
+ // finished first would publish whatever partial state was on the shared
6964
+ // branch and mark every swarm member "done" via the shared-branch PR probe,
6965
+ // releasing the still-working lead. The lead alone is authoritative for
6966
+ // publication and completion in a swarm.
6967
+ if (exiting?.spec.swarmRole === 'worker') {
6968
+ this.#increment('swarmWorkerExitsSuppressed');
6969
+ return;
6970
+ }
6660
6971
  if (isCompletionReason(reason)) {
6661
6972
  if (exiting?.spec.role === 'implementer' && await this.#issueHasCompletionPr(record, {
6662
6973
  openOnly: this.#config.babysitter.enabled,
@@ -6901,6 +7212,7 @@ export class FactoryLoop {
6901
7212
  const result = await this.#fleet.spawn({
6902
7213
  name: tracked.spec.name,
6903
7214
  capability: tracked.spec.capability,
7215
+ identityKey: dispatchAgentIdentityKey(record.issue, tracked.spec.role),
6904
7216
  node: tracked.result?.node ?? tracked.spec.node ?? 'self',
6905
7217
  repo: tracked.spec.repo,
6906
7218
  task: tracked.spec.task,
@@ -7024,6 +7336,7 @@ export class FactoryLoop {
7024
7336
  }
7025
7337
  async #publishImplementerPullRequest(record, implementer, opts = {}) {
7026
7338
  const key = `${issueKey(record.issue)}:${implementer.spec.repo}`;
7339
+ const trajectorySessionRef = canonicalTrajectorySessionRef(implementer.sessionRef);
7027
7340
  const expectedHeadRef = implementer.spec.branch;
7028
7341
  if (!expectedHeadRef) {
7029
7342
  throw new Error(`Refusing to publish ${record.issue.key}: implementer has no Factory-derived branch`);
@@ -7089,7 +7402,7 @@ export class FactoryLoop {
7089
7402
  expectedHeadRef,
7090
7403
  baseRef,
7091
7404
  title: `${issue.key}: ${issue.title}`,
7092
- body: githubPullRequestBody(issue, implementer.spec.preview),
7405
+ body: githubPullRequestBody(issue, implementer.spec.preview, trajectorySessionRef),
7093
7406
  ...(implementer.sessionRef ? { sessionRef: implementer.sessionRef } : {}),
7094
7407
  });
7095
7408
  const published = result.author
@@ -7678,7 +7991,7 @@ export class FactoryLoop {
7678
7991
  await this.#recordDispatchTerminal(record.issue);
7679
7992
  const next = (await this.#batch()).complete(record.issue);
7680
7993
  await this.#drainReadyClarificationWake();
7681
- await this.#stopSlackWatcher(record.issue);
7994
+ await this.#retireSlackWatcher(record);
7682
7995
  await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
7683
7996
  await this.#writeInFlightRegistry();
7684
7997
  if (next) {
@@ -7782,6 +8095,7 @@ export class FactoryLoop {
7782
8095
  const result = await this.#fleet.resume({
7783
8096
  name,
7784
8097
  sessionRef: tracked.sessionRef,
8098
+ identityKey: dispatchAgentIdentityKey(record.issue, tracked.spec.role),
7785
8099
  node: tracked.result?.node ?? tracked.spec.node ?? 'self',
7786
8100
  capability: tracked.spec.capability,
7787
8101
  repo: tracked.spec.repo,
@@ -7910,6 +8224,7 @@ export class FactoryLoop {
7910
8224
  const result = await this.#fleet.spawn({
7911
8225
  name: replacementSpec.name,
7912
8226
  capability: replacementSpec.capability,
8227
+ identityKey: dispatchAgentIdentityKey(record.issue, replacementSpec.role),
7913
8228
  node: tracked.result?.node ?? replacementSpec.node ?? 'self',
7914
8229
  repo: replacementSpec.repo,
7915
8230
  task: replacementSpec.task,
@@ -9377,6 +9692,15 @@ export class FactoryLoop {
9377
9692
  previewStartCommand: spec.preview?.startCommand,
9378
9693
  } : {}),
9379
9694
  ...(this.#fleet.lifecycleActionName ? { lifecycleActionName: this.#fleet.lifecycleActionName } : {}),
9695
+ ...(spec.swarmRole && spec.channel ? {
9696
+ swarm: {
9697
+ role: spec.swarmRole,
9698
+ channel: spec.channel,
9699
+ otherMemberNames: decision.implementers
9700
+ .filter((implementer) => implementer.channel === spec.channel && implementer.name !== spec.name)
9701
+ .map((implementer) => implementer.name),
9702
+ },
9703
+ } : {}),
9380
9704
  }),
9381
9705
  };
9382
9706
  };
@@ -10897,7 +11221,7 @@ export class FactoryLoop {
10897
11221
  });
10898
11222
  return;
10899
11223
  }
10900
- const issue = await this.#findMergeAdvanceIssueForPr(snapshot);
11224
+ const issue = await this.#findMergeAdvanceIssueForPr(snapshot, repo);
10901
11225
  if (!issue) {
10902
11226
  this.#increment('mergedPrAdvanceNoIssue');
10903
11227
  return;
@@ -10952,7 +11276,7 @@ export class FactoryLoop {
10952
11276
  this.#error(error, issueRef(issue));
10953
11277
  }
10954
11278
  }
10955
- async #findMergeAdvanceIssueForPr(snapshot) {
11279
+ async #findMergeAdvanceIssueForPr(snapshot, eventRepo) {
10956
11280
  // An issue is "upstream" of a merge if it sits in the agent-implementing or
10957
11281
  // human-review role for its team. UUIDs are globally unique, so the reverse
10958
11282
  // role lookup covers every team without per-team scoping here.
@@ -10961,16 +11285,34 @@ export class FactoryLoop {
10961
11285
  return role === 'agentImplementing' || role === 'humanReview';
10962
11286
  };
10963
11287
  let best;
11288
+ let ambiguous = false;
10964
11289
  const scanStartedAtMs = this.#clock.now();
10965
11290
  // This no-record path runs after agents are released, so there is no
10966
11291
  // tracked PR identity left. Keep the scan simple and prefer branch identity
10967
11292
  // over title/body references to avoid "related to AR-N" body false positives.
10968
11293
  const githubSource = await this.#issueSource() === 'github';
11294
+ // A GitHub-native issue key is a bare number; the same number exists
11295
+ // independently in every configured repository. Only a same-repo candidate
11296
+ // may complete the merge, so a merged PR in one repo can never close an
11297
+ // unrelated issue that happens to share its number in another. #276.
11298
+ const eventRepoKey = validGithubRepo(eventRepo) ? eventRepo.toLowerCase() : undefined;
11299
+ if (githubSource && !eventRepoKey) {
11300
+ this.#logger.warn?.('[factory] merge advance skipped: event repo is not identifiable', {
11301
+ prNumber: snapshot.number,
11302
+ });
11303
+ return undefined;
11304
+ }
10969
11305
  const paths = githubSource ? await this.#githubIssuePaths() : await this.#listRelayfileTree(ISSUE_ROOT, 'merge advance issue scan');
10970
11306
  for (const path of paths) {
10971
11307
  if (githubSource ? !isGithubIssueFilePath(path) : !isIssueFilePath(path)) {
10972
11308
  continue;
10973
11309
  }
11310
+ if (githubSource) {
11311
+ const issueParts = githubIssuePathParts(path);
11312
+ if (!issueParts || `${issueParts.owner}/${issueParts.repo}`.toLowerCase() !== eventRepoKey) {
11313
+ continue;
11314
+ }
11315
+ }
10974
11316
  const issue = await this.#readIssue(path);
10975
11317
  if (!issue || (githubSource
10976
11318
  ? issue.state?.name?.trim().toLowerCase() === 'closed'
@@ -10983,10 +11325,27 @@ export class FactoryLoop {
10983
11325
  const score = prSnapshotIssueMatchScore(snapshot, issue.key);
10984
11326
  if (score > 0 && (!best || score > best.score)) {
10985
11327
  best = { issue, score };
11328
+ ambiguous = false;
11329
+ }
11330
+ else if (score > 0 && best && score === best.score) {
11331
+ ambiguous = true;
10986
11332
  }
10987
11333
  }
11334
+ // Fail-closed on a tie. A silent first-wins on unrelated same-score
11335
+ // candidates is the exact class of quiet corruption we are removing.
11336
+ if (ambiguous) {
11337
+ this.#increment('mergedPrAdvanceAmbiguous');
11338
+ this.#logger.warn?.('[factory] merge advance skipped: multiple candidates tied on match score', {
11339
+ prNumber: snapshot.number,
11340
+ eventRepo,
11341
+ matchedIssue: best?.issue.key,
11342
+ matchScore: best?.score,
11343
+ });
11344
+ return undefined;
11345
+ }
10988
11346
  this.#logger.debug?.(`[factory] scanned ${githubSource ? 'GitHub' : 'Linear'} issues for merged PR advance`, {
10989
11347
  prNumber: snapshot.number,
11348
+ eventRepo,
10990
11349
  durationMs: this.#clock.now() - scanStartedAtMs,
10991
11350
  matchedIssue: best?.issue.key,
10992
11351
  matchScore: best?.score,
@@ -11592,7 +11951,7 @@ export class FactoryLoop {
11592
11951
  }
11593
11952
  if (!await this.#saveDispatchLifecycle(record, 'releasing', undefined, releaseReason))
11594
11953
  return;
11595
- await this.#stopSlackWatcher(record.issue);
11954
+ await this.#retireSlackWatcher(record);
11596
11955
  await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
11597
11956
  await this.#recordDispatchTerminal(record.issue);
11598
11957
  await this.#finishDurableRelease(record, releaseReason);
@@ -12070,6 +12429,21 @@ export class FactoryLoop {
12070
12429
  return;
12071
12430
  }
12072
12431
  const key = issueKey(record.issue);
12432
+ const previousWatch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
12433
+ .find(([watchKey]) => watchKey === key)?.[1];
12434
+ if (previousWatch?.kind === 'terminal-grace') {
12435
+ // A reopened work unit needs a fresh dispatch notification and a fresh
12436
+ // conversation. Do not let the old grace-period watcher (or its expiry
12437
+ // timer) capture and later tear down the new dispatch.
12438
+ if (!await this.#stopSlackWatcher(record.issue)) {
12439
+ // Fail closed. An undrained reply route still holds the retired thread
12440
+ // and would bind it to this dispatch, delivering a stale human reply to
12441
+ // fresh work. Leave the fence up; the next reconcile retries the drain.
12442
+ this.#logger.warn?.('[factory] deferring Slack dispatch thread for reopened work unit; in-flight reply route not drained', { issue: record.issue.key });
12443
+ this.#increment('slackDispatchThreadsDeferredUndrainedReply');
12444
+ return;
12445
+ }
12446
+ }
12073
12447
  const existingThread = await this.#persistedSlackThread(key);
12074
12448
  const watcherStart = this.#slackWatcherStarts.get(key);
12075
12449
  if (existingThread || watcherStart) {
@@ -12152,11 +12526,13 @@ export class FactoryLoop {
12152
12526
  if (existing) {
12153
12527
  const sessionRef = owned?.tracked.sessionRef;
12154
12528
  const agentName = owned ? (owned.tracked.result?.name ?? owned.name) : undefined;
12529
+ let rebound = false;
12155
12530
  if (owned && sessionRef &&
12156
- (agentName !== existing.agent.name || (options.forceAgentRebind === true && sessionRef !== existing.agent.sessionRef))) {
12157
- const rebound = await this.#state.rebindConversationSession(this.#workspaceId, conversationId, {
12531
+ (!existing.agent || agentName !== existing.agent.name || (options.forceAgentRebind === true && sessionRef !== existing.agent.sessionRef))) {
12532
+ rebound = await this.#state.rebindConversationSession(this.#workspaceId, conversationId, {
12158
12533
  name: agentName,
12159
12534
  sessionRef,
12535
+ role: owned.tracked.spec.role,
12160
12536
  node: owned.tracked.result?.node ?? owned.tracked.spec.node,
12161
12537
  capability: owned.tracked.spec.capability,
12162
12538
  repo: owned.tracked.spec.repo,
@@ -12167,39 +12543,39 @@ export class FactoryLoop {
12167
12543
  }
12168
12544
  if (existing.pending.length > 0 || existing.delivery) {
12169
12545
  const waiting = await this.#state.getWaitingClarification(this.#workspaceId, issueKey(existing.issue));
12170
- if (!waiting)
12546
+ if (!waiting && (existing.agent || rebound))
12171
12547
  this.#slackConversationTurns.schedule(conversationId);
12172
12548
  }
12173
12549
  return;
12174
12550
  }
12175
12551
  const sessionRef = owned?.tracked.sessionRef;
12176
- if (!owned || !sessionRef) {
12177
- this.#increment('slackConversationSessionsSkippedMissingSession');
12178
- return;
12179
- }
12180
12552
  const channelDir = await this.#slackChannelDir() ?? this.#config.slack?.channel;
12181
12553
  if (!channelDir)
12182
12554
  return;
12183
- const agentName = owned.tracked.result?.name ?? owned.name;
12555
+ const agentName = owned ? (owned.tracked.result?.name ?? owned.name) : undefined;
12184
12556
  const reserved = await this.#state.reserveConversationSession(this.#workspaceId, conversationId, {
12185
12557
  provider: 'slack',
12186
12558
  issue: { ...record.issue },
12187
12559
  externalId: threadId,
12188
12560
  context: { channelDir },
12189
- agent: {
12190
- name: agentName,
12191
- sessionRef,
12192
- node: owned.tracked.result?.node ?? owned.tracked.spec.node,
12193
- capability: owned.tracked.spec.capability,
12194
- repo: owned.tracked.spec.repo,
12195
- clonePath: owned.tracked.spec.clonePath,
12196
- },
12561
+ ...(owned && sessionRef && agentName ? { agent: {
12562
+ name: agentName,
12563
+ sessionRef,
12564
+ role: owned.tracked.spec.role,
12565
+ node: owned.tracked.result?.node ?? owned.tracked.spec.node,
12566
+ capability: owned.tracked.spec.capability,
12567
+ repo: owned.tracked.spec.repo,
12568
+ clonePath: owned.tracked.spec.clonePath,
12569
+ } } : {}),
12197
12570
  history: [],
12198
12571
  processedMessageIds: [],
12572
+ acknowledgedMessageIds: [],
12573
+ acknowledgementClaims: {},
12199
12574
  pending: [],
12200
12575
  });
12201
- if (reserved)
12202
- this.#increment('slackConversationSessionsOwned');
12576
+ if (reserved) {
12577
+ this.#increment(owned && sessionRef ? 'slackConversationSessionsOwned' : 'slackConversationSessionsReservedUnowned');
12578
+ }
12203
12579
  }
12204
12580
  // Called right after a babysitter is spawned/reattached for an issue's PR so
12205
12581
  // an already-owned Slack conversation session (reserved earlier by the
@@ -12227,11 +12603,15 @@ export class FactoryLoop {
12227
12603
  const claimed = await this.#state.claimConversationTurn(this.#workspaceId, conversationId, this.#slackConversationOwner, claimId, this.#clock.now(), SLACK_CONVERSATION_TURN_LEASE_MS);
12228
12604
  if (!claimed?.delivery) {
12229
12605
  const current = await this.#state.getConversationSession(this.#workspaceId, conversationId);
12230
- if (current && (current.pending.length > 0 || current.delivery)) {
12606
+ if (current?.agent && (current.pending.length > 0 || current.delivery)) {
12231
12607
  this.#slackConversationTurns.schedule(conversationId, SLACK_CONVERSATION_TURN_RETRY_MS);
12232
12608
  }
12233
12609
  return;
12234
12610
  }
12611
+ if (!claimed.agent) {
12612
+ await this.#state.releaseConversationTurn(this.#workspaceId, conversationId, this.#slackConversationOwner, claimId);
12613
+ return;
12614
+ }
12235
12615
  if (!await this.#ownsActiveSlackConversationIssue(claimed.issue)) {
12236
12616
  await this.#state.releaseConversationTurn(this.#workspaceId, conversationId, this.#slackConversationOwner, claimId);
12237
12617
  this.#increment('slackConversationTurnsSuppressedStaleOwner');
@@ -12252,9 +12632,20 @@ export class FactoryLoop {
12252
12632
  }, Math.max(1_000, Math.floor(SLACK_CONVERSATION_TURN_LEASE_MS / 3)));
12253
12633
  heartbeat.unref?.();
12254
12634
  try {
12635
+ // Sessions bound after this field was introduced carry their role
12636
+ // durably, so resume never depends on a live lookup by the agent's
12637
+ // current name — which a since-completed rename (e.g. babysitter
12638
+ // retarget) can otherwise leave unresolvable forever. Older
12639
+ // already-persisted sessions fall back to the live lookup.
12640
+ const conversationRole = claimed.agent.role
12641
+ ?? (await this.#batch()).getIssueByAgent(claimed.agent.name)?.agents.get(claimed.agent.name)?.spec.role;
12642
+ if (!conversationRole) {
12643
+ throw new Error(`Cannot resume ${claimed.agent.name}: its dispatch role is unavailable for identity proof`);
12644
+ }
12255
12645
  const result = await this.#fleet.resume({
12256
12646
  name: claimed.agent.name,
12257
12647
  sessionRef: claimed.agent.sessionRef,
12648
+ identityKey: dispatchAgentIdentityKey(claimed.issue, conversationRole),
12258
12649
  node: claimed.agent.node ?? 'self',
12259
12650
  capability: claimed.agent.capability,
12260
12651
  repo: claimed.agent.repo,
@@ -12304,10 +12695,13 @@ export class FactoryLoop {
12304
12695
  }
12305
12696
  }
12306
12697
  async #recordSlackConversationResume(session, result) {
12698
+ const sessionAgent = session.agent;
12699
+ if (!sessionAgent)
12700
+ return;
12307
12701
  const record = (await this.#batch()).getIssue(session.issue);
12308
12702
  if (!record)
12309
12703
  return;
12310
- const entry = [...record.agents.entries()].find(([name, tracked]) => name === session.agent.name || tracked.result?.name === session.agent.name);
12704
+ const entry = [...record.agents.entries()].find(([name, tracked]) => name === sessionAgent.name || tracked.result?.name === sessionAgent.name);
12311
12705
  if (!entry)
12312
12706
  return;
12313
12707
  const [previousName, tracked] = entry;
@@ -12633,7 +13027,14 @@ export class FactoryLoop {
12633
13027
  `Question: ${triageEscalationQuestion(decision, issue)}`,
12634
13028
  ].join('\n'),
12635
13029
  });
12636
- await this.#state.setSlackThread(this.#workspaceId, issueKey(decision.issue), root.threadId);
13030
+ const key = issueKey(decision.issue);
13031
+ await this.#state.setSlackThread(this.#workspaceId, key, root.threadId);
13032
+ await this.#state.setSlackThreadWatch(this.#workspaceId, key, {
13033
+ kind: 'triage',
13034
+ issue: { ...decision.issue },
13035
+ decision: structuredClone(decision),
13036
+ threadId: root.threadId,
13037
+ });
12637
13038
  const replayedResult = await this.#watchSlackThread(escalationWatchRecord(decision), root.threadId);
12638
13039
  this.#recordSlackWritebackSuccess('triage-escalation');
12639
13040
  return replayedResult;
@@ -12699,6 +13100,11 @@ export class FactoryLoop {
12699
13100
  if (!reply || !reply.isThreadReply || reply.threadTs !== threadId || reply.channelDir !== channelDir) {
12700
13101
  return;
12701
13102
  }
13103
+ if (allowPreExisting &&
13104
+ options.replayAfterMs !== undefined &&
13105
+ slackMessageReceivedAtMs(reply.messageTs, Number.MAX_SAFE_INTEGER) < options.replayAfterMs) {
13106
+ return;
13107
+ }
12702
13108
  const replyMessageKey = `${reply.threadTs}:${reply.messageTs}`;
12703
13109
  if (seenReplyMessages.has(replyMessageKey)) {
12704
13110
  this.#logger.debug?.('[factory] suppressed duplicate Slack reply message', { issue: record.issue.key, path });
@@ -12877,6 +13283,47 @@ export class FactoryLoop {
12877
13283
  this.#slackConversationTurns.schedule(conversationId);
12878
13284
  }
12879
13285
  }
13286
+ for (const [key, watch] of await this.#state.listSlackThreadWatches(this.#workspaceId)) {
13287
+ if (this.#slackWatchers.has(key) || this.#slackWatcherStarts.has(key))
13288
+ continue;
13289
+ if (watch.kind === 'terminal-grace' && watch.expiresAtMs <= this.#clock.now()) {
13290
+ await this.#stopSlackWatcher(watch.issue);
13291
+ continue;
13292
+ }
13293
+ await this.#state.setSlackThread(this.#workspaceId, key, watch.threadId);
13294
+ const watchRecord = escalationWatchRecord(watch.decision);
13295
+ if (watch.kind === 'terminal-grace') {
13296
+ const retiredAtMs = terminalSlackWatchRetiredAtMs(watch);
13297
+ if (watch.retiredAtMs !== retiredAtMs) {
13298
+ await this.#state.setSlackThreadWatch(this.#workspaceId, key, { ...watch, retiredAtMs });
13299
+ }
13300
+ this.#terminalSlackWatchIssues.add(key);
13301
+ const conversationId = slackConversationId(watch.threadId);
13302
+ await this.#slackConversationTurns.cancel(conversationId);
13303
+ try {
13304
+ await this.#surfaceUndeliveredSlackConversation(watch.threadId);
13305
+ await this.#state.clearConversationSession(this.#workspaceId, conversationId);
13306
+ }
13307
+ catch (error) {
13308
+ // The undelivered-reply receipt needs Slack writeback, which may be
13309
+ // unavailable at startup. That is retryable state maintenance for this
13310
+ // one thread, not a reason to abandon rehydration: aborting here would
13311
+ // leave every remaining thread watched by nobody. Keep the queued
13312
+ // replies (clearing them now would drop replies nobody was told about)
13313
+ // and carry on re-arming.
13314
+ this.#logger.warn?.('[factory] failed to settle undelivered Slack replies for terminal watch; will retry', { issue: watch.issue.key, error });
13315
+ this.#increment('slackTerminalWatchReceiptsDeferred');
13316
+ this.#scheduleSlackTerminalReceiptRetry(watch.issue, watch.threadId, watch.expiresAtMs);
13317
+ }
13318
+ await this.#rearmSlackWatcher(watchRecord, watch.threadId, {
13319
+ replayConversationReplies: true,
13320
+ replayAfterMs: retiredAtMs,
13321
+ });
13322
+ this.#scheduleSlackTerminalWatchExpiry(watch.issue, watch.expiresAtMs);
13323
+ continue;
13324
+ }
13325
+ await this.#rearmSlackWatcher(watchRecord, watch.threadId, { replayConversationReplies: true });
13326
+ }
12880
13327
  await this.#sweepWaitingClarifications();
12881
13328
  for (const [, waiting] of await this.#state.listWaitingClarifications(this.#workspaceId)) {
12882
13329
  if (!waiting.threadId)
@@ -13022,8 +13469,73 @@ export class FactoryLoop {
13022
13469
  this.#clarificationSweepTimer = timer;
13023
13470
  this.#clarificationSweepDueAtMs = dueAtMs;
13024
13471
  }
13472
+ // The terminal fence is the only thing that makes an in-flight reply route
13473
+ // answer "no active agent" instead of binding the retired thread to whatever
13474
+ // dispatch owns this key. Routes are chained per work unit, so awaiting the
13475
+ // newest one drains every reply queued behind it.
13476
+ async #drainSlackReplyRoutes(key) {
13477
+ // Snapshotting #slackReplyRoutes is not enough on its own. A reply handler
13478
+ // that is still inside its mount read when the drain starts registers its
13479
+ // route *after* the snapshot, so it would run once the fence is gone and
13480
+ // bind the retired thread to the next dispatch — the same escape one level
13481
+ // in. Bar *routing* for this key first (the bar and the registration are
13482
+ // both synchronous, so nothing can slip between them), then drain whatever
13483
+ // is already chained, then prove the set is empty before reporting success.
13484
+ // A barred reply still answers the human, and that writeback registers here
13485
+ // like any other effect, so the extra passes are what pick it up: quiescence
13486
+ // means every effect this work unit started has settled, not merely the ones
13487
+ // that existed when the drain began.
13488
+ const nested = this.#slackReplyRouteDrains.has(key);
13489
+ this.#slackReplyRouteDrains.add(key);
13490
+ try {
13491
+ for (let pass = 0; pass < SLACK_REPLY_ROUTE_DRAIN_PASSES; pass += 1) {
13492
+ const route = this.#slackReplyRoutes.get(key);
13493
+ if (!route)
13494
+ return true;
13495
+ try {
13496
+ await route;
13497
+ }
13498
+ catch (error) {
13499
+ // A route that *rejects* is not drained: the watcher replays it after
13500
+ // SLACK_REPLY_ROUTE_RETRY_MS, and that replay would land on the next
13501
+ // dispatch. Fail closed and let the caller keep the fence up rather
13502
+ // than leak a stale human reply onto fresh work.
13503
+ this.#logger.warn?.('[factory] in-flight Slack reply route did not drain; keeping terminal Slack fence', { issue: key, error });
13504
+ this.#increment('slackReplyRouteDrainsFailed');
13505
+ return false;
13506
+ }
13507
+ // The owner clears its own entry when it settles; retiring it here too
13508
+ // keeps the loop monotonic if that finally has not run yet.
13509
+ if (this.#slackReplyRoutes.get(key) === route)
13510
+ this.#slackReplyRoutes.delete(key);
13511
+ }
13512
+ // Not provably quiescent. Fail closed for the same reason as a rejection.
13513
+ this.#logger.warn?.('[factory] Slack reply routes did not quiesce; keeping terminal Slack fence', { issue: key });
13514
+ this.#increment('slackReplyRouteDrainsFailed');
13515
+ return false;
13516
+ }
13517
+ finally {
13518
+ if (!nested)
13519
+ this.#slackReplyRouteDrains.delete(key);
13520
+ }
13521
+ }
13025
13522
  async #stopSlackWatcher(issue) {
13026
13523
  const key = issueKey(issue);
13524
+ // Drain before clearing the fence. Clearing it first lets a reply that is
13525
+ // already mid-route — or one queued behind it — fall through the fence check
13526
+ // in #routeSlackConversationAnswerUnlocked and rebind the retired thread to
13527
+ // the next dispatch of this work unit.
13528
+ if (!await this.#drainSlackReplyRoutes(key))
13529
+ return false;
13530
+ this.#terminalSlackWatchIssues.delete(key);
13531
+ const expiryTimer = this.#slackTerminalWatchExpiryTimers.get(key);
13532
+ if (expiryTimer)
13533
+ clearTimeout(expiryTimer);
13534
+ this.#slackTerminalWatchExpiryTimers.delete(key);
13535
+ const receiptRetryTimer = this.#slackTerminalReceiptRetryTimers.get(key);
13536
+ if (receiptRetryTimer)
13537
+ clearTimeout(receiptRetryTimer);
13538
+ this.#slackTerminalReceiptRetryTimers.delete(key);
13027
13539
  const watcher = this.#slackWatchers.get(key);
13028
13540
  this.#slackWatchers.delete(key);
13029
13541
  const threadId = await this.#state.getSlackThread(this.#workspaceId, key);
@@ -13034,6 +13546,259 @@ export class FactoryLoop {
13034
13546
  await this.#state.clearConversationSession(this.#workspaceId, conversationId);
13035
13547
  }
13036
13548
  await this.#state.clearSlackThread(this.#workspaceId, key);
13549
+ await this.#state.clearSlackThreadWatch(this.#workspaceId, key);
13550
+ return true;
13551
+ }
13552
+ async #retireSlackWatcher(record) {
13553
+ const key = issueKey(record.issue);
13554
+ const threadId = await this.#state.getSlackThread(this.#workspaceId, key);
13555
+ if (!threadId) {
13556
+ await this.#stopSlackWatcher(record.issue);
13557
+ return;
13558
+ }
13559
+ const existingWatch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
13560
+ .find(([watchKey]) => watchKey === key)?.[1];
13561
+ const retiredAtMs = existingWatch?.kind === 'terminal-grace'
13562
+ ? terminalSlackWatchRetiredAtMs(existingWatch)
13563
+ : this.#clock.now();
13564
+ const expiresAtMs = existingWatch?.kind === 'terminal-grace'
13565
+ ? existingWatch.expiresAtMs
13566
+ : retiredAtMs + SLACK_TERMINAL_THREAD_GRACE_MS;
13567
+ await this.#state.setSlackThreadWatch(this.#workspaceId, key, {
13568
+ kind: 'terminal-grace',
13569
+ issue: { ...record.issue },
13570
+ decision: structuredClone(record.decision),
13571
+ threadId,
13572
+ retiredAtMs,
13573
+ expiresAtMs,
13574
+ });
13575
+ // A terminal thread must never retain a resumable session for an agent that
13576
+ // has already exited. Keep only the exact-thread listener so a late human
13577
+ // reply receives the explicit no-active-agent writeback below.
13578
+ this.#terminalSlackWatchIssues.add(key);
13579
+ await this.#slackReplyRoutes.get(key)?.catch(() => undefined);
13580
+ const conversationId = slackConversationId(threadId);
13581
+ await this.#slackConversationTurns.cancel(conversationId);
13582
+ try {
13583
+ await this.#surfaceUndeliveredSlackConversation(threadId);
13584
+ await this.#state.clearConversationSession(this.#workspaceId, conversationId);
13585
+ }
13586
+ catch (error) {
13587
+ // The receipt fails whenever another handler holds the claim or Slack
13588
+ // writeback is down — neither is a reason to abort retirement. Callers
13589
+ // reach here having already committed the terminal phase and dropped the
13590
+ // pending abandon reason, so a rejection escaping would strand the
13591
+ // registry rewrite, the GitHub watcher stop, and the queued next dispatch
13592
+ // with nothing left to re-run them. Keep the queued replies and let the
13593
+ // retry that owns this receipt settle it inside the grace window.
13594
+ this.#logger.warn?.('[factory] failed to settle undelivered Slack replies while retiring the watcher; will retry', { issue: record.issue.key, error });
13595
+ this.#increment('slackTerminalWatchReceiptsDeferred');
13596
+ this.#scheduleSlackTerminalReceiptRetry(record.issue, threadId, expiresAtMs);
13597
+ }
13598
+ if (!this.#slackWatchers.has(key) && !this.#stopping) {
13599
+ await this.#rearmSlackWatcher(record, threadId);
13600
+ }
13601
+ this.#scheduleSlackTerminalWatchExpiry(record.issue, expiresAtMs);
13602
+ this.#increment('slackTerminalWatchersRetained');
13603
+ }
13604
+ // The durable half of the same record. Every caller here follows the receipt
13605
+ // with a state write (clearing the session), and those two cannot be one
13606
+ // durable step: when the state write fails, the retry that owns it must not
13607
+ // read "replies still queued" as "the human has not been told" and post the
13608
+ // notice again. So the receipt is claimed before the provider write and marked
13609
+ // posted after it, and a retry finds it already settled.
13610
+ async #surfaceUndeliveredSlackConversation(threadId) {
13611
+ const conversationId = slackConversationId(threadId);
13612
+ const session = await this.#state.getConversationSession(this.#workspaceId, conversationId);
13613
+ const pendingCount = session
13614
+ ? session.pending.length + (session.delivery?.messages.length ?? 0)
13615
+ : 0;
13616
+ if (pendingCount === 0)
13617
+ return;
13618
+ if (session?.terminalReceipt?.posted) {
13619
+ this.#increment('slackTerminalReceiptsAlreadySettled');
13620
+ return;
13621
+ }
13622
+ if (!this.#slack)
13623
+ throw new Error(`Slack thread ${threadId} cannot surface undelivered replies without writeback`);
13624
+ const claimId = randomUUID();
13625
+ if (!await this.#state.claimConversationTerminalReceipt(this.#workspaceId, conversationId, claimId, this.#clock.now(), SLACK_TERMINAL_RECEIPT_CLAIM_LEASE_MS)) {
13626
+ const current = await this.#state.getConversationSession(this.#workspaceId, conversationId);
13627
+ if (current?.terminalReceipt?.posted) {
13628
+ this.#increment('slackTerminalReceiptsAlreadySettled');
13629
+ return;
13630
+ }
13631
+ // Another handler is mid-write. Fail closed so the queued replies survive
13632
+ // for whoever settles them rather than racing a second notice onto the
13633
+ // same thread.
13634
+ throw new Error(`Slack thread ${threadId} terminal receipt is claimed by another handler; retrying`);
13635
+ }
13636
+ const noun = pendingCount === 1 ? 'reply' : 'replies';
13637
+ const slack = this.#slack;
13638
+ try {
13639
+ await this.#withRenewedProviderLease('terminal Slack receipt', SLACK_TERMINAL_RECEIPT_CLAIM_LEASE_MS, () => this.#state.renewConversationTerminalReceipt(this.#workspaceId, conversationId, claimId, this.#clock.now()), () => slack.reply(threadId, `Factory could not deliver ${pendingCount} queued ${noun} because this work unit no longer has an active agent. Please continue on the linked issue or pull request.`));
13640
+ }
13641
+ catch (error) {
13642
+ await this.#state.releaseConversationTerminalReceipt(this.#workspaceId, conversationId, claimId);
13643
+ throw error;
13644
+ }
13645
+ if (!await this.#state.completeConversationTerminalReceipt(this.#workspaceId, conversationId, claimId)) {
13646
+ throw new Error(`Slack thread ${threadId} terminal receipt could not be recorded`);
13647
+ }
13648
+ this.#increment('slackConversationRepliesSurfacedTerminal');
13649
+ }
13650
+ // A claim only means something for as long as it outlives the work it covers.
13651
+ // A provider write can legitimately run past a fixed lease, at which point the
13652
+ // claim stops protecting the write it was taken for and a second handler can
13653
+ // post the same thing to the same human. Renewing on a heartbeat scopes the
13654
+ // lease to the work instead of to a guessed duration, and leaves the idle
13655
+ // timeout short enough that a holder that dies mid-write still frees it.
13656
+ //
13657
+ // The heartbeat is bounded on both ends, because a renewal loop that never
13658
+ // stops is a lock with no owner check: a provider write that hangs would hold
13659
+ // the receipt past every retry and past shutdown, and the human whose reply is
13660
+ // queued behind it would be told nothing until the process restarts. So it
13661
+ // stops at the ceiling and it stops when this daemon is stopping, and either
13662
+ // way it says so — from there the claim ages out on its own idle lease and
13663
+ // becomes reclaimable. The write may still land afterwards and duplicate the
13664
+ // notice; a reply nobody can ever reclaim is the worse of the two.
13665
+ async #withRenewedProviderLease(label, leaseMs, renew, run) {
13666
+ const renewUntilMs = this.#clock.now() + SLACK_PROVIDER_LEASE_MAX_RENEWAL_MS;
13667
+ let renewalStopped = false;
13668
+ let renewalInFlight = false;
13669
+ const stopRenewing = (counter, reason) => {
13670
+ renewalStopped = true;
13671
+ this.#increment(counter);
13672
+ this.#logger.warn?.(`[factory] ${label} lease will not be renewed further (${reason}); ` +
13673
+ 'its claim expires and the queued replies return to whoever retries them');
13674
+ };
13675
+ const heartbeat = setInterval(() => {
13676
+ if (renewalInFlight || renewalStopped)
13677
+ return;
13678
+ if (this.#stopping) {
13679
+ stopRenewing('slackProviderReceiptLeaseRenewalsStoppedForShutdown', 'shutting down');
13680
+ return;
13681
+ }
13682
+ if (this.#clock.now() >= renewUntilMs) {
13683
+ stopRenewing('slackProviderReceiptLeaseRenewalsExpired', `provider write exceeded ${SLACK_PROVIDER_LEASE_MAX_RENEWAL_MS}ms`);
13684
+ return;
13685
+ }
13686
+ renewalInFlight = true;
13687
+ void renew()
13688
+ .then((renewed) => {
13689
+ if (renewed)
13690
+ return;
13691
+ // Losing the lease mid-write is not recoverable from in here: the
13692
+ // write may already have landed. Stop renewing and let the caller's
13693
+ // completion check fail closed, which keeps the queued replies for
13694
+ // whoever holds the claim now.
13695
+ renewalStopped = true;
13696
+ this.#increment('slackProviderReceiptLeasesLost');
13697
+ this.#logger.warn?.(`[factory] ${label} lease was lost while its provider write was in flight`);
13698
+ })
13699
+ .catch((error) => this.#logger.warn?.(`[factory] ${label} lease renewal failed`, {
13700
+ error: describeError(error).errorMessage,
13701
+ }))
13702
+ .finally(() => { renewalInFlight = false; });
13703
+ }, Math.max(1_000, Math.floor(leaseMs / 3)));
13704
+ heartbeat.unref?.();
13705
+ try {
13706
+ return await run();
13707
+ }
13708
+ finally {
13709
+ clearInterval(heartbeat);
13710
+ }
13711
+ }
13712
+ // A terminal receipt that could not be written leaves the queued replies
13713
+ // pending with the human who wrote them told nothing. That is retryable
13714
+ // maintenance this daemon owns, not work to leave for the next restart: the
13715
+ // grace watch is the only window in which the receipt can still land on the
13716
+ // retired thread, so keep reattempting inside it and give up when it closes.
13717
+ #scheduleSlackTerminalReceiptRetry(issue, threadId, expiresAtMs, attempt = 0) {
13718
+ if (this.#stopping)
13719
+ return;
13720
+ const key = issueKey(issue);
13721
+ const existing = this.#slackTerminalReceiptRetryTimers.get(key);
13722
+ if (existing)
13723
+ clearTimeout(existing);
13724
+ this.#slackTerminalReceiptRetryTimers.delete(key);
13725
+ const remainingMs = expiresAtMs - this.#clock.now();
13726
+ if (remainingMs <= 0) {
13727
+ this.#increment('slackTerminalWatchReceiptsAbandoned');
13728
+ return;
13729
+ }
13730
+ const backoffMs = Math.min(SLACK_TERMINAL_RECEIPT_RETRY_MAX_MS, SLACK_TERMINAL_RECEIPT_RETRY_MS * 2 ** Math.min(attempt, 16));
13731
+ const timer = setTimeout(() => {
13732
+ this.#slackTerminalReceiptRetryTimers.delete(key);
13733
+ void this.#retrySlackTerminalReceipt(issue, threadId, attempt);
13734
+ }, Math.max(0, Math.min(backoffMs, remainingMs)));
13735
+ timer.unref?.();
13736
+ this.#slackTerminalReceiptRetryTimers.set(key, timer);
13737
+ }
13738
+ async #retrySlackTerminalReceipt(issue, threadId, attempt) {
13739
+ if (this.#stopping)
13740
+ return;
13741
+ const key = issueKey(issue);
13742
+ const watch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
13743
+ .find(([watchKey]) => watchKey === key)?.[1];
13744
+ // The grace watch is gone (expired, or the work unit reopened): the thread
13745
+ // this receipt would settle no longer exists, so there is nothing to say.
13746
+ if (watch?.kind !== 'terminal-grace' || watch.threadId !== threadId)
13747
+ return;
13748
+ try {
13749
+ await this.#surfaceUndeliveredSlackConversation(threadId);
13750
+ await this.#state.clearConversationSession(this.#workspaceId, slackConversationId(threadId));
13751
+ this.#increment('slackTerminalWatchReceiptsRecovered');
13752
+ }
13753
+ catch (error) {
13754
+ this.#logger.warn?.('[factory] terminal Slack receipt retry failed; rescheduling', {
13755
+ issue: issue.key,
13756
+ error,
13757
+ });
13758
+ this.#increment('slackTerminalWatchReceiptRetryFailures');
13759
+ this.#scheduleSlackTerminalReceiptRetry(issue, threadId, watch.expiresAtMs, attempt + 1);
13760
+ }
13761
+ }
13762
+ #scheduleSlackTerminalWatchExpiry(issue, expiresAtMs, retryDelayMs) {
13763
+ if (this.#stopping)
13764
+ return;
13765
+ const key = issueKey(issue);
13766
+ const existing = this.#slackTerminalWatchExpiryTimers.get(key);
13767
+ if (existing)
13768
+ clearTimeout(existing);
13769
+ const timer = setTimeout(() => {
13770
+ this.#slackTerminalWatchExpiryTimers.delete(key);
13771
+ void this.#expireSlackTerminalWatcher(issue, expiresAtMs).catch((error) => {
13772
+ this.#logger.warn?.('[factory] failed to expire terminal Slack reply watcher; retrying', {
13773
+ issue: issue.key,
13774
+ error,
13775
+ });
13776
+ this.#scheduleSlackTerminalWatchExpiry(issue, expiresAtMs, SLACK_REPLY_ROUTE_RETRY_MS);
13777
+ });
13778
+ }, retryDelayMs ?? Math.max(0, expiresAtMs - this.#clock.now()));
13779
+ timer.unref?.();
13780
+ this.#slackTerminalWatchExpiryTimers.set(key, timer);
13781
+ }
13782
+ async #expireSlackTerminalWatcher(issue, expiresAtMs) {
13783
+ const key = issueKey(issue);
13784
+ const watch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
13785
+ .find(([watchKey]) => watchKey === key)?.[1];
13786
+ if (watch?.kind !== 'terminal-grace' || watch.expiresAtMs !== expiresAtMs)
13787
+ return;
13788
+ if (watch.expiresAtMs > this.#clock.now()) {
13789
+ this.#scheduleSlackTerminalWatchExpiry(issue, watch.expiresAtMs);
13790
+ return;
13791
+ }
13792
+ // #stopSlackWatcher fails closed when an in-flight reply route will not
13793
+ // drain, leaving the watch and its terminal fence in place. Counting that as
13794
+ // an expiration retires the watch in the metrics while the real one lives
13795
+ // on unwatched by any expiry timer, so reschedule and count only on success.
13796
+ if (!await this.#stopSlackWatcher(issue)) {
13797
+ this.#increment('slackTerminalWatchExpiriesDeferred');
13798
+ this.#scheduleSlackTerminalWatchExpiry(issue, expiresAtMs, SLACK_REPLY_ROUTE_RETRY_MS);
13799
+ return;
13800
+ }
13801
+ this.#increment('slackTerminalWatchersExpired');
13037
13802
  }
13038
13803
  async #readSlackReply(path) {
13039
13804
  try {
@@ -13098,33 +13863,147 @@ export class FactoryLoop {
13098
13863
  await this.#wakeWaitingClarification(clarificationKey, claimed);
13099
13864
  return;
13100
13865
  }
13866
+ return await this.#routeSlackConversationAnswer(record, reply, text, clarificationKey);
13867
+ }
13868
+ async #routeSlackConversationAnswer(record, reply, text, clarificationKey) {
13869
+ if (this.#slackReplyRouteDrains.has(clarificationKey)) {
13870
+ // The terminal fence for this work unit is being drained right now.
13871
+ // Registering an ordinary route here would put it past the drain's
13872
+ // snapshot and run it once the fence is gone. Answer it the way the fence
13873
+ // would have — but as a tracked effect, because this writeback is still a
13874
+ // side effect of this work unit. Left untracked it is the same escape one
13875
+ // level further in: the drain reports quiescence without it, the watcher
13876
+ // stop clears the retry timer that owns this reply, and a slow or failed
13877
+ // receipt leaves the human told nothing at all.
13878
+ this.#increment('slackReplyRoutesFencedDuringDrain');
13879
+ return await this.#trackSlackWorkUnitEffect(clarificationKey, async () => {
13880
+ await this.#writeUnroutableSlackReply(reply.threadTs);
13881
+ return undefined;
13882
+ });
13883
+ }
13884
+ return await this.#trackSlackWorkUnitEffect(clarificationKey, () => this.#routeSlackConversationAnswerUnlocked(record, reply, text, clarificationKey));
13885
+ }
13886
+ // Every Slack side effect a work unit makes on its own behalf runs through
13887
+ // here, so #slackReplyRoutes stays the single record the terminal drain
13888
+ // consults. Effects are chained per work unit: awaiting the newest one drains
13889
+ // everything queued behind it, and a rejection propagates to the drain, which
13890
+ // fails closed rather than tearing the effect's retry path down.
13891
+ async #trackSlackWorkUnitEffect(key, run) {
13892
+ const preceding = this.#slackReplyRoutes.get(key);
13893
+ const effect = (async () => {
13894
+ await preceding?.catch(() => undefined);
13895
+ return await run();
13896
+ })();
13897
+ this.#slackReplyRoutes.set(key, effect);
13898
+ try {
13899
+ return await effect;
13900
+ }
13901
+ finally {
13902
+ if (this.#slackReplyRoutes.get(key) === effect)
13903
+ this.#slackReplyRoutes.delete(key);
13904
+ }
13905
+ }
13906
+ async #routeSlackConversationAnswerUnlocked(record, reply, text, clarificationKey) {
13907
+ if (this.#terminalSlackWatchIssues.has(clarificationKey)) {
13908
+ await this.#writeUnroutableSlackReply(reply.threadTs);
13909
+ return;
13910
+ }
13101
13911
  const conversationId = slackConversationId(reply.threadTs);
13102
- const conversation = await this.#state.getConversationSession(this.#workspaceId, conversationId);
13912
+ let conversation = await this.#state.getConversationSession(this.#workspaceId, conversationId);
13913
+ let liveRecord;
13914
+ if (!conversation) {
13915
+ liveRecord = (await this.#batch()).getIssue(record.issue);
13916
+ if (liveRecord && !liveRecord.dryRun) {
13917
+ await this.#ensureSlackConversationSession(liveRecord, reply.threadTs);
13918
+ conversation = await this.#state.getConversationSession(this.#workspaceId, conversationId);
13919
+ }
13920
+ }
13103
13921
  if (conversation && issueKey(conversation.issue) === clarificationKey) {
13922
+ const replyId = `${reply.threadTs}:${reply.messageTs}`;
13104
13923
  const queued = await this.#state.appendConversationMessage(this.#workspaceId, conversationId, {
13105
- id: `${reply.threadTs}:${reply.messageTs}`,
13924
+ id: replyId,
13106
13925
  text,
13107
13926
  receivedAtMs: slackMessageReceivedAtMs(reply.messageTs, this.#clock.now()),
13108
13927
  providerSequence: reply.messageTs,
13109
13928
  author: reply.author,
13110
13929
  });
13111
- if (!queued) {
13930
+ const durable = queued ?? await this.#state.getConversationSession(this.#workspaceId, conversationId);
13931
+ if (!durable || !durable.processedMessageIds.includes(replyId)) {
13932
+ throw new Error(`Slack reply ${replyId} was not durably queued`);
13933
+ }
13934
+ if (!(durable.acknowledgedMessageIds ?? []).includes(replyId)) {
13935
+ const acknowledgementClaimId = randomUUID();
13936
+ const acknowledgementClaimed = await this.#state.claimConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId, this.#clock.now(), SLACK_REPLY_ACKNOWLEDGEMENT_LEASE_MS);
13937
+ if (acknowledgementClaimed) {
13938
+ try {
13939
+ if (!this.#slack)
13940
+ throw new Error(`Slack reply ${replyId} cannot be acknowledged without writeback`);
13941
+ const owner = durable.agent?.role === 'babysitter'
13942
+ ? 'the PR babysitter'
13943
+ : durable.agent
13944
+ ? 'the issue implementer'
13945
+ : 'an issue agent';
13946
+ const receipt = durable.agent
13947
+ ? `Factory received this reply and durably queued it for ${owner}.`
13948
+ : 'Factory received and durably stored this reply; it will route when an issue agent is resumable.';
13949
+ const slack = this.#slack;
13950
+ // Same lease scope as the terminal receipt: this claim covers a
13951
+ // provider write that can outrun any fixed duration, so it is
13952
+ // renewed for as long as that write is actually running.
13953
+ await this.#withRenewedProviderLease('Slack reply acknowledgement', SLACK_REPLY_ACKNOWLEDGEMENT_LEASE_MS, () => this.#state.renewConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId, this.#clock.now()), () => slack.reply(reply.threadTs, receipt));
13954
+ if (!await this.#state.completeConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId)) {
13955
+ throw new Error(`Slack reply ${replyId} receipt could not be recorded`);
13956
+ }
13957
+ this.#increment('slackConversationRepliesAcknowledged');
13958
+ }
13959
+ catch (error) {
13960
+ await this.#state.releaseConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId);
13961
+ throw error;
13962
+ }
13963
+ }
13964
+ else {
13965
+ const acknowledgementState = await this.#state.getConversationSession(this.#workspaceId, conversationId);
13966
+ if (!(acknowledgementState?.acknowledgedMessageIds ?? []).includes(replyId)) {
13967
+ throw new Error(`Slack reply ${replyId} receipt is claimed by another handler; retrying`);
13968
+ }
13969
+ }
13970
+ }
13971
+ if (queued) {
13972
+ this.#increment('slackConversationRepliesQueued');
13973
+ }
13974
+ else {
13112
13975
  this.#increment('slackConversationDuplicateRepliesSuppressed');
13113
- return;
13114
13976
  }
13115
- this.#increment('slackConversationRepliesQueued');
13116
- this.#slackConversationTurns.schedule(conversationId);
13977
+ const pending = durable.pending.some((message) => message.id === replyId) ||
13978
+ Boolean(durable.delivery?.messages.some((message) => message.id === replyId));
13979
+ if (pending && durable.agent) {
13980
+ this.#slackConversationTurns.schedule(conversationId);
13981
+ }
13982
+ else if (pending) {
13983
+ this.#increment('slackConversationRepliesWaitingForOwner');
13984
+ }
13117
13985
  return;
13118
13986
  }
13119
- const liveRecord = (await this.#batch()).getIssue(record.issue);
13987
+ liveRecord ??= (await this.#batch()).getIssue(record.issue);
13120
13988
  if (!liveRecord || liveRecord.dryRun) {
13121
13989
  if (isTriageEscalationWatchRecord(record)) {
13122
13990
  return await this.#handleTriageEscalationSlackAnswer(record, text);
13123
13991
  }
13124
- this.#increment('slackAnswersIgnoredNoInFlight');
13992
+ await this.#writeUnroutableSlackReply(reply.threadTs);
13125
13993
  return;
13126
13994
  }
13127
13995
  this.#increment('slackAnswersIgnoredNoConversationSession');
13996
+ if (this.#slack) {
13997
+ await this.#slack.reply(reply.threadTs, 'Factory received this reply but could not create a durable agent route. It will remain replayable; please also continue on the linked issue or pull request.');
13998
+ this.#increment('slackAnswersUnroutableVisible');
13999
+ }
14000
+ }
14001
+ async #writeUnroutableSlackReply(threadId) {
14002
+ this.#increment('slackAnswersIgnoredNoInFlight');
14003
+ if (!this.#slack)
14004
+ return;
14005
+ await this.#slack.reply(threadId, 'Factory received this reply but could not route it because this work unit no longer has an active agent. Please continue on the linked issue or pull request.');
14006
+ this.#increment('slackAnswersUnroutableVisible');
13128
14007
  }
13129
14008
  async #wakeWaitingClarification(key, waiting) {
13130
14009
  const existing = this.#clarificationWakeInFlight.get(key);
@@ -13414,6 +14293,7 @@ export class FactoryLoop {
13414
14293
  const resumed = await this.#fleet.resume({
13415
14294
  name,
13416
14295
  sessionRef: tracked.sessionRef,
14296
+ identityKey: dispatchAgentIdentityKey(waiting.issue, tracked.spec.role),
13417
14297
  node: tracked.result?.node ?? tracked.spec.node ?? 'self',
13418
14298
  capability: tracked.spec.capability,
13419
14299
  repo: tracked.spec.repo,
@@ -13444,6 +14324,7 @@ export class FactoryLoop {
13444
14324
  return await this.#fleet.spawn({
13445
14325
  name,
13446
14326
  capability: tracked.spec.capability,
14327
+ identityKey: dispatchAgentIdentityKey(waiting.issue, tracked.spec.role),
13447
14328
  node: tracked.result?.node ?? tracked.spec.node ?? 'self',
13448
14329
  task,
13449
14330
  workflow: tracked.spec.workflow,
@@ -13494,6 +14375,7 @@ export class FactoryLoop {
13494
14375
  const batch = await this.#batch();
13495
14376
  if (batch.isInFlight(record.issue) || batch.isQueued(record.issue)) {
13496
14377
  this.#increment('slackTriageAnswersIgnoredAlreadyActive');
14378
+ await this.#state.clearSlackThreadWatch(this.#workspaceId, issueKey(record.issue));
13497
14379
  return;
13498
14380
  }
13499
14381
  if (await this.#dispatchBlockReason(record.issue)) {
@@ -13510,6 +14392,10 @@ export class FactoryLoop {
13510
14392
  if (hasDispatchableRoute(decision)) {
13511
14393
  this.#pendingSlackClarifications.set(issueKey(decision.issue), text);
13512
14394
  const result = await this.#startOrQueueSlackClarifiedDecision(dispatchAfterSlackClarification(decision, escalationReason));
14395
+ const active = await this.#batch();
14396
+ if (result || active.isInFlight(decision.issue) || active.isQueued(decision.issue)) {
14397
+ await this.#state.clearSlackThreadWatch(this.#workspaceId, issueKey(record.issue));
14398
+ }
13513
14399
  this.#increment('slackTriageAnswersDispatchedWithRemainingEscalation');
13514
14400
  return result;
13515
14401
  }
@@ -13522,6 +14408,10 @@ export class FactoryLoop {
13522
14408
  }
13523
14409
  this.#pendingSlackClarifications.set(issueKey(decision.issue), text);
13524
14410
  const result = await this.#startOrQueueSlackClarifiedDecision(decision);
14411
+ const active = await this.#batch();
14412
+ if (result || active.isInFlight(decision.issue) || active.isQueued(decision.issue)) {
14413
+ await this.#state.clearSlackThreadWatch(this.#workspaceId, issueKey(record.issue));
14414
+ }
13525
14415
  this.#increment('slackTriageAnswersDispatched');
13526
14416
  return result;
13527
14417
  }
@@ -13757,6 +14647,15 @@ export class FactoryLoop {
13757
14647
  return hasTitlePrefix(issue.title, FACTORY_E2E_MARKER);
13758
14648
  }
13759
14649
  }
14650
+ const defaultGithubWriteback = (config, mount) => {
14651
+ if (config.github.identity !== 'app') {
14652
+ return new GhCliGithubWriteback();
14653
+ }
14654
+ if (!mount.githubWrite) {
14655
+ throw new Error('GitHub identity "app" requires a connected workspace GitHub App lifecycle write path; refusing to fall back to the local gh user');
14656
+ }
14657
+ return new AppGithubWriteback(mount.githubWrite);
14658
+ };
13760
14659
  export function parseLinearIssue(path, content) {
13761
14660
  const parsed = parseJsonContent(content);
13762
14661
  const payload = wrappedPayload(parsed);
@@ -14352,13 +15251,19 @@ function labelDerivedDispatchDecision(liveIssue, decision, config) {
14352
15251
  maxImplementers,
14353
15252
  };
14354
15253
  }
14355
- const implementers = routesByLabel.routes.map(({ slug, route }) => routeImplementerSpec(liveIssue, config, slug, route));
14356
- const selectedRoutes = scope === 'single' ? routesByLabel.routes.slice(0, 1) : routesByLabel.routes;
15254
+ // Swarm always shares one checkout (lead + workers collaborate live over a
15255
+ // shared relay channel), so — like single/workflow only the first matched
15256
+ // label route is used; it is never fanned out across repo labels like team.
15257
+ const selectedRoutes = scope === 'single' || scope === 'swarm'
15258
+ ? routesByLabel.routes.slice(0, 1)
15259
+ : routesByLabel.routes;
14357
15260
  const selectedImplementers = scope === 'team'
14358
- ? implementers
15261
+ ? routesByLabel.routes.map(({ slug, route }) => routeImplementerSpec(liveIssue, config, slug, route))
14359
15262
  : scope === 'single'
14360
- ? implementers.slice(0, 1)
14361
- : [];
15263
+ ? routesByLabel.routes.slice(0, 1).map(({ slug, route }) => routeImplementerSpec(liveIssue, config, slug, route))
15264
+ : scope === 'swarm'
15265
+ ? routeSwarmImplementerSpecs(liveIssue, config, selectedRoutes[0]?.route, maxImplementers)
15266
+ : [];
14362
15267
  const routes = selectedRoutes.map(({ route }) => route);
14363
15268
  const workflow = scope === 'workflow'
14364
15269
  ? routeWorkflowSpec(liveIssue, config, selectedRoutes, decision.workflow)
@@ -14487,6 +15392,24 @@ function routeImplementerSpec(issue, config, slug, route) {
14487
15392
  node: 'self',
14488
15393
  };
14489
15394
  }
15395
+ function routeSwarmImplementerSpecs(issue, config, route, maxImplementers) {
15396
+ if (!route) {
15397
+ return [];
15398
+ }
15399
+ const channel = swarmChannel(issue);
15400
+ return swarmMemberSlugs(maxImplementers).map((slug) => ({
15401
+ name: agentNameForRole(issue, 'impl', { repo: route.repo, discriminator: slug }),
15402
+ role: 'implementer',
15403
+ capability: config.agentCapabilities.implementer,
15404
+ model: config.models.implementer,
15405
+ task: swarmTaskFor(issue, route, slug, channel),
15406
+ repo: route.repo,
15407
+ clonePath: route.clonePath,
15408
+ channel,
15409
+ swarmRole: slug === 'lead' ? 'lead' : 'worker',
15410
+ node: 'self',
15411
+ }));
15412
+ }
14490
15413
  function decisionWithLifecycleBranches(decision, runId, opts = {}) {
14491
15414
  const implementerBranch = (spec) => {
14492
15415
  const runSuffix = `-${runId.slice(0, 8)}`;
@@ -15633,12 +16556,123 @@ const isAllowedFactoryDraft = async (path, content, opts, mount, config) => {
15633
16556
  if (/^\/slack\/channels\/[^/]+\/messages\/.+/u.test(path)) {
15634
16557
  return true;
15635
16558
  }
15636
- if (isFactoryGithubWritebackPath(path)) {
16559
+ if (await isAllowedFactoryGithubDraft(path, content, opts, mount, config))
15637
16560
  return true;
16561
+ return false;
16562
+ };
16563
+ const isFactoryGithubAuthoredArtifactPath = (path) => /^\/github\/repos\/[^/]+\/[^/]+\/(?:pull-requests\/factory-[^/]+\.json|refs\/(?:factory\.json|refs%2Fheads%2Ffactory%2F[^/]+\.json)|pulls\/[1-9]\d*\/close\.json)$/iu.test(path);
16564
+ export const isAllowedFactoryGithubArtifactDraft = (path, opts) => opts?.guarded === true && isFactoryGithubAuthoredArtifactPath(path);
16565
+ const factoryGithubIssueWriteTarget = (path) => {
16566
+ const match = /^\/github\/repos\/([^/]+)\/([^/]+)\/issues\/([1-9]\d*)(?:\.json|\/(comments|labels)\/([^/]+))$/iu.exec(path);
16567
+ if (!match?.[1] || !match[2] || !match[3])
16568
+ return undefined;
16569
+ const child = match[4];
16570
+ const filename = match[5];
16571
+ if (child === 'comments' && (!filename || !isFactoryGithubIssueCommentDraftName(filename)))
16572
+ return undefined;
16573
+ if (child === 'labels' && (!filename || !isFactoryGithubOperationDraftName(filename)))
16574
+ return undefined;
16575
+ try {
16576
+ return {
16577
+ owner: decodeURIComponent(match[1]),
16578
+ repo: decodeURIComponent(match[2]),
16579
+ number: Number(match[3]),
16580
+ kind: child === 'comments' ? 'comment' : child === 'labels' ? 'label-operation' : 'issue-update',
16581
+ };
16582
+ }
16583
+ catch {
16584
+ return undefined;
16585
+ }
16586
+ };
16587
+ const factoryGithubRepositoryLabelWriteTarget = (path) => {
16588
+ const match = /^\/github\/repos\/([^/]+)\/([^/]+)\/labels\/([^/]+)$/iu.exec(path);
16589
+ if (!match?.[1] || !match[2] || !match[3] || !isFactoryGithubOperationDraftName(match[3]))
16590
+ return undefined;
16591
+ try {
16592
+ return { owner: decodeURIComponent(match[1]), repo: decodeURIComponent(match[2]) };
16593
+ }
16594
+ catch {
16595
+ return undefined;
16596
+ }
16597
+ };
16598
+ const githubLifecycleLabel = (name) => Object.values(FACTORY_GITHUB_STATUS_LABELS).find((label) => label.name === name);
16599
+ const hasExactKeys = (value, keys) => {
16600
+ const actual = Object.keys(value).sort();
16601
+ const expected = [...keys].sort();
16602
+ return actual.length === expected.length && actual.every((key, index) => key === expected[index]);
16603
+ };
16604
+ const isAllowedFactoryGithubIssueWriteContent = (kind, content) => {
16605
+ const value = asRecord(content);
16606
+ if (!value)
16607
+ return false;
16608
+ if (kind === 'issue-update') {
16609
+ return hasExactKeys(value, ['state']) && value.state === 'closed';
16610
+ }
16611
+ if (kind === 'comment') {
16612
+ return hasExactKeys(value, ['body']) && typeof value.body === 'string' && value.body.trim().length > 0;
16613
+ }
16614
+ if (value.operation === 'add') {
16615
+ return hasExactKeys(value, ['labels', 'operation']) &&
16616
+ Array.isArray(value.labels) && value.labels.length === 1 && Boolean(githubLifecycleLabel(value.labels[0]));
16617
+ }
16618
+ return value.operation === 'remove' && hasExactKeys(value, ['label', 'operation']) && Boolean(githubLifecycleLabel(value.label));
16619
+ };
16620
+ const isAllowedFactoryGithubRepositoryLabelContent = (content) => {
16621
+ const value = asRecord(content);
16622
+ if (!value || !hasExactKeys(value, ['color', 'description', 'name']))
16623
+ return false;
16624
+ const expected = githubLifecycleLabel(value.name);
16625
+ return Boolean(expected && expected.color === value.color && expected.description === value.description);
16626
+ };
16627
+ /**
16628
+ * Last-resort mount guard for Factory-authored GitHub drafts. PR/ref paths are
16629
+ * intrinsically Factory-owned. Issue mutations additionally require a current
16630
+ * in-scope issue projection in one of Relayfile's supported repository layouts.
16631
+ */
16632
+ export const isAllowedFactoryGithubDraft = async (path, content, opts, mount, config) => {
16633
+ if (!opts?.guarded)
16634
+ return false;
16635
+ if (isAllowedFactoryGithubArtifactDraft(path, opts))
16636
+ return true;
16637
+ const repositoryLabelTarget = factoryGithubRepositoryLabelWriteTarget(path);
16638
+ if (repositoryLabelTarget) {
16639
+ const repoPath = `/github/repos/${encodeURIComponent(repositoryLabelTarget.owner)}/${encodeURIComponent(repositoryLabelTarget.repo)}/`;
16640
+ return isConfiguredGithubRepoPath(repoPath, config) && isAllowedFactoryGithubRepositoryLabelContent(content);
16641
+ }
16642
+ const target = factoryGithubIssueWriteTarget(path);
16643
+ if (!target)
16644
+ return false;
16645
+ if (!isAllowedFactoryGithubIssueWriteContent(target.kind, content))
16646
+ return false;
16647
+ if (target.kind === 'comment') {
16648
+ const body = asRecord(content)?.body;
16649
+ const draftName = path.slice(path.lastIndexOf('/') + 1);
16650
+ if (typeof body !== 'string' || draftName !== factoryGithubIssueCommentDraftName(body))
16651
+ return false;
16652
+ }
16653
+ const repoPath = `/github/repos/${encodeURIComponent(target.owner)}/${encodeURIComponent(target.repo)}`;
16654
+ if (!isConfiguredGithubRepoPath(`${repoPath}/`, config))
16655
+ return false;
16656
+ const compactRepo = `${encodeURIComponent(target.owner)}__${encodeURIComponent(target.repo)}`;
16657
+ const candidates = [
16658
+ `${repoPath}/issues/by-id/${target.number}.json`,
16659
+ `/github/repos/${compactRepo}/issues/by-id/${target.number}.json`,
16660
+ `${repoPath}/issues/${target.number}/meta.json`,
16661
+ `/github/repos/${compactRepo}/issues/${target.number}/meta.json`,
16662
+ `${repoPath}/issues/${target.number}.json`,
16663
+ `/github/repos/${compactRepo}/issues/${target.number}.json`,
16664
+ ];
16665
+ for (const candidate of candidates) {
16666
+ try {
16667
+ const issue = parseGithubFactoryIssue(candidate, (await mount.readFile(candidate)).content);
16668
+ return issue.state?.name === 'open' && isInFactoryScope(issue, config.safety);
16669
+ }
16670
+ catch {
16671
+ // Try the next canonical/alias shape. Any total miss fails closed.
16672
+ }
15638
16673
  }
15639
16674
  return false;
15640
16675
  };
15641
- const isFactoryGithubWritebackPath = (path) => /^\/github\/repos\/[^/]+\/[^/]+\/(?:pull-requests\/factory-[^/]+\.json|refs\/(?:factory\.json|refs%2Fheads%2Ffactory%2F[^/]+\.json)|pulls\/[1-9]\d*\/close\.json)$/iu.test(path);
15642
16676
  const isIssuePathInFactoryScope = async (mount, path, config) => {
15643
16677
  try {
15644
16678
  return isInFactoryScope(parseLinearIssue(path, (await mount.readFile(path)).content), config.safety);
@@ -15888,8 +16922,8 @@ const normalizeGithubRepo = (repo, defaultOwner) => {
15888
16922
  }
15889
16923
  return `${owner}/${repo}`;
15890
16924
  };
15891
- const githubPullRequestBody = (issue, preview) => [
15892
- issue.description,
16925
+ const githubPullRequestBody = (issue, preview, sessionRef) => [
16926
+ stripTrajectoryPointers(issue.description),
15893
16927
  '',
15894
16928
  isGithubIssue(issue) && /^\d+$/u.test(issue.key)
15895
16929
  ? `Fixes #${issue.key}`
@@ -15899,7 +16933,25 @@ const githubPullRequestBody = (issue, preview) => [
15899
16933
  `Live preview: ${preview.url}`,
15900
16934
  'Access: Tailscale tailnet membership and the tailnet grants/ACLs are required; this URL is not public.',
15901
16935
  ] : []),
16936
+ '',
16937
+ renderTrajectoryPointer({
16938
+ ...trajectoryWorkUnitForIssue(issue),
16939
+ sessionRef,
16940
+ }),
15902
16941
  ].join('\n').trim();
16942
+ const trajectoryWorkUnitForIssue = (issue) => {
16943
+ const github = githubIssueSourceRef(issue);
16944
+ if (github) {
16945
+ return {
16946
+ workUnitId: `${github.owner}/${github.repo}#${github.number}`,
16947
+ workUnitSurface: 'github',
16948
+ };
16949
+ }
16950
+ if (isRealLinearIssue(issue)) {
16951
+ return { workUnitId: issue.key, workUnitSurface: 'linear' };
16952
+ }
16953
+ return { workUnitId: `factory:${issue.uuid}`, workUnitSurface: 'factory' };
16954
+ };
15903
16955
  // The broker rejects re-registering a name it never released on exit
15904
16956
  // (relay#1116-family) with a 500 "agent '<name>' already exists". Detect it from
15905
16957
  // the structured payload or the message so resume can treat it as terminal
@@ -15928,6 +16980,9 @@ const slackMessageReceivedAtMs = (messageTs, fallback) => {
15928
16980
  const seconds = Number(messageTs);
15929
16981
  return Number.isFinite(seconds) && seconds > 0 ? Math.floor(seconds * 1_000) : fallback;
15930
16982
  };
16983
+ const terminalSlackWatchRetiredAtMs = (watch) => typeof watch.retiredAtMs === 'number' && Number.isFinite(watch.retiredAtMs)
16984
+ ? watch.retiredAtMs
16985
+ : Math.max(0, watch.expiresAtMs - SLACK_TERMINAL_THREAD_GRACE_MS);
15931
16986
  const eventIdentity = (event) => {
15932
16987
  const record = event;
15933
16988
  const rawId = record.id ?? record.event_id ?? record.seq;
@@ -16091,6 +17146,7 @@ const cloneTrackedAgent = (tracked) => ({
16091
17146
  result: tracked.result ? { ...tracked.result } : undefined,
16092
17147
  sessionRef: tracked.sessionRef,
16093
17148
  unreachableWakeResumedSessionRef: tracked.unreachableWakeResumedSessionRef,
17149
+ releasedAtMs: tracked.releasedAtMs,
16094
17150
  });
16095
17151
  const durableBabysitterTrackedAgent = (session, capability = 'spawn:claude') => ({
16096
17152
  spec: {
@@ -16167,7 +17223,11 @@ const lifecycleFromInFlightRecord = (record, runId, phase, updatedAtMs, pullRequ
16167
17223
  decision: structuredClone(record.decision),
16168
17224
  dryRun: record.dryRun,
16169
17225
  phase,
16170
- agents: [...record.agents].map(([name, tracked]) => ({ name, tracked: cloneTrackedAgent(tracked) })),
17226
+ agents: [...record.agents].map(([name, tracked]) => ({
17227
+ name,
17228
+ tracked: cloneTrackedAgent(tracked),
17229
+ ...(tracked.releasedAtMs !== undefined ? { releasedAtMs: tracked.releasedAtMs } : {}),
17230
+ })),
16171
17231
  invocationIds: [...record.invocationIds],
16172
17232
  result: record.result ? structuredClone(record.result) : undefined,
16173
17233
  ...(record.dispatchClaim ? { dispatchClaim: { ...record.dispatchClaim } } : {}),
@@ -16182,7 +17242,13 @@ const inFlightRecordFromLifecycle = (lifecycle) => ({
16182
17242
  issue: { ...lifecycle.issue },
16183
17243
  decision: structuredClone(lifecycle.decision),
16184
17244
  dryRun: lifecycle.dryRun,
16185
- agents: new Map(lifecycle.agents.map((agent) => [agent.name, cloneTrackedAgent(agent.tracked)])),
17245
+ // Release state has to survive takeover with the agents themselves. Without
17246
+ // it the rebuilt record reads a released placement as a live worker and the
17247
+ // spawn gate answers for a process that no longer exists.
17248
+ agents: new Map(lifecycle.agents.map((agent) => [agent.name, {
17249
+ ...cloneTrackedAgent(agent.tracked),
17250
+ releasedAtMs: agent.releasedAtMs ?? agent.tracked.releasedAtMs,
17251
+ }])),
16186
17252
  invocationIds: new Set(lifecycle.invocationIds),
16187
17253
  result: lifecycle.result ? structuredClone(lifecycle.result) : undefined,
16188
17254
  ...(lifecycle.dispatchClaim ? { dispatchClaim: { ...lifecycle.dispatchClaim } } : {}),
@@ -16209,7 +17275,14 @@ const triageEscalationReason = (decision) => {
16209
17275
  }
16210
17276
  return `${reasons.join(' and ')}${decision.rationale ? `: ${decision.rationale}` : ''}`;
16211
17277
  };
16212
- class LiveDispatchStateChangedError extends Error {
17278
+ /**
17279
+ * The dispatch claim race: another writer changed the issue's live state
17280
+ * between this process reading it and writing back, so the dispatch was
17281
+ * abandoned. Exported so callers outside this module — notably the CLI, which
17282
+ * turns it into a distinct exit code — can recognize it by type rather than by
17283
+ * matching on `error.name`, and so tests can construct a genuine instance.
17284
+ */
17285
+ export class LiveDispatchStateChangedError extends Error {
16213
17286
  issueKey;
16214
17287
  constructor(issueKey) {
16215
17288
  super(`Live state changed before writeback for ${issueKey}`);
@@ -16217,6 +17290,59 @@ class LiveDispatchStateChangedError extends Error {
16217
17290
  this.issueKey = issueKey;
16218
17291
  }
16219
17292
  }
17293
+ /** Whether a thrown value is a {@link LiveDispatchStateChangedError}. */
17294
+ export function isLiveDispatchStateChangedError(error) {
17295
+ return error instanceof LiveDispatchStateChangedError;
17296
+ }
17297
+ /** How deep to follow `cause` when classifying a wrapped failure. */
17298
+ const PASS_FATAL_CAUSE_DEPTH = 4;
17299
+ /**
17300
+ * Whether `error`, or anything it wraps, is an instance of `type`.
17301
+ * `contextualError` and the fleet control-plane guard both rethrow wrapped, so
17302
+ * classification has to follow the cause chain rather than trust the outermost
17303
+ * type.
17304
+ */
17305
+ const wrapsErrorOfType = (error, type, depth = 0) => {
17306
+ if (depth > PASS_FATAL_CAUSE_DEPTH || !(error instanceof Error))
17307
+ return false;
17308
+ if (error instanceof type)
17309
+ return true;
17310
+ return wrapsErrorOfType(error.cause, type, depth + 1);
17311
+ };
17312
+ /**
17313
+ * How many *unclassified* per-item failures without an intervening successful
17314
+ * dispatch end the pass. Named per-item conditions (a lifecycle claim refusal,
17315
+ * a live-state race) never count toward it and never reset it: those
17316
+ * legitimately affect many units at once and are exactly the benign case #292
17317
+ * asks the loop to survive, so they are neither evidence of a pass-wide fault
17318
+ * nor evidence against one.
17319
+ */
17320
+ const UNCLASSIFIED_DISPATCH_FAILURE_LIMIT = 5;
17321
+ /**
17322
+ * Failures the loop recognizes as belonging to one work unit. They are always
17323
+ * skippable and are exempt from the consecutive-failure fuse.
17324
+ */
17325
+ const isClassifiedPerItemDispatchFailure = (error) => error instanceof LiveDispatchStateChangedError ||
17326
+ error instanceof DispatchLifecycleClaimRefusedError;
17327
+ /**
17328
+ * The run-report reason recorded for a work unit the pass could not dispatch.
17329
+ *
17330
+ * `factory run-once` serializes the whole report to stdout, so this string is
17331
+ * a public surface: it stays a fixed classification plus an allowlisted error
17332
+ * class name, never raw provider text or filesystem paths. The full message
17333
+ * goes to the operator log instead, the same split
17334
+ * `describeControlPlaneError` makes for circuit state.
17335
+ */
17336
+ const perItemDispatchSkipReason = (error) => {
17337
+ if (error instanceof LiveDispatchStateChangedError)
17338
+ return 'live state changed during dispatch';
17339
+ if (error instanceof DispatchLifecycleClaimRefusedError) {
17340
+ return error.refusal === 'terminal'
17341
+ ? 'dispatch lifecycle already terminal'
17342
+ : 'dispatch lifecycle owned by another publisher';
17343
+ }
17344
+ return `dispatch failed (${telemetryErrorClass(error)})`;
17345
+ };
16220
17346
  const triageEscalationQuestion = (decision, issue) => {
16221
17347
  const routedRepos = decision.routes.map((route) => route.repo).filter(Boolean);
16222
17348
  const subject = issue?.title?.trim() || decision.issue.key;