@agent-relay/factory 0.1.63 → 0.1.64

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/dist/cli/fleet.d.ts +4 -0
  2. package/dist/cli/fleet.d.ts.map +1 -1
  3. package/dist/cli/fleet.js +31 -7
  4. package/dist/cli/fleet.js.map +1 -1
  5. package/dist/fleet/control-plane-circuit.d.ts.map +1 -1
  6. package/dist/fleet/control-plane-circuit.js +13 -1
  7. package/dist/fleet/control-plane-circuit.js.map +1 -1
  8. package/dist/index.d.ts +2 -0
  9. package/dist/index.d.ts.map +1 -1
  10. package/dist/index.js +1 -0
  11. package/dist/index.js.map +1 -1
  12. package/dist/mount/relayfile-cloud-mount-client.d.ts +23 -1
  13. package/dist/mount/relayfile-cloud-mount-client.d.ts.map +1 -1
  14. package/dist/mount/relayfile-cloud-mount-client.js +38 -9
  15. package/dist/mount/relayfile-cloud-mount-client.js.map +1 -1
  16. package/dist/observability/cloud-reporter.d.ts +1 -0
  17. package/dist/observability/cloud-reporter.d.ts.map +1 -1
  18. package/dist/observability/cloud-reporter.js +182 -50
  19. package/dist/observability/cloud-reporter.js.map +1 -1
  20. package/dist/orchestrator/factory.d.ts.map +1 -1
  21. package/dist/orchestrator/factory.js +826 -52
  22. package/dist/orchestrator/factory.js.map +1 -1
  23. package/dist/ports/state.d.ts +60 -3
  24. package/dist/ports/state.d.ts.map +1 -1
  25. package/dist/state/document-store.d.ts +2 -1
  26. package/dist/state/document-store.d.ts.map +1 -1
  27. package/dist/state/document-store.js.map +1 -1
  28. package/dist/state/file-state-store.d.ts +16 -5
  29. package/dist/state/file-state-store.d.ts.map +1 -1
  30. package/dist/state/file-state-store.js +124 -8
  31. package/dist/state/file-state-store.js.map +1 -1
  32. package/dist/state/in-memory-state-store.d.ts +13 -2
  33. package/dist/state/in-memory-state-store.d.ts.map +1 -1
  34. package/dist/state/in-memory-state-store.js +87 -4
  35. package/dist/state/in-memory-state-store.js.map +1 -1
  36. package/dist/state/watch-state-document.d.ts.map +1 -1
  37. package/dist/state/watch-state-document.js +45 -3
  38. package/dist/state/watch-state-document.js.map +1 -1
  39. package/dist/trajectory.d.ts +25 -0
  40. package/dist/trajectory.d.ts.map +1 -0
  41. package/dist/trajectory.js +51 -0
  42. package/dist/trajectory.js.map +1 -0
  43. package/package.json +1 -1
@@ -31,6 +31,7 @@ import { readFactoryInFlightRegistry, terminatePids } from './reaper.js';
31
31
  import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../observability/events.js';
32
32
  import { boundedRunCostTotal, CostLedger } from '../cost/ledger.js';
33
33
  import { createTicketDispatchDelivery } from '../delivery/ticket-dispatch.js';
34
+ import { canonicalTrajectorySessionRef, renderTrajectoryPointer, stripTrajectoryPointers, } from '../trajectory.js';
34
35
  import { FleetControlPlaneCircuit, FleetControlPlaneCircuitOpenError, guardFleetControlPlane, } from '../fleet/control-plane-circuit.js';
35
36
  class ClarificationWakeLeaseLostError extends Error {
36
37
  }
@@ -146,8 +147,32 @@ const STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS = 30_000;
146
147
  const RECONCILED_AGENT_EXIT_CONCURRENCY = 4;
147
148
  const SLACK_EVENT_WATERMARK_CACHE_MS = 60_000;
148
149
  const SLACK_CONVERSATION_TURN_LEASE_MS = 60_000;
150
+ // Both receipt leases guard an in-flight Slack writeback, and no fixed lease can
151
+ // cover one: MountSlackWriteback budgets 90s for the confirm alone, on top of an
152
+ // unbounded writeFile. Sizing them past that worst case would only trade a stolen
153
+ // claim for a stranded one — a lease long enough to survive the slowest write is
154
+ // equally long enough to hold the receipt hostage to a dead holder. So these
155
+ // bound the *idle* claim and #withRenewedProviderLease extends them for exactly
156
+ // as long as the write they cover is still running.
157
+ const SLACK_REPLY_ACKNOWLEDGEMENT_LEASE_MS = 60_000;
158
+ const SLACK_TERMINAL_RECEIPT_CLAIM_LEASE_MS = 60_000;
159
+ // Renewal without a ceiling is the same defect from the other side: a heartbeat
160
+ // that extends the claim for as long as the write runs also extends it forever
161
+ // when the write never returns, and nothing else can reclaim the receipt short
162
+ // of a restart. So renewal is bounded past the slowest write this daemon budgets
163
+ // for — MountSlackWriteback's 90s confirm on top of its writeFile — and beyond
164
+ // that the write is not slow, it is wedged: the heartbeat stops, the idle lease
165
+ // runs out, and the retry that owns the queued replies can take them back.
166
+ const SLACK_PROVIDER_LEASE_MAX_RENEWAL_MS = 5 * 60_000;
149
167
  const SLACK_CONVERSATION_TURN_RETRY_MS = 1_000;
150
168
  const SLACK_REPLY_ROUTE_RETRY_MS = 1_000;
169
+ // One pass drains the whole chain (#slackReplyRoutes holds only the newest
170
+ // route per key and every route awaits its predecessor). The extra passes only
171
+ // exist so the drain can prove quiescence rather than assume it.
172
+ const SLACK_REPLY_ROUTE_DRAIN_PASSES = 8;
173
+ const SLACK_TERMINAL_THREAD_GRACE_MS = 24 * 60 * 60_000;
174
+ const SLACK_TERMINAL_RECEIPT_RETRY_MS = 1_000;
175
+ const SLACK_TERMINAL_RECEIPT_RETRY_MAX_MS = 5 * 60_000;
151
176
  const MERGE_GATE_MAX_ATTEMPTS = 12;
152
177
  const MERGE_GATE_POLL_DELAY_MS = 10_000;
153
178
  const MAX_LABEL_IMPLEMENTERS = 4;
@@ -215,6 +240,25 @@ class DispatchLifecycleOwnedElsewhereError extends Error {
215
240
  this.leaseUntilMs = leaseUntilMs;
216
241
  }
217
242
  }
243
+ /**
244
+ * The durable dispatch-lifecycle claim was refused for one work unit: its
245
+ * record is already terminal, or another publisher currently holds the lease.
246
+ * Both are facts about that single unit — the rest of the pass is unaffected —
247
+ * so the readiness loop skips it and keeps going (#292).
248
+ *
249
+ * Typed rather than left as a plain `Error` so the loop can classify it by
250
+ * construction instead of by matching on `Refusing to dispatch ...` text.
251
+ */
252
+ class DispatchLifecycleClaimRefusedError extends Error {
253
+ issueKey;
254
+ refusal;
255
+ constructor(issueKey, refusal, message) {
256
+ super(message);
257
+ this.issueKey = issueKey;
258
+ this.refusal = refusal;
259
+ this.name = 'DispatchLifecycleClaimRefusedError';
260
+ }
261
+ }
218
262
  const realClock = {
219
263
  now: () => Date.now(),
220
264
  sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
@@ -266,6 +310,19 @@ export class FactoryLoop {
266
310
  #dispatchInFlight = new Map();
267
311
  #slackWatchers = new Map();
268
312
  #slackWatcherStarts = new Map();
313
+ #slackTerminalWatchExpiryTimers = new Map();
314
+ #slackTerminalReceiptRetryTimers = new Map();
315
+ #terminalSlackWatchIssues = new Set();
316
+ /**
317
+ * The one in-memory record of "this work unit has an in-flight Slack side
318
+ * effect". Both ordinary reply routes and the writebacks the terminal fence
319
+ * issues on their behalf register here, because this map is what the terminal
320
+ * drain waits on: anything that touches Slack for a work unit without
321
+ * registering is invisible to the drain, and the watcher teardown that follows
322
+ * a successful drain then pulls that effect's retry timer out from under it.
323
+ */
324
+ #slackReplyRoutes = new Map();
325
+ #slackReplyRouteDrains = new Set();
269
326
  #slackConversationTurns;
270
327
  #slackConversationOwner = `${process.pid}:${randomUUID()}`;
271
328
  #githubIssueCommentWatchers = new Map();
@@ -439,6 +496,11 @@ export class FactoryLoop {
439
496
  #discoverySweepRenewTimer;
440
497
  #discoverySweepRenewalInFlight;
441
498
  #discoverySweepLeaseLost = false;
499
+ // Registry/heartbeat paths the in-flight runLoop iteration would use. A
500
+ // per-item dispatch failure now skips instead of aborting the pass (#292),
501
+ // so the loop's catch no longer runs the failure-handoff reaper for it; the
502
+ // pass reaps inline and must write to the same paths runLoop would.
503
+ #loopReapPaths;
442
504
  #discoveryOverloadError;
443
505
  #resolvedIssueSource;
444
506
  #integrationInstructions;
@@ -907,6 +969,14 @@ export class FactoryLoop {
907
969
  await this.#boundedStopTeardown('factory subscription unsubscribe', () => subscription?.unsubscribe());
908
970
  await Promise.all([...this.#slackWatchers.values()].map((watcher) => watcher.stop()));
909
971
  this.#slackWatchers.clear();
972
+ for (const timer of this.#slackTerminalWatchExpiryTimers.values())
973
+ clearTimeout(timer);
974
+ this.#slackTerminalWatchExpiryTimers.clear();
975
+ for (const timer of this.#slackTerminalReceiptRetryTimers.values())
976
+ clearTimeout(timer);
977
+ this.#slackTerminalReceiptRetryTimers.clear();
978
+ this.#terminalSlackWatchIssues.clear();
979
+ this.#slackReplyRouteDrains.clear();
910
980
  await Promise.all([...this.#githubIssueCommentWatchers.values()].map((watcher) => watcher.stop()));
911
981
  this.#githubIssueCommentWatchers.clear();
912
982
  this.#githubIssueCommentWatchStates.clear();
@@ -2047,6 +2117,10 @@ export class FactoryLoop {
2047
2117
  reason: entry.reason,
2048
2118
  });
2049
2119
  };
2120
+ // Backstop for the skip-by-default catch below: see #292. Reset only by
2121
+ // a completed dispatch, so the name says "since a dispatch" rather than
2122
+ // "consecutive" — a benign classified skip in between does not clear it.
2123
+ let unclassifiedFailuresSinceDispatch = 0;
2050
2124
  let lastReadyReadProgressAtMs = this.#clock.now();
2051
2125
  let readyIssueReads = 0;
2052
2126
  const issueEntries = [];
@@ -2144,19 +2218,10 @@ export class FactoryLoop {
2144
2218
  }
2145
2219
  const decision = await this.triageIssue(issue);
2146
2220
  triaged.push(decision);
2147
- let result;
2148
- try {
2149
- result = await this.dispatch(decision, { dryRun });
2150
- }
2151
- catch (error) {
2152
- if (!(error instanceof LiveDispatchStateChangedError))
2153
- throw error;
2154
- recordSkip({ issue: decision.issue, reason: 'live state changed during dispatch' });
2155
- this.#logger.info?.('[factory] skipped issue whose live state changed during dispatch', {
2156
- issue: decision.issue.key,
2157
- });
2158
- continue;
2159
- }
2221
+ const result = await this.dispatch(decision, { dryRun });
2222
+ // A completed dispatch — even one that parks or escalates the issue —
2223
+ // proves the pipeline still works, so the fuse below starts over.
2224
+ unclassifiedFailuresSinceDispatch = 0;
2160
2225
  if (result.agents.length === 0 && !dryRun) {
2161
2226
  const reason = result.hold?.kind === 'dependency-cycle'
2162
2227
  ? `dependency cycle detected: ${result.hold.cycle?.join(' -> ') ?? 'unknown cycle'}`
@@ -2169,6 +2234,52 @@ export class FactoryLoop {
2169
2234
  dispatched.push(result);
2170
2235
  }
2171
2236
  }
2237
+ catch (error) {
2238
+ // #292: issues in a pass are independent work units, so a failure
2239
+ // that is about ONE unit costs that unit and nothing else. Only the
2240
+ // conditions named in `#isPassFatalFailure` — the ones where
2241
+ // continuing the pass is meaningless — abort the whole sweep.
2242
+ if (this.#isPassFatalFailure(error, dryRun))
2243
+ throw error;
2244
+ if (!isClassifiedPerItemDispatchFailure(error)) {
2245
+ unclassifiedFailuresSinceDispatch += 1;
2246
+ // A pass-wide fault can arrive disguised as a run of per-item
2247
+ // faults. Skipping every unit would then hand back a green report
2248
+ // that dispatched nothing, which is the same silent wedge #292 is
2249
+ // about, wearing the opposite costume. Fail the pass loudly so
2250
+ // `readinessReconcile.lastError` carries the cause.
2251
+ if (unclassifiedFailuresSinceDispatch >= UNCLASSIFIED_DISPATCH_FAILURE_LIMIT) {
2252
+ throw contextualError(`Aborting readiness pass after ${unclassifiedFailuresSinceDispatch} unclassified dispatch failures without a successful dispatch`, error);
2253
+ }
2254
+ this.#increment('dispatchItemFailuresSkipped');
2255
+ // The raw message is operator-facing only; the run report carries
2256
+ // the sanitized classification from `perItemDispatchSkipReason`.
2257
+ this.#logger.warn?.('[factory] skipped a work unit whose dispatch failed; continuing the pass', {
2258
+ issue: issueRef(issue).key,
2259
+ unclassifiedFailuresSinceDispatch,
2260
+ error: describeError(error).errorMessage,
2261
+ });
2262
+ this.#error(error, issueRef(issue));
2263
+ // The failure may have left half-spawned agents behind. runLoop's
2264
+ // catch used to reap them because this error aborted the pass;
2265
+ // now that the pass survives, the reap has to happen here or the
2266
+ // agents leak until the next failed iteration.
2267
+ await this.#reapDispatchFailureHandoffsNow();
2268
+ }
2269
+ else {
2270
+ // Not an error — the unit simply cannot be dispatched right now —
2271
+ // so this stays out of `counters.errors` and gets its own counter
2272
+ // instead, or a terminal-lifecycle backlog would be invisible to
2273
+ // anyone watching only `dispatchItemFailuresSkipped`.
2274
+ this.#increment('dispatchItemsSkippedUndispatchable');
2275
+ this.#logger.info?.('[factory] skipped a work unit that cannot be dispatched right now', {
2276
+ issue: issueRef(issue).key,
2277
+ error: describeError(error).errorMessage,
2278
+ });
2279
+ }
2280
+ recordSkip({ issue: issueRef(issue), reason: perItemDispatchSkipReason(error) });
2281
+ continue;
2282
+ }
2172
2283
  finally {
2173
2284
  if (recoveredIdentity)
2174
2285
  this.#reconciledGithubInProgress.delete(recoveredIdentity);
@@ -2202,6 +2313,79 @@ export class FactoryLoop {
2202
2313
  }
2203
2314
  }
2204
2315
  }
2316
+ /**
2317
+ * Whether a failure raised while processing ONE work unit must abort the
2318
+ * whole readiness pass instead of skipping that unit.
2319
+ *
2320
+ * The default is the opposite, and that inversion is the fix for #292.
2321
+ * Issues in a pass are independent work units: a failure that is *about one
2322
+ * unit* — its dispatch-lifecycle record, its live state, a provider fault on
2323
+ * its own writeback — costs that unit and nothing else. Before this, every
2324
+ * error except `LiveDispatchStateChangedError` escaped the `for` loop, so a
2325
+ * single issue whose lifecycle record had gone terminal stopped all dispatch
2326
+ * indefinitely, every pass.
2327
+ *
2328
+ * A condition belongs here only when continuing the pass is meaningless or
2329
+ * actively harmful — when the failure is about the *pass*, not the item:
2330
+ *
2331
+ * - The discovery sweep lease is gone. Another process now owns this
2332
+ * workspace's sweep, so every remaining read throws the same way and each
2333
+ * one would be recorded as an ordinary per-issue skip. The run report
2334
+ * would then claim a clean pass over work this process no longer has the
2335
+ * right to touch.
2336
+ * - Relayfile signalled overload for this sweep. The backend is shedding
2337
+ * load; grinding through the remaining units makes it worse, and
2338
+ * `#runOnceWithDiscoveryFence` is going to rethrow this at the fence
2339
+ * anyway.
2340
+ * - The factory is stopping. Teardown is in progress and dispatching more
2341
+ * agents now leaks them past the shutdown deadline.
2342
+ * - The fleet control-plane circuit is no longer closed, **on a live pass**.
2343
+ * Dispatch is globally paused — the same condition
2344
+ * `#assertFleetControlPlaneAvailable` refuses to *start* a live pass on,
2345
+ * so it must also stop one already in flight. A dry run is exempt: it
2346
+ * never calls that admission gate and never spawns, so a paused control
2347
+ * plane is irrelevant to it rather than fatal to it. Without the
2348
+ * exemption, one live pass that trips the circuit would poison every
2349
+ * later dry run — including the boot gate's own `run-once --dry-run`
2350
+ * probe, turning a recoverable circuit-open condition into a failed boot.
2351
+ * That is a nastier version of the wedge this whole change removes.
2352
+ *
2353
+ * Deliberately NOT here: JavaScript builtin error types. Classifying
2354
+ * "programmer faults" such as `TypeError` as fatal is the obvious next rule
2355
+ * and it is a trap — Node reports a failed `fetch` as `TypeError: fetch
2356
+ * failed`, which is precisely the transient per-item roster lookup that
2357
+ * wedged the second instance (#291). A rule keyed on builtin types would
2358
+ * have preserved that outage verbatim.
2359
+ *
2360
+ * Everything else — a refused lifecycle claim, a terminal lifecycle record,
2361
+ * a transient network fault on one issue — is per-item: record a skip and
2362
+ * keep going. The unclassified-failure fuse in `#performRunOnce` is the
2363
+ * backstop for a pass-wide fault that does not announce itself as one.
2364
+ */
2365
+ #isPassFatalFailure(error, dryRun) {
2366
+ // Sweep-scoped: these are about this process's right or ability to run the
2367
+ // pass at all, so they hold for a dry run exactly as for a live one.
2368
+ if (this.#discoverySweepLeaseLost || this.#discoveryOverloadError !== undefined || this.#stopping) {
2369
+ return true;
2370
+ }
2371
+ // Fleet-scoped, and therefore live-only. See the doc comment above.
2372
+ return !dryRun && this.#isFleetControlPlaneHalted(error);
2373
+ }
2374
+ /**
2375
+ * Whether dispatch is globally paused by the fleet control-plane circuit.
2376
+ *
2377
+ * Two reads, because the circuit announces itself two different ways. The
2378
+ * state read covers `guardedMutation`, which records a mutation's own
2379
+ * transport failure and rethrows the *original* error rather than the
2380
+ * circuit-open type — converting it there would be wrong, since the mutation
2381
+ * may already have reached the broker and callers key spawn-failure handling
2382
+ * off that original error. The type check covers a rejection raised without
2383
+ * any state transition, such as an already-open circuit refusing admission.
2384
+ */
2385
+ #isFleetControlPlaneHalted(error) {
2386
+ return this.#fleetControlPlane.status().state !== 'closed' ||
2387
+ wrapsErrorOfType(error, FleetControlPlaneCircuitOpenError);
2388
+ }
2205
2389
  #startDiscoverySweepRenewal(epoch) {
2206
2390
  this.#discoverySweepRenewTimer = setInterval(() => {
2207
2391
  if (this.#discoverySweepRenewalInFlight || this.#discoverySweepLeaseLost)
@@ -3085,6 +3269,7 @@ export class FactoryLoop {
3085
3269
  const maxConsecutiveFailures = Math.min(5, Math.max(1, Math.trunc(opts.maxConsecutiveFailures ?? this.#config.loop.maxConsecutiveFailures)));
3086
3270
  const heartbeatPath = opts.heartbeatPath ?? this.#config.loop.heartbeatPath;
3087
3271
  const registryPath = opts.registryPath ?? this.#config.loop.registryPath;
3272
+ this.#loopReapPaths = { heartbeatPath, registryPath };
3088
3273
  const reports = [];
3089
3274
  let consecutiveFailures = 0;
3090
3275
  let completed = false;
@@ -3140,6 +3325,7 @@ export class FactoryLoop {
3140
3325
  return reports;
3141
3326
  }
3142
3327
  finally {
3328
+ this.#loopReapPaths = undefined;
3143
3329
  if (!completed) {
3144
3330
  await this.#writeLoopHeartbeat(heartbeatPath, registryPath, 'stopping', reports.length, maxIterations);
3145
3331
  }
@@ -4142,10 +4328,11 @@ export class FactoryLoop {
4142
4328
  seed.decision = decisionWithLifecycleBranches(seed.decision, seed.runId);
4143
4329
  const claim = await this.#state.claimDispatchLifecycle(this.#workspaceId, key, seed, this.#dispatchLifecycleOwner, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS);
4144
4330
  if (!claim.acquired || !claim.lease) {
4145
- const reason = isTerminalDispatchLifecycle(claim.lifecycle)
4331
+ const terminal = isTerminalDispatchLifecycle(claim.lifecycle);
4332
+ const reason = terminal
4146
4333
  ? 'dispatch lifecycle is already terminal'
4147
4334
  : `dispatch lifecycle is owned by ${claim.lifecycle.lease?.owner ?? 'another publisher'}`;
4148
- throw new Error(`Refusing to dispatch ${decision.issue.key}: ${reason}`);
4335
+ throw new DispatchLifecycleClaimRefusedError(decision.issue.key, terminal ? 'terminal' : 'owned-elsewhere', `Refusing to dispatch ${decision.issue.key}: ${reason}`);
4149
4336
  }
4150
4337
  this.#dispatchLifecycleEpochs.set(claim.key ?? key, claim.lease.epoch);
4151
4338
  this.#hydrateCostLedger(claim.lifecycle);
@@ -5008,7 +5195,7 @@ export class FactoryLoop {
5008
5195
  for (const [name] of record.agents) {
5009
5196
  this.#fleet.markAgentTerminal?.(name, 'durable-dispatch-abandoned');
5010
5197
  }
5011
- await this.#stopSlackWatcher(record.issue);
5198
+ await this.#retireSlackWatcher(record);
5012
5199
  await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
5013
5200
  await this.#writeInFlightRegistry();
5014
5201
  this.#increment('dispatchLifecycleStaleIssuesAbandoned');
@@ -6131,7 +6318,7 @@ export class FactoryLoop {
6131
6318
  },
6132
6319
  });
6133
6320
  }
6134
- async #reapDispatchFailureHandoffsNow(heartbeatPath, registryPath) {
6321
+ async #reapDispatchFailureHandoffsNow(heartbeatPath = this.#loopReapPaths?.heartbeatPath ?? this.#config.loop.heartbeatPath, registryPath = this.#loopReapPaths?.registryPath ?? this.#config.loop.registryPath) {
6135
6322
  const handoffs = await this.#state.listFailureHandoffs(this.#workspaceId);
6136
6323
  if (handoffs.length === 0) {
6137
6324
  return;
@@ -7149,6 +7336,7 @@ export class FactoryLoop {
7149
7336
  }
7150
7337
  async #publishImplementerPullRequest(record, implementer, opts = {}) {
7151
7338
  const key = `${issueKey(record.issue)}:${implementer.spec.repo}`;
7339
+ const trajectorySessionRef = canonicalTrajectorySessionRef(implementer.sessionRef);
7152
7340
  const expectedHeadRef = implementer.spec.branch;
7153
7341
  if (!expectedHeadRef) {
7154
7342
  throw new Error(`Refusing to publish ${record.issue.key}: implementer has no Factory-derived branch`);
@@ -7214,7 +7402,7 @@ export class FactoryLoop {
7214
7402
  expectedHeadRef,
7215
7403
  baseRef,
7216
7404
  title: `${issue.key}: ${issue.title}`,
7217
- body: githubPullRequestBody(issue, implementer.spec.preview),
7405
+ body: githubPullRequestBody(issue, implementer.spec.preview, trajectorySessionRef),
7218
7406
  ...(implementer.sessionRef ? { sessionRef: implementer.sessionRef } : {}),
7219
7407
  });
7220
7408
  const published = result.author
@@ -7803,7 +7991,7 @@ export class FactoryLoop {
7803
7991
  await this.#recordDispatchTerminal(record.issue);
7804
7992
  const next = (await this.#batch()).complete(record.issue);
7805
7993
  await this.#drainReadyClarificationWake();
7806
- await this.#stopSlackWatcher(record.issue);
7994
+ await this.#retireSlackWatcher(record);
7807
7995
  await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
7808
7996
  await this.#writeInFlightRegistry();
7809
7997
  if (next) {
@@ -11763,7 +11951,7 @@ export class FactoryLoop {
11763
11951
  }
11764
11952
  if (!await this.#saveDispatchLifecycle(record, 'releasing', undefined, releaseReason))
11765
11953
  return;
11766
- await this.#stopSlackWatcher(record.issue);
11954
+ await this.#retireSlackWatcher(record);
11767
11955
  await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
11768
11956
  await this.#recordDispatchTerminal(record.issue);
11769
11957
  await this.#finishDurableRelease(record, releaseReason);
@@ -12241,6 +12429,21 @@ export class FactoryLoop {
12241
12429
  return;
12242
12430
  }
12243
12431
  const key = issueKey(record.issue);
12432
+ const previousWatch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
12433
+ .find(([watchKey]) => watchKey === key)?.[1];
12434
+ if (previousWatch?.kind === 'terminal-grace') {
12435
+ // A reopened work unit needs a fresh dispatch notification and a fresh
12436
+ // conversation. Do not let the old grace-period watcher (or its expiry
12437
+ // timer) capture and later tear down the new dispatch.
12438
+ if (!await this.#stopSlackWatcher(record.issue)) {
12439
+ // Fail closed. An undrained reply route still holds the retired thread
12440
+ // and would bind it to this dispatch, delivering a stale human reply to
12441
+ // fresh work. Leave the fence up; the next reconcile retries the drain.
12442
+ this.#logger.warn?.('[factory] deferring Slack dispatch thread for reopened work unit; in-flight reply route not drained', { issue: record.issue.key });
12443
+ this.#increment('slackDispatchThreadsDeferredUndrainedReply');
12444
+ return;
12445
+ }
12446
+ }
12244
12447
  const existingThread = await this.#persistedSlackThread(key);
12245
12448
  const watcherStart = this.#slackWatcherStarts.get(key);
12246
12449
  if (existingThread || watcherStart) {
@@ -12323,9 +12526,10 @@ export class FactoryLoop {
12323
12526
  if (existing) {
12324
12527
  const sessionRef = owned?.tracked.sessionRef;
12325
12528
  const agentName = owned ? (owned.tracked.result?.name ?? owned.name) : undefined;
12529
+ let rebound = false;
12326
12530
  if (owned && sessionRef &&
12327
- (agentName !== existing.agent.name || (options.forceAgentRebind === true && sessionRef !== existing.agent.sessionRef))) {
12328
- const rebound = await this.#state.rebindConversationSession(this.#workspaceId, conversationId, {
12531
+ (!existing.agent || agentName !== existing.agent.name || (options.forceAgentRebind === true && sessionRef !== existing.agent.sessionRef))) {
12532
+ rebound = await this.#state.rebindConversationSession(this.#workspaceId, conversationId, {
12329
12533
  name: agentName,
12330
12534
  sessionRef,
12331
12535
  role: owned.tracked.spec.role,
@@ -12339,40 +12543,39 @@ export class FactoryLoop {
12339
12543
  }
12340
12544
  if (existing.pending.length > 0 || existing.delivery) {
12341
12545
  const waiting = await this.#state.getWaitingClarification(this.#workspaceId, issueKey(existing.issue));
12342
- if (!waiting)
12546
+ if (!waiting && (existing.agent || rebound))
12343
12547
  this.#slackConversationTurns.schedule(conversationId);
12344
12548
  }
12345
12549
  return;
12346
12550
  }
12347
12551
  const sessionRef = owned?.tracked.sessionRef;
12348
- if (!owned || !sessionRef) {
12349
- this.#increment('slackConversationSessionsSkippedMissingSession');
12350
- return;
12351
- }
12352
12552
  const channelDir = await this.#slackChannelDir() ?? this.#config.slack?.channel;
12353
12553
  if (!channelDir)
12354
12554
  return;
12355
- const agentName = owned.tracked.result?.name ?? owned.name;
12555
+ const agentName = owned ? (owned.tracked.result?.name ?? owned.name) : undefined;
12356
12556
  const reserved = await this.#state.reserveConversationSession(this.#workspaceId, conversationId, {
12357
12557
  provider: 'slack',
12358
12558
  issue: { ...record.issue },
12359
12559
  externalId: threadId,
12360
12560
  context: { channelDir },
12361
- agent: {
12362
- name: agentName,
12363
- sessionRef,
12364
- role: owned.tracked.spec.role,
12365
- node: owned.tracked.result?.node ?? owned.tracked.spec.node,
12366
- capability: owned.tracked.spec.capability,
12367
- repo: owned.tracked.spec.repo,
12368
- clonePath: owned.tracked.spec.clonePath,
12369
- },
12561
+ ...(owned && sessionRef && agentName ? { agent: {
12562
+ name: agentName,
12563
+ sessionRef,
12564
+ role: owned.tracked.spec.role,
12565
+ node: owned.tracked.result?.node ?? owned.tracked.spec.node,
12566
+ capability: owned.tracked.spec.capability,
12567
+ repo: owned.tracked.spec.repo,
12568
+ clonePath: owned.tracked.spec.clonePath,
12569
+ } } : {}),
12370
12570
  history: [],
12371
12571
  processedMessageIds: [],
12572
+ acknowledgedMessageIds: [],
12573
+ acknowledgementClaims: {},
12372
12574
  pending: [],
12373
12575
  });
12374
- if (reserved)
12375
- this.#increment('slackConversationSessionsOwned');
12576
+ if (reserved) {
12577
+ this.#increment(owned && sessionRef ? 'slackConversationSessionsOwned' : 'slackConversationSessionsReservedUnowned');
12578
+ }
12376
12579
  }
12377
12580
  // Called right after a babysitter is spawned/reattached for an issue's PR so
12378
12581
  // an already-owned Slack conversation session (reserved earlier by the
@@ -12400,11 +12603,15 @@ export class FactoryLoop {
12400
12603
  const claimed = await this.#state.claimConversationTurn(this.#workspaceId, conversationId, this.#slackConversationOwner, claimId, this.#clock.now(), SLACK_CONVERSATION_TURN_LEASE_MS);
12401
12604
  if (!claimed?.delivery) {
12402
12605
  const current = await this.#state.getConversationSession(this.#workspaceId, conversationId);
12403
- if (current && (current.pending.length > 0 || current.delivery)) {
12606
+ if (current?.agent && (current.pending.length > 0 || current.delivery)) {
12404
12607
  this.#slackConversationTurns.schedule(conversationId, SLACK_CONVERSATION_TURN_RETRY_MS);
12405
12608
  }
12406
12609
  return;
12407
12610
  }
12611
+ if (!claimed.agent) {
12612
+ await this.#state.releaseConversationTurn(this.#workspaceId, conversationId, this.#slackConversationOwner, claimId);
12613
+ return;
12614
+ }
12408
12615
  if (!await this.#ownsActiveSlackConversationIssue(claimed.issue)) {
12409
12616
  await this.#state.releaseConversationTurn(this.#workspaceId, conversationId, this.#slackConversationOwner, claimId);
12410
12617
  this.#increment('slackConversationTurnsSuppressedStaleOwner');
@@ -12488,10 +12695,13 @@ export class FactoryLoop {
12488
12695
  }
12489
12696
  }
12490
12697
  async #recordSlackConversationResume(session, result) {
12698
+ const sessionAgent = session.agent;
12699
+ if (!sessionAgent)
12700
+ return;
12491
12701
  const record = (await this.#batch()).getIssue(session.issue);
12492
12702
  if (!record)
12493
12703
  return;
12494
- const entry = [...record.agents.entries()].find(([name, tracked]) => name === session.agent.name || tracked.result?.name === session.agent.name);
12704
+ const entry = [...record.agents.entries()].find(([name, tracked]) => name === sessionAgent.name || tracked.result?.name === sessionAgent.name);
12495
12705
  if (!entry)
12496
12706
  return;
12497
12707
  const [previousName, tracked] = entry;
@@ -12817,7 +13027,14 @@ export class FactoryLoop {
12817
13027
  `Question: ${triageEscalationQuestion(decision, issue)}`,
12818
13028
  ].join('\n'),
12819
13029
  });
12820
- await this.#state.setSlackThread(this.#workspaceId, issueKey(decision.issue), root.threadId);
13030
+ const key = issueKey(decision.issue);
13031
+ await this.#state.setSlackThread(this.#workspaceId, key, root.threadId);
13032
+ await this.#state.setSlackThreadWatch(this.#workspaceId, key, {
13033
+ kind: 'triage',
13034
+ issue: { ...decision.issue },
13035
+ decision: structuredClone(decision),
13036
+ threadId: root.threadId,
13037
+ });
12821
13038
  const replayedResult = await this.#watchSlackThread(escalationWatchRecord(decision), root.threadId);
12822
13039
  this.#recordSlackWritebackSuccess('triage-escalation');
12823
13040
  return replayedResult;
@@ -12883,6 +13100,11 @@ export class FactoryLoop {
12883
13100
  if (!reply || !reply.isThreadReply || reply.threadTs !== threadId || reply.channelDir !== channelDir) {
12884
13101
  return;
12885
13102
  }
13103
+ if (allowPreExisting &&
13104
+ options.replayAfterMs !== undefined &&
13105
+ slackMessageReceivedAtMs(reply.messageTs, Number.MAX_SAFE_INTEGER) < options.replayAfterMs) {
13106
+ return;
13107
+ }
12886
13108
  const replyMessageKey = `${reply.threadTs}:${reply.messageTs}`;
12887
13109
  if (seenReplyMessages.has(replyMessageKey)) {
12888
13110
  this.#logger.debug?.('[factory] suppressed duplicate Slack reply message', { issue: record.issue.key, path });
@@ -13061,6 +13283,47 @@ export class FactoryLoop {
13061
13283
  this.#slackConversationTurns.schedule(conversationId);
13062
13284
  }
13063
13285
  }
13286
+ for (const [key, watch] of await this.#state.listSlackThreadWatches(this.#workspaceId)) {
13287
+ if (this.#slackWatchers.has(key) || this.#slackWatcherStarts.has(key))
13288
+ continue;
13289
+ if (watch.kind === 'terminal-grace' && watch.expiresAtMs <= this.#clock.now()) {
13290
+ await this.#stopSlackWatcher(watch.issue);
13291
+ continue;
13292
+ }
13293
+ await this.#state.setSlackThread(this.#workspaceId, key, watch.threadId);
13294
+ const watchRecord = escalationWatchRecord(watch.decision);
13295
+ if (watch.kind === 'terminal-grace') {
13296
+ const retiredAtMs = terminalSlackWatchRetiredAtMs(watch);
13297
+ if (watch.retiredAtMs !== retiredAtMs) {
13298
+ await this.#state.setSlackThreadWatch(this.#workspaceId, key, { ...watch, retiredAtMs });
13299
+ }
13300
+ this.#terminalSlackWatchIssues.add(key);
13301
+ const conversationId = slackConversationId(watch.threadId);
13302
+ await this.#slackConversationTurns.cancel(conversationId);
13303
+ try {
13304
+ await this.#surfaceUndeliveredSlackConversation(watch.threadId);
13305
+ await this.#state.clearConversationSession(this.#workspaceId, conversationId);
13306
+ }
13307
+ catch (error) {
13308
+ // The undelivered-reply receipt needs Slack writeback, which may be
13309
+ // unavailable at startup. That is retryable state maintenance for this
13310
+ // one thread, not a reason to abandon rehydration: aborting here would
13311
+ // leave every remaining thread watched by nobody. Keep the queued
13312
+ // replies (clearing them now would drop replies nobody was told about)
13313
+ // and carry on re-arming.
13314
+ this.#logger.warn?.('[factory] failed to settle undelivered Slack replies for terminal watch; will retry', { issue: watch.issue.key, error });
13315
+ this.#increment('slackTerminalWatchReceiptsDeferred');
13316
+ this.#scheduleSlackTerminalReceiptRetry(watch.issue, watch.threadId, watch.expiresAtMs);
13317
+ }
13318
+ await this.#rearmSlackWatcher(watchRecord, watch.threadId, {
13319
+ replayConversationReplies: true,
13320
+ replayAfterMs: retiredAtMs,
13321
+ });
13322
+ this.#scheduleSlackTerminalWatchExpiry(watch.issue, watch.expiresAtMs);
13323
+ continue;
13324
+ }
13325
+ await this.#rearmSlackWatcher(watchRecord, watch.threadId, { replayConversationReplies: true });
13326
+ }
13064
13327
  await this.#sweepWaitingClarifications();
13065
13328
  for (const [, waiting] of await this.#state.listWaitingClarifications(this.#workspaceId)) {
13066
13329
  if (!waiting.threadId)
@@ -13206,8 +13469,73 @@ export class FactoryLoop {
13206
13469
  this.#clarificationSweepTimer = timer;
13207
13470
  this.#clarificationSweepDueAtMs = dueAtMs;
13208
13471
  }
13472
+ // The terminal fence is the only thing that makes an in-flight reply route
13473
+ // answer "no active agent" instead of binding the retired thread to whatever
13474
+ // dispatch owns this key. Routes are chained per work unit, so awaiting the
13475
+ // newest one drains every reply queued behind it.
13476
+ async #drainSlackReplyRoutes(key) {
13477
+ // Snapshotting #slackReplyRoutes is not enough on its own. A reply handler
13478
+ // that is still inside its mount read when the drain starts registers its
13479
+ // route *after* the snapshot, so it would run once the fence is gone and
13480
+ // bind the retired thread to the next dispatch — the same escape one level
13481
+ // in. Bar *routing* for this key first (the bar and the registration are
13482
+ // both synchronous, so nothing can slip between them), then drain whatever
13483
+ // is already chained, then prove the set is empty before reporting success.
13484
+ // A barred reply still answers the human, and that writeback registers here
13485
+ // like any other effect, so the extra passes are what pick it up: quiescence
13486
+ // means every effect this work unit started has settled, not merely the ones
13487
+ // that existed when the drain began.
13488
+ const nested = this.#slackReplyRouteDrains.has(key);
13489
+ this.#slackReplyRouteDrains.add(key);
13490
+ try {
13491
+ for (let pass = 0; pass < SLACK_REPLY_ROUTE_DRAIN_PASSES; pass += 1) {
13492
+ const route = this.#slackReplyRoutes.get(key);
13493
+ if (!route)
13494
+ return true;
13495
+ try {
13496
+ await route;
13497
+ }
13498
+ catch (error) {
13499
+ // A route that *rejects* is not drained: the watcher replays it after
13500
+ // SLACK_REPLY_ROUTE_RETRY_MS, and that replay would land on the next
13501
+ // dispatch. Fail closed and let the caller keep the fence up rather
13502
+ // than leak a stale human reply onto fresh work.
13503
+ this.#logger.warn?.('[factory] in-flight Slack reply route did not drain; keeping terminal Slack fence', { issue: key, error });
13504
+ this.#increment('slackReplyRouteDrainsFailed');
13505
+ return false;
13506
+ }
13507
+ // The owner clears its own entry when it settles; retiring it here too
13508
+ // keeps the loop monotonic if that finally has not run yet.
13509
+ if (this.#slackReplyRoutes.get(key) === route)
13510
+ this.#slackReplyRoutes.delete(key);
13511
+ }
13512
+ // Not provably quiescent. Fail closed for the same reason as a rejection.
13513
+ this.#logger.warn?.('[factory] Slack reply routes did not quiesce; keeping terminal Slack fence', { issue: key });
13514
+ this.#increment('slackReplyRouteDrainsFailed');
13515
+ return false;
13516
+ }
13517
+ finally {
13518
+ if (!nested)
13519
+ this.#slackReplyRouteDrains.delete(key);
13520
+ }
13521
+ }
13209
13522
  async #stopSlackWatcher(issue) {
13210
13523
  const key = issueKey(issue);
13524
+ // Drain before clearing the fence. Clearing it first lets a reply that is
13525
+ // already mid-route — or one queued behind it — fall through the fence check
13526
+ // in #routeSlackConversationAnswerUnlocked and rebind the retired thread to
13527
+ // the next dispatch of this work unit.
13528
+ if (!await this.#drainSlackReplyRoutes(key))
13529
+ return false;
13530
+ this.#terminalSlackWatchIssues.delete(key);
13531
+ const expiryTimer = this.#slackTerminalWatchExpiryTimers.get(key);
13532
+ if (expiryTimer)
13533
+ clearTimeout(expiryTimer);
13534
+ this.#slackTerminalWatchExpiryTimers.delete(key);
13535
+ const receiptRetryTimer = this.#slackTerminalReceiptRetryTimers.get(key);
13536
+ if (receiptRetryTimer)
13537
+ clearTimeout(receiptRetryTimer);
13538
+ this.#slackTerminalReceiptRetryTimers.delete(key);
13211
13539
  const watcher = this.#slackWatchers.get(key);
13212
13540
  this.#slackWatchers.delete(key);
13213
13541
  const threadId = await this.#state.getSlackThread(this.#workspaceId, key);
@@ -13218,6 +13546,259 @@ export class FactoryLoop {
13218
13546
  await this.#state.clearConversationSession(this.#workspaceId, conversationId);
13219
13547
  }
13220
13548
  await this.#state.clearSlackThread(this.#workspaceId, key);
13549
+ await this.#state.clearSlackThreadWatch(this.#workspaceId, key);
13550
+ return true;
13551
+ }
13552
+ async #retireSlackWatcher(record) {
13553
+ const key = issueKey(record.issue);
13554
+ const threadId = await this.#state.getSlackThread(this.#workspaceId, key);
13555
+ if (!threadId) {
13556
+ await this.#stopSlackWatcher(record.issue);
13557
+ return;
13558
+ }
13559
+ const existingWatch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
13560
+ .find(([watchKey]) => watchKey === key)?.[1];
13561
+ const retiredAtMs = existingWatch?.kind === 'terminal-grace'
13562
+ ? terminalSlackWatchRetiredAtMs(existingWatch)
13563
+ : this.#clock.now();
13564
+ const expiresAtMs = existingWatch?.kind === 'terminal-grace'
13565
+ ? existingWatch.expiresAtMs
13566
+ : retiredAtMs + SLACK_TERMINAL_THREAD_GRACE_MS;
13567
+ await this.#state.setSlackThreadWatch(this.#workspaceId, key, {
13568
+ kind: 'terminal-grace',
13569
+ issue: { ...record.issue },
13570
+ decision: structuredClone(record.decision),
13571
+ threadId,
13572
+ retiredAtMs,
13573
+ expiresAtMs,
13574
+ });
13575
+ // A terminal thread must never retain a resumable session for an agent that
13576
+ // has already exited. Keep only the exact-thread listener so a late human
13577
+ // reply receives the explicit no-active-agent writeback below.
13578
+ this.#terminalSlackWatchIssues.add(key);
13579
+ await this.#slackReplyRoutes.get(key)?.catch(() => undefined);
13580
+ const conversationId = slackConversationId(threadId);
13581
+ await this.#slackConversationTurns.cancel(conversationId);
13582
+ try {
13583
+ await this.#surfaceUndeliveredSlackConversation(threadId);
13584
+ await this.#state.clearConversationSession(this.#workspaceId, conversationId);
13585
+ }
13586
+ catch (error) {
13587
+ // The receipt fails whenever another handler holds the claim or Slack
13588
+ // writeback is down — neither is a reason to abort retirement. Callers
13589
+ // reach here having already committed the terminal phase and dropped the
13590
+ // pending abandon reason, so a rejection escaping would strand the
13591
+ // registry rewrite, the GitHub watcher stop, and the queued next dispatch
13592
+ // with nothing left to re-run them. Keep the queued replies and let the
13593
+ // retry that owns this receipt settle it inside the grace window.
13594
+ this.#logger.warn?.('[factory] failed to settle undelivered Slack replies while retiring the watcher; will retry', { issue: record.issue.key, error });
13595
+ this.#increment('slackTerminalWatchReceiptsDeferred');
13596
+ this.#scheduleSlackTerminalReceiptRetry(record.issue, threadId, expiresAtMs);
13597
+ }
13598
+ if (!this.#slackWatchers.has(key) && !this.#stopping) {
13599
+ await this.#rearmSlackWatcher(record, threadId);
13600
+ }
13601
+ this.#scheduleSlackTerminalWatchExpiry(record.issue, expiresAtMs);
13602
+ this.#increment('slackTerminalWatchersRetained');
13603
+ }
13604
+ // The durable half of the same record. Every caller here follows the receipt
13605
+ // with a state write (clearing the session), and those two cannot be one
13606
+ // durable step: when the state write fails, the retry that owns it must not
13607
+ // read "replies still queued" as "the human has not been told" and post the
13608
+ // notice again. So the receipt is claimed before the provider write and marked
13609
+ // posted after it, and a retry finds it already settled.
13610
+ async #surfaceUndeliveredSlackConversation(threadId) {
13611
+ const conversationId = slackConversationId(threadId);
13612
+ const session = await this.#state.getConversationSession(this.#workspaceId, conversationId);
13613
+ const pendingCount = session
13614
+ ? session.pending.length + (session.delivery?.messages.length ?? 0)
13615
+ : 0;
13616
+ if (pendingCount === 0)
13617
+ return;
13618
+ if (session?.terminalReceipt?.posted) {
13619
+ this.#increment('slackTerminalReceiptsAlreadySettled');
13620
+ return;
13621
+ }
13622
+ if (!this.#slack)
13623
+ throw new Error(`Slack thread ${threadId} cannot surface undelivered replies without writeback`);
13624
+ const claimId = randomUUID();
13625
+ if (!await this.#state.claimConversationTerminalReceipt(this.#workspaceId, conversationId, claimId, this.#clock.now(), SLACK_TERMINAL_RECEIPT_CLAIM_LEASE_MS)) {
13626
+ const current = await this.#state.getConversationSession(this.#workspaceId, conversationId);
13627
+ if (current?.terminalReceipt?.posted) {
13628
+ this.#increment('slackTerminalReceiptsAlreadySettled');
13629
+ return;
13630
+ }
13631
+ // Another handler is mid-write. Fail closed so the queued replies survive
13632
+ // for whoever settles them rather than racing a second notice onto the
13633
+ // same thread.
13634
+ throw new Error(`Slack thread ${threadId} terminal receipt is claimed by another handler; retrying`);
13635
+ }
13636
+ const noun = pendingCount === 1 ? 'reply' : 'replies';
13637
+ const slack = this.#slack;
13638
+ try {
13639
+ await this.#withRenewedProviderLease('terminal Slack receipt', SLACK_TERMINAL_RECEIPT_CLAIM_LEASE_MS, () => this.#state.renewConversationTerminalReceipt(this.#workspaceId, conversationId, claimId, this.#clock.now()), () => slack.reply(threadId, `Factory could not deliver ${pendingCount} queued ${noun} because this work unit no longer has an active agent. Please continue on the linked issue or pull request.`));
13640
+ }
13641
+ catch (error) {
13642
+ await this.#state.releaseConversationTerminalReceipt(this.#workspaceId, conversationId, claimId);
13643
+ throw error;
13644
+ }
13645
+ if (!await this.#state.completeConversationTerminalReceipt(this.#workspaceId, conversationId, claimId)) {
13646
+ throw new Error(`Slack thread ${threadId} terminal receipt could not be recorded`);
13647
+ }
13648
+ this.#increment('slackConversationRepliesSurfacedTerminal');
13649
+ }
13650
+ // A claim only means something for as long as it outlives the work it covers.
13651
+ // A provider write can legitimately run past a fixed lease, at which point the
13652
+ // claim stops protecting the write it was taken for and a second handler can
13653
+ // post the same thing to the same human. Renewing on a heartbeat scopes the
13654
+ // lease to the work instead of to a guessed duration, and leaves the idle
13655
+ // timeout short enough that a holder that dies mid-write still frees it.
13656
+ //
13657
+ // The heartbeat is bounded on both ends, because a renewal loop that never
13658
+ // stops is a lock with no owner check: a provider write that hangs would hold
13659
+ // the receipt past every retry and past shutdown, and the human whose reply is
13660
+ // queued behind it would be told nothing until the process restarts. So it
13661
+ // stops at the ceiling and it stops when this daemon is stopping, and either
13662
+ // way it says so — from there the claim ages out on its own idle lease and
13663
+ // becomes reclaimable. The write may still land afterwards and duplicate the
13664
+ // notice; a reply nobody can ever reclaim is the worse of the two.
13665
+ async #withRenewedProviderLease(label, leaseMs, renew, run) {
13666
+ const renewUntilMs = this.#clock.now() + SLACK_PROVIDER_LEASE_MAX_RENEWAL_MS;
13667
+ let renewalStopped = false;
13668
+ let renewalInFlight = false;
13669
+ const stopRenewing = (counter, reason) => {
13670
+ renewalStopped = true;
13671
+ this.#increment(counter);
13672
+ this.#logger.warn?.(`[factory] ${label} lease will not be renewed further (${reason}); ` +
13673
+ 'its claim expires and the queued replies return to whoever retries them');
13674
+ };
13675
+ const heartbeat = setInterval(() => {
13676
+ if (renewalInFlight || renewalStopped)
13677
+ return;
13678
+ if (this.#stopping) {
13679
+ stopRenewing('slackProviderReceiptLeaseRenewalsStoppedForShutdown', 'shutting down');
13680
+ return;
13681
+ }
13682
+ if (this.#clock.now() >= renewUntilMs) {
13683
+ stopRenewing('slackProviderReceiptLeaseRenewalsExpired', `provider write exceeded ${SLACK_PROVIDER_LEASE_MAX_RENEWAL_MS}ms`);
13684
+ return;
13685
+ }
13686
+ renewalInFlight = true;
13687
+ void renew()
13688
+ .then((renewed) => {
13689
+ if (renewed)
13690
+ return;
13691
+ // Losing the lease mid-write is not recoverable from in here: the
13692
+ // write may already have landed. Stop renewing and let the caller's
13693
+ // completion check fail closed, which keeps the queued replies for
13694
+ // whoever holds the claim now.
13695
+ renewalStopped = true;
13696
+ this.#increment('slackProviderReceiptLeasesLost');
13697
+ this.#logger.warn?.(`[factory] ${label} lease was lost while its provider write was in flight`);
13698
+ })
13699
+ .catch((error) => this.#logger.warn?.(`[factory] ${label} lease renewal failed`, {
13700
+ error: describeError(error).errorMessage,
13701
+ }))
13702
+ .finally(() => { renewalInFlight = false; });
13703
+ }, Math.max(1_000, Math.floor(leaseMs / 3)));
13704
+ heartbeat.unref?.();
13705
+ try {
13706
+ return await run();
13707
+ }
13708
+ finally {
13709
+ clearInterval(heartbeat);
13710
+ }
13711
+ }
13712
+ // A terminal receipt that could not be written leaves the queued replies
13713
+ // pending with the human who wrote them told nothing. That is retryable
13714
+ // maintenance this daemon owns, not work to leave for the next restart: the
13715
+ // grace watch is the only window in which the receipt can still land on the
13716
+ // retired thread, so keep reattempting inside it and give up when it closes.
13717
+ #scheduleSlackTerminalReceiptRetry(issue, threadId, expiresAtMs, attempt = 0) {
13718
+ if (this.#stopping)
13719
+ return;
13720
+ const key = issueKey(issue);
13721
+ const existing = this.#slackTerminalReceiptRetryTimers.get(key);
13722
+ if (existing)
13723
+ clearTimeout(existing);
13724
+ this.#slackTerminalReceiptRetryTimers.delete(key);
13725
+ const remainingMs = expiresAtMs - this.#clock.now();
13726
+ if (remainingMs <= 0) {
13727
+ this.#increment('slackTerminalWatchReceiptsAbandoned');
13728
+ return;
13729
+ }
13730
+ const backoffMs = Math.min(SLACK_TERMINAL_RECEIPT_RETRY_MAX_MS, SLACK_TERMINAL_RECEIPT_RETRY_MS * 2 ** Math.min(attempt, 16));
13731
+ const timer = setTimeout(() => {
13732
+ this.#slackTerminalReceiptRetryTimers.delete(key);
13733
+ void this.#retrySlackTerminalReceipt(issue, threadId, attempt);
13734
+ }, Math.max(0, Math.min(backoffMs, remainingMs)));
13735
+ timer.unref?.();
13736
+ this.#slackTerminalReceiptRetryTimers.set(key, timer);
13737
+ }
13738
+ async #retrySlackTerminalReceipt(issue, threadId, attempt) {
13739
+ if (this.#stopping)
13740
+ return;
13741
+ const key = issueKey(issue);
13742
+ const watch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
13743
+ .find(([watchKey]) => watchKey === key)?.[1];
13744
+ // The grace watch is gone (expired, or the work unit reopened): the thread
13745
+ // this receipt would settle no longer exists, so there is nothing to say.
13746
+ if (watch?.kind !== 'terminal-grace' || watch.threadId !== threadId)
13747
+ return;
13748
+ try {
13749
+ await this.#surfaceUndeliveredSlackConversation(threadId);
13750
+ await this.#state.clearConversationSession(this.#workspaceId, slackConversationId(threadId));
13751
+ this.#increment('slackTerminalWatchReceiptsRecovered');
13752
+ }
13753
+ catch (error) {
13754
+ this.#logger.warn?.('[factory] terminal Slack receipt retry failed; rescheduling', {
13755
+ issue: issue.key,
13756
+ error,
13757
+ });
13758
+ this.#increment('slackTerminalWatchReceiptRetryFailures');
13759
+ this.#scheduleSlackTerminalReceiptRetry(issue, threadId, watch.expiresAtMs, attempt + 1);
13760
+ }
13761
+ }
13762
+ #scheduleSlackTerminalWatchExpiry(issue, expiresAtMs, retryDelayMs) {
13763
+ if (this.#stopping)
13764
+ return;
13765
+ const key = issueKey(issue);
13766
+ const existing = this.#slackTerminalWatchExpiryTimers.get(key);
13767
+ if (existing)
13768
+ clearTimeout(existing);
13769
+ const timer = setTimeout(() => {
13770
+ this.#slackTerminalWatchExpiryTimers.delete(key);
13771
+ void this.#expireSlackTerminalWatcher(issue, expiresAtMs).catch((error) => {
13772
+ this.#logger.warn?.('[factory] failed to expire terminal Slack reply watcher; retrying', {
13773
+ issue: issue.key,
13774
+ error,
13775
+ });
13776
+ this.#scheduleSlackTerminalWatchExpiry(issue, expiresAtMs, SLACK_REPLY_ROUTE_RETRY_MS);
13777
+ });
13778
+ }, retryDelayMs ?? Math.max(0, expiresAtMs - this.#clock.now()));
13779
+ timer.unref?.();
13780
+ this.#slackTerminalWatchExpiryTimers.set(key, timer);
13781
+ }
13782
+ async #expireSlackTerminalWatcher(issue, expiresAtMs) {
13783
+ const key = issueKey(issue);
13784
+ const watch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
13785
+ .find(([watchKey]) => watchKey === key)?.[1];
13786
+ if (watch?.kind !== 'terminal-grace' || watch.expiresAtMs !== expiresAtMs)
13787
+ return;
13788
+ if (watch.expiresAtMs > this.#clock.now()) {
13789
+ this.#scheduleSlackTerminalWatchExpiry(issue, watch.expiresAtMs);
13790
+ return;
13791
+ }
13792
+ // #stopSlackWatcher fails closed when an in-flight reply route will not
13793
+ // drain, leaving the watch and its terminal fence in place. Counting that as
13794
+ // an expiration retires the watch in the metrics while the real one lives
13795
+ // on unwatched by any expiry timer, so reschedule and count only on success.
13796
+ if (!await this.#stopSlackWatcher(issue)) {
13797
+ this.#increment('slackTerminalWatchExpiriesDeferred');
13798
+ this.#scheduleSlackTerminalWatchExpiry(issue, expiresAtMs, SLACK_REPLY_ROUTE_RETRY_MS);
13799
+ return;
13800
+ }
13801
+ this.#increment('slackTerminalWatchersExpired');
13221
13802
  }
13222
13803
  async #readSlackReply(path) {
13223
13804
  try {
@@ -13282,33 +13863,147 @@ export class FactoryLoop {
13282
13863
  await this.#wakeWaitingClarification(clarificationKey, claimed);
13283
13864
  return;
13284
13865
  }
13866
+ return await this.#routeSlackConversationAnswer(record, reply, text, clarificationKey);
13867
+ }
13868
+ async #routeSlackConversationAnswer(record, reply, text, clarificationKey) {
13869
+ if (this.#slackReplyRouteDrains.has(clarificationKey)) {
13870
+ // The terminal fence for this work unit is being drained right now.
13871
+ // Registering an ordinary route here would put it past the drain's
13872
+ // snapshot and run it once the fence is gone. Answer it the way the fence
13873
+ // would have — but as a tracked effect, because this writeback is still a
13874
+ // side effect of this work unit. Left untracked it is the same escape one
13875
+ // level further in: the drain reports quiescence without it, the watcher
13876
+ // stop clears the retry timer that owns this reply, and a slow or failed
13877
+ // receipt leaves the human told nothing at all.
13878
+ this.#increment('slackReplyRoutesFencedDuringDrain');
13879
+ return await this.#trackSlackWorkUnitEffect(clarificationKey, async () => {
13880
+ await this.#writeUnroutableSlackReply(reply.threadTs);
13881
+ return undefined;
13882
+ });
13883
+ }
13884
+ return await this.#trackSlackWorkUnitEffect(clarificationKey, () => this.#routeSlackConversationAnswerUnlocked(record, reply, text, clarificationKey));
13885
+ }
13886
+ // Every Slack side effect a work unit makes on its own behalf runs through
13887
+ // here, so #slackReplyRoutes stays the single record the terminal drain
13888
+ // consults. Effects are chained per work unit: awaiting the newest one drains
13889
+ // everything queued behind it, and a rejection propagates to the drain, which
13890
+ // fails closed rather than tearing the effect's retry path down.
13891
+ async #trackSlackWorkUnitEffect(key, run) {
13892
+ const preceding = this.#slackReplyRoutes.get(key);
13893
+ const effect = (async () => {
13894
+ await preceding?.catch(() => undefined);
13895
+ return await run();
13896
+ })();
13897
+ this.#slackReplyRoutes.set(key, effect);
13898
+ try {
13899
+ return await effect;
13900
+ }
13901
+ finally {
13902
+ if (this.#slackReplyRoutes.get(key) === effect)
13903
+ this.#slackReplyRoutes.delete(key);
13904
+ }
13905
+ }
13906
+ async #routeSlackConversationAnswerUnlocked(record, reply, text, clarificationKey) {
13907
+ if (this.#terminalSlackWatchIssues.has(clarificationKey)) {
13908
+ await this.#writeUnroutableSlackReply(reply.threadTs);
13909
+ return;
13910
+ }
13285
13911
  const conversationId = slackConversationId(reply.threadTs);
13286
- const conversation = await this.#state.getConversationSession(this.#workspaceId, conversationId);
13912
+ let conversation = await this.#state.getConversationSession(this.#workspaceId, conversationId);
13913
+ let liveRecord;
13914
+ if (!conversation) {
13915
+ liveRecord = (await this.#batch()).getIssue(record.issue);
13916
+ if (liveRecord && !liveRecord.dryRun) {
13917
+ await this.#ensureSlackConversationSession(liveRecord, reply.threadTs);
13918
+ conversation = await this.#state.getConversationSession(this.#workspaceId, conversationId);
13919
+ }
13920
+ }
13287
13921
  if (conversation && issueKey(conversation.issue) === clarificationKey) {
13922
+ const replyId = `${reply.threadTs}:${reply.messageTs}`;
13288
13923
  const queued = await this.#state.appendConversationMessage(this.#workspaceId, conversationId, {
13289
- id: `${reply.threadTs}:${reply.messageTs}`,
13924
+ id: replyId,
13290
13925
  text,
13291
13926
  receivedAtMs: slackMessageReceivedAtMs(reply.messageTs, this.#clock.now()),
13292
13927
  providerSequence: reply.messageTs,
13293
13928
  author: reply.author,
13294
13929
  });
13295
- if (!queued) {
13930
+ const durable = queued ?? await this.#state.getConversationSession(this.#workspaceId, conversationId);
13931
+ if (!durable || !durable.processedMessageIds.includes(replyId)) {
13932
+ throw new Error(`Slack reply ${replyId} was not durably queued`);
13933
+ }
13934
+ if (!(durable.acknowledgedMessageIds ?? []).includes(replyId)) {
13935
+ const acknowledgementClaimId = randomUUID();
13936
+ const acknowledgementClaimed = await this.#state.claimConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId, this.#clock.now(), SLACK_REPLY_ACKNOWLEDGEMENT_LEASE_MS);
13937
+ if (acknowledgementClaimed) {
13938
+ try {
13939
+ if (!this.#slack)
13940
+ throw new Error(`Slack reply ${replyId} cannot be acknowledged without writeback`);
13941
+ const owner = durable.agent?.role === 'babysitter'
13942
+ ? 'the PR babysitter'
13943
+ : durable.agent
13944
+ ? 'the issue implementer'
13945
+ : 'an issue agent';
13946
+ const receipt = durable.agent
13947
+ ? `Factory received this reply and durably queued it for ${owner}.`
13948
+ : 'Factory received and durably stored this reply; it will route when an issue agent is resumable.';
13949
+ const slack = this.#slack;
13950
+ // Same lease scope as the terminal receipt: this claim covers a
13951
+ // provider write that can outrun any fixed duration, so it is
13952
+ // renewed for as long as that write is actually running.
13953
+ await this.#withRenewedProviderLease('Slack reply acknowledgement', SLACK_REPLY_ACKNOWLEDGEMENT_LEASE_MS, () => this.#state.renewConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId, this.#clock.now()), () => slack.reply(reply.threadTs, receipt));
13954
+ if (!await this.#state.completeConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId)) {
13955
+ throw new Error(`Slack reply ${replyId} receipt could not be recorded`);
13956
+ }
13957
+ this.#increment('slackConversationRepliesAcknowledged');
13958
+ }
13959
+ catch (error) {
13960
+ await this.#state.releaseConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId);
13961
+ throw error;
13962
+ }
13963
+ }
13964
+ else {
13965
+ const acknowledgementState = await this.#state.getConversationSession(this.#workspaceId, conversationId);
13966
+ if (!(acknowledgementState?.acknowledgedMessageIds ?? []).includes(replyId)) {
13967
+ throw new Error(`Slack reply ${replyId} receipt is claimed by another handler; retrying`);
13968
+ }
13969
+ }
13970
+ }
13971
+ if (queued) {
13972
+ this.#increment('slackConversationRepliesQueued');
13973
+ }
13974
+ else {
13296
13975
  this.#increment('slackConversationDuplicateRepliesSuppressed');
13297
- return;
13298
13976
  }
13299
- this.#increment('slackConversationRepliesQueued');
13300
- this.#slackConversationTurns.schedule(conversationId);
13977
+ const pending = durable.pending.some((message) => message.id === replyId) ||
13978
+ Boolean(durable.delivery?.messages.some((message) => message.id === replyId));
13979
+ if (pending && durable.agent) {
13980
+ this.#slackConversationTurns.schedule(conversationId);
13981
+ }
13982
+ else if (pending) {
13983
+ this.#increment('slackConversationRepliesWaitingForOwner');
13984
+ }
13301
13985
  return;
13302
13986
  }
13303
- const liveRecord = (await this.#batch()).getIssue(record.issue);
13987
+ liveRecord ??= (await this.#batch()).getIssue(record.issue);
13304
13988
  if (!liveRecord || liveRecord.dryRun) {
13305
13989
  if (isTriageEscalationWatchRecord(record)) {
13306
13990
  return await this.#handleTriageEscalationSlackAnswer(record, text);
13307
13991
  }
13308
- this.#increment('slackAnswersIgnoredNoInFlight');
13992
+ await this.#writeUnroutableSlackReply(reply.threadTs);
13309
13993
  return;
13310
13994
  }
13311
13995
  this.#increment('slackAnswersIgnoredNoConversationSession');
13996
+ if (this.#slack) {
13997
+ await this.#slack.reply(reply.threadTs, 'Factory received this reply but could not create a durable agent route. It will remain replayable; please also continue on the linked issue or pull request.');
13998
+ this.#increment('slackAnswersUnroutableVisible');
13999
+ }
14000
+ }
14001
+ async #writeUnroutableSlackReply(threadId) {
14002
+ this.#increment('slackAnswersIgnoredNoInFlight');
14003
+ if (!this.#slack)
14004
+ return;
14005
+ await this.#slack.reply(threadId, 'Factory received this reply but could not route it because this work unit no longer has an active agent. Please continue on the linked issue or pull request.');
14006
+ this.#increment('slackAnswersUnroutableVisible');
13312
14007
  }
13313
14008
  async #wakeWaitingClarification(key, waiting) {
13314
14009
  const existing = this.#clarificationWakeInFlight.get(key);
@@ -13680,6 +14375,7 @@ export class FactoryLoop {
13680
14375
  const batch = await this.#batch();
13681
14376
  if (batch.isInFlight(record.issue) || batch.isQueued(record.issue)) {
13682
14377
  this.#increment('slackTriageAnswersIgnoredAlreadyActive');
14378
+ await this.#state.clearSlackThreadWatch(this.#workspaceId, issueKey(record.issue));
13683
14379
  return;
13684
14380
  }
13685
14381
  if (await this.#dispatchBlockReason(record.issue)) {
@@ -13696,6 +14392,10 @@ export class FactoryLoop {
13696
14392
  if (hasDispatchableRoute(decision)) {
13697
14393
  this.#pendingSlackClarifications.set(issueKey(decision.issue), text);
13698
14394
  const result = await this.#startOrQueueSlackClarifiedDecision(dispatchAfterSlackClarification(decision, escalationReason));
14395
+ const active = await this.#batch();
14396
+ if (result || active.isInFlight(decision.issue) || active.isQueued(decision.issue)) {
14397
+ await this.#state.clearSlackThreadWatch(this.#workspaceId, issueKey(record.issue));
14398
+ }
13699
14399
  this.#increment('slackTriageAnswersDispatchedWithRemainingEscalation');
13700
14400
  return result;
13701
14401
  }
@@ -13708,6 +14408,10 @@ export class FactoryLoop {
13708
14408
  }
13709
14409
  this.#pendingSlackClarifications.set(issueKey(decision.issue), text);
13710
14410
  const result = await this.#startOrQueueSlackClarifiedDecision(decision);
14411
+ const active = await this.#batch();
14412
+ if (result || active.isInFlight(decision.issue) || active.isQueued(decision.issue)) {
14413
+ await this.#state.clearSlackThreadWatch(this.#workspaceId, issueKey(record.issue));
14414
+ }
13711
14415
  this.#increment('slackTriageAnswersDispatched');
13712
14416
  return result;
13713
14417
  }
@@ -16218,8 +16922,8 @@ const normalizeGithubRepo = (repo, defaultOwner) => {
16218
16922
  }
16219
16923
  return `${owner}/${repo}`;
16220
16924
  };
16221
- const githubPullRequestBody = (issue, preview) => [
16222
- issue.description,
16925
+ const githubPullRequestBody = (issue, preview, sessionRef) => [
16926
+ stripTrajectoryPointers(issue.description),
16223
16927
  '',
16224
16928
  isGithubIssue(issue) && /^\d+$/u.test(issue.key)
16225
16929
  ? `Fixes #${issue.key}`
@@ -16229,7 +16933,25 @@ const githubPullRequestBody = (issue, preview) => [
16229
16933
  `Live preview: ${preview.url}`,
16230
16934
  'Access: Tailscale tailnet membership and the tailnet grants/ACLs are required; this URL is not public.',
16231
16935
  ] : []),
16936
+ '',
16937
+ renderTrajectoryPointer({
16938
+ ...trajectoryWorkUnitForIssue(issue),
16939
+ sessionRef,
16940
+ }),
16232
16941
  ].join('\n').trim();
16942
+ const trajectoryWorkUnitForIssue = (issue) => {
16943
+ const github = githubIssueSourceRef(issue);
16944
+ if (github) {
16945
+ return {
16946
+ workUnitId: `${github.owner}/${github.repo}#${github.number}`,
16947
+ workUnitSurface: 'github',
16948
+ };
16949
+ }
16950
+ if (isRealLinearIssue(issue)) {
16951
+ return { workUnitId: issue.key, workUnitSurface: 'linear' };
16952
+ }
16953
+ return { workUnitId: `factory:${issue.uuid}`, workUnitSurface: 'factory' };
16954
+ };
16233
16955
  // The broker rejects re-registering a name it never released on exit
16234
16956
  // (relay#1116-family) with a 500 "agent '<name>' already exists". Detect it from
16235
16957
  // the structured payload or the message so resume can treat it as terminal
@@ -16258,6 +16980,9 @@ const slackMessageReceivedAtMs = (messageTs, fallback) => {
16258
16980
  const seconds = Number(messageTs);
16259
16981
  return Number.isFinite(seconds) && seconds > 0 ? Math.floor(seconds * 1_000) : fallback;
16260
16982
  };
16983
+ const terminalSlackWatchRetiredAtMs = (watch) => typeof watch.retiredAtMs === 'number' && Number.isFinite(watch.retiredAtMs)
16984
+ ? watch.retiredAtMs
16985
+ : Math.max(0, watch.expiresAtMs - SLACK_TERMINAL_THREAD_GRACE_MS);
16261
16986
  const eventIdentity = (event) => {
16262
16987
  const record = event;
16263
16988
  const rawId = record.id ?? record.event_id ?? record.seq;
@@ -16569,6 +17294,55 @@ export class LiveDispatchStateChangedError extends Error {
16569
17294
  export function isLiveDispatchStateChangedError(error) {
16570
17295
  return error instanceof LiveDispatchStateChangedError;
16571
17296
  }
17297
+ /** How deep to follow `cause` when classifying a wrapped failure. */
17298
+ const PASS_FATAL_CAUSE_DEPTH = 4;
17299
+ /**
17300
+ * Whether `error`, or anything it wraps, is an instance of `type`.
17301
+ * `contextualError` and the fleet control-plane guard both rethrow wrapped, so
17302
+ * classification has to follow the cause chain rather than trust the outermost
17303
+ * type.
17304
+ */
17305
+ const wrapsErrorOfType = (error, type, depth = 0) => {
17306
+ if (depth > PASS_FATAL_CAUSE_DEPTH || !(error instanceof Error))
17307
+ return false;
17308
+ if (error instanceof type)
17309
+ return true;
17310
+ return wrapsErrorOfType(error.cause, type, depth + 1);
17311
+ };
17312
+ /**
17313
+ * How many *unclassified* per-item failures without an intervening successful
17314
+ * dispatch end the pass. Named per-item conditions (a lifecycle claim refusal,
17315
+ * a live-state race) never count toward it and never reset it: those
17316
+ * legitimately affect many units at once and are exactly the benign case #292
17317
+ * asks the loop to survive, so they are neither evidence of a pass-wide fault
17318
+ * nor evidence against one.
17319
+ */
17320
+ const UNCLASSIFIED_DISPATCH_FAILURE_LIMIT = 5;
17321
+ /**
17322
+ * Failures the loop recognizes as belonging to one work unit. They are always
17323
+ * skippable and are exempt from the consecutive-failure fuse.
17324
+ */
17325
+ const isClassifiedPerItemDispatchFailure = (error) => error instanceof LiveDispatchStateChangedError ||
17326
+ error instanceof DispatchLifecycleClaimRefusedError;
17327
+ /**
17328
+ * The run-report reason recorded for a work unit the pass could not dispatch.
17329
+ *
17330
+ * `factory run-once` serializes the whole report to stdout, so this string is
17331
+ * a public surface: it stays a fixed classification plus an allowlisted error
17332
+ * class name, never raw provider text or filesystem paths. The full message
17333
+ * goes to the operator log instead, the same split
17334
+ * `describeControlPlaneError` makes for circuit state.
17335
+ */
17336
+ const perItemDispatchSkipReason = (error) => {
17337
+ if (error instanceof LiveDispatchStateChangedError)
17338
+ return 'live state changed during dispatch';
17339
+ if (error instanceof DispatchLifecycleClaimRefusedError) {
17340
+ return error.refusal === 'terminal'
17341
+ ? 'dispatch lifecycle already terminal'
17342
+ : 'dispatch lifecycle owned by another publisher';
17343
+ }
17344
+ return `dispatch failed (${telemetryErrorClass(error)})`;
17345
+ };
16572
17346
  const triageEscalationQuestion = (decision, issue) => {
16573
17347
  const routedRepos = decision.routes.map((route) => route.repo).filter(Boolean);
16574
17348
  const subject = issue?.title?.trim() || decision.issue.key;