@agent-relay/factory 0.1.72 → 0.1.74

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/README.md +9 -0
  2. package/dist/cli/diagnose.d.ts +18 -2
  3. package/dist/cli/diagnose.d.ts.map +1 -1
  4. package/dist/cli/diagnose.js +94 -5
  5. package/dist/cli/diagnose.js.map +1 -1
  6. package/dist/cli/fleet.d.ts.map +1 -1
  7. package/dist/cli/fleet.js +10 -0
  8. package/dist/cli/fleet.js.map +1 -1
  9. package/dist/cli/teammate-mcp.d.ts +19 -0
  10. package/dist/cli/teammate-mcp.d.ts.map +1 -0
  11. package/dist/cli/teammate-mcp.js +102 -0
  12. package/dist/cli/teammate-mcp.js.map +1 -0
  13. package/dist/environments/load-profile.d.ts +4 -4
  14. package/dist/environments/stack-descriptor.d.ts +17 -17
  15. package/dist/environments/verification-stack-descriptor.d.ts +32 -32
  16. package/dist/fleet/control-plane-circuit.d.ts +10 -0
  17. package/dist/fleet/control-plane-circuit.d.ts.map +1 -1
  18. package/dist/fleet/control-plane-circuit.js +9 -1
  19. package/dist/fleet/control-plane-circuit.js.map +1 -1
  20. package/dist/fleet/create-fleet.d.ts +2 -0
  21. package/dist/fleet/create-fleet.d.ts.map +1 -1
  22. package/dist/fleet/create-fleet.js +1 -0
  23. package/dist/fleet/create-fleet.js.map +1 -1
  24. package/dist/fleet/internal-fleet-client.d.ts +29 -1
  25. package/dist/fleet/internal-fleet-client.d.ts.map +1 -1
  26. package/dist/fleet/internal-fleet-client.js +161 -11
  27. package/dist/fleet/internal-fleet-client.js.map +1 -1
  28. package/dist/fleet/relay-fleet-client.d.ts +43 -2
  29. package/dist/fleet/relay-fleet-client.d.ts.map +1 -1
  30. package/dist/fleet/relay-fleet-client.js +264 -19
  31. package/dist/fleet/relay-fleet-client.js.map +1 -1
  32. package/dist/fleet/teammates.d.ts +46 -0
  33. package/dist/fleet/teammates.d.ts.map +1 -0
  34. package/dist/fleet/teammates.js +424 -0
  35. package/dist/fleet/teammates.js.map +1 -0
  36. package/dist/index.d.ts +11 -2
  37. package/dist/index.d.ts.map +1 -1
  38. package/dist/index.js +5 -0
  39. package/dist/index.js.map +1 -1
  40. package/dist/mount/relayfile-cloud-mount-client.d.ts +4 -1
  41. package/dist/mount/relayfile-cloud-mount-client.d.ts.map +1 -1
  42. package/dist/mount/relayfile-cloud-mount-client.js +18 -6
  43. package/dist/mount/relayfile-cloud-mount-client.js.map +1 -1
  44. package/dist/mount/relayfile-github-connection-write.d.ts +3 -2
  45. package/dist/mount/relayfile-github-connection-write.d.ts.map +1 -1
  46. package/dist/mount/relayfile-github-connection-write.js +33 -0
  47. package/dist/mount/relayfile-github-connection-write.js.map +1 -1
  48. package/dist/node/factory-node-runtime.d.ts +17 -0
  49. package/dist/node/factory-node-runtime.d.ts.map +1 -0
  50. package/dist/node/factory-node-runtime.js +100 -0
  51. package/dist/node/factory-node-runtime.js.map +1 -0
  52. package/dist/node/factory-node.d.ts +10 -1
  53. package/dist/node/factory-node.d.ts.map +1 -1
  54. package/dist/node/factory-node.js +13 -2
  55. package/dist/node/factory-node.js.map +1 -1
  56. package/dist/node/factory-persona-card.d.ts +38 -0
  57. package/dist/node/factory-persona-card.d.ts.map +1 -0
  58. package/dist/node/factory-persona-card.js +163 -0
  59. package/dist/node/factory-persona-card.js.map +1 -0
  60. package/dist/node/factory.node.d.ts +1 -1
  61. package/dist/observability/events.d.ts +38 -38
  62. package/dist/orchestrator/dispatch-failure-reason.d.ts +60 -0
  63. package/dist/orchestrator/dispatch-failure-reason.d.ts.map +1 -0
  64. package/dist/orchestrator/dispatch-failure-reason.js +127 -0
  65. package/dist/orchestrator/dispatch-failure-reason.js.map +1 -0
  66. package/dist/orchestrator/factory.d.ts.map +1 -1
  67. package/dist/orchestrator/factory.js +1175 -103
  68. package/dist/orchestrator/factory.js.map +1 -1
  69. package/dist/orchestrator/public-health.d.ts.map +1 -1
  70. package/dist/orchestrator/public-health.js +152 -6
  71. package/dist/orchestrator/public-health.js.map +1 -1
  72. package/dist/orchestrator/release-error.d.ts +26 -0
  73. package/dist/orchestrator/release-error.d.ts.map +1 -0
  74. package/dist/orchestrator/release-error.js +33 -0
  75. package/dist/orchestrator/release-error.js.map +1 -0
  76. package/dist/ports/fleet.d.ts +104 -0
  77. package/dist/ports/fleet.d.ts.map +1 -1
  78. package/dist/ports/fleet.js +10 -1
  79. package/dist/ports/fleet.js.map +1 -1
  80. package/dist/ports/index.d.ts +4 -3
  81. package/dist/ports/index.d.ts.map +1 -1
  82. package/dist/ports/index.js +1 -1
  83. package/dist/ports/index.js.map +1 -1
  84. package/dist/ports/mount.d.ts +10 -2
  85. package/dist/ports/mount.d.ts.map +1 -1
  86. package/dist/ports/writeback.d.ts +45 -4
  87. package/dist/ports/writeback.d.ts.map +1 -1
  88. package/dist/state/watch-state-document.js +4 -1
  89. package/dist/state/watch-state-document.js.map +1 -1
  90. package/dist/testing/fakes.d.ts +8 -3
  91. package/dist/testing/fakes.d.ts.map +1 -1
  92. package/dist/testing/fakes.js +25 -1
  93. package/dist/testing/fakes.js.map +1 -1
  94. package/dist/triage/schema.d.ts +14 -14
  95. package/dist/types.d.ts +170 -14
  96. package/dist/types.d.ts.map +1 -1
  97. package/dist/writeback/github.d.ts +17 -9
  98. package/dist/writeback/github.d.ts.map +1 -1
  99. package/dist/writeback/github.js +331 -9
  100. package/dist/writeback/github.js.map +1 -1
  101. package/dist/writeback/linear.d.ts +5 -1
  102. package/dist/writeback/linear.d.ts.map +1 -1
  103. package/dist/writeback/linear.js +89 -1
  104. package/dist/writeback/linear.js.map +1 -1
  105. package/package.json +4 -2
@@ -1,3 +1,4 @@
1
+ import { AsyncLocalStorage } from 'node:async_hooks';
1
2
  import { randomUUID } from 'node:crypto';
2
3
  import { readFile } from 'node:fs/promises';
3
4
  import { dirname, isAbsolute, resolve } from 'node:path';
@@ -36,6 +37,8 @@ import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../obse
36
37
  import { telemetryErrorClass } from '../observability/error-class.js';
37
38
  import { derivedReadinessReconcileState, publicHealthFromHeartbeat, readinessReconcileInFlightMs, } from './public-health.js';
38
39
  import { factorySweepSkipReasonCounts } from './sweep-skip-reason.js';
40
+ import { factoryDispatchFailureReasonCodeForErrorClass, factoryDispatchFailureReasonCounts, } from './dispatch-failure-reason.js';
41
+ import { isAgentAlreadyGoneOnRelease } from './release-error.js';
39
42
  import { boundedRunCostTotal, CostLedger } from '../cost/ledger.js';
40
43
  import { createTicketDispatchDelivery } from '../delivery/ticket-dispatch.js';
41
44
  import { canonicalTrajectorySessionRef, renderTrajectoryPointer, stripTrajectoryPointers, } from '../trajectory.js';
@@ -143,6 +146,12 @@ const CLARIFICATION_ESCALATION_LEASE_MS = 2 * 60_000;
143
146
  const CLARIFICATION_ESCALATION_RETRY_MS = 5_000;
144
147
  const CLARIFICATION_STALE_WARN_MS = 7 * 24 * 60 * 60_000;
145
148
  const STOP_TEARDOWN_TIMEOUT_MS = 2_500;
149
+ // A rejected post-spawn dispatch normally gets a final opportunity to undo an
150
+ // external claim before shutdown relinquishes its lifecycle lease. Provider
151
+ // writes are not guaranteed to settle, so that opportunity must stay inside a
152
+ // bounded shutdown budget; a successor can recover from the retained durable
153
+ // lifecycle after this process releases its local agents.
154
+ const STOP_REJECTED_DISPATCH_DRAIN_TIMEOUT_MS = 2_500;
146
155
  const DISPATCH_LIFECYCLE_LEASE_MS = 5 * 60_000;
147
156
  const DISPATCH_LIFECYCLE_RENEW_MS = 60_000;
148
157
  const DISPATCH_LIFECYCLE_RETRY_MS = 1_000;
@@ -347,6 +356,16 @@ class ReadinessReconcileTimeoutError extends Error {
347
356
  this.name = 'ReadinessReconcileTimeoutError';
348
357
  }
349
358
  }
359
+ class PostSpawnDispatchWaitRejectedError extends Error {
360
+ issueKey;
361
+ compensationError;
362
+ constructor(issueKey, compensationError) {
363
+ super(`Post-spawn dispatch waits were rejected for ${issueKey}`);
364
+ this.issueKey = issueKey;
365
+ this.compensationError = compensationError;
366
+ this.name = 'PostSpawnDispatchWaitRejectedError';
367
+ }
368
+ }
350
369
  const realClock = {
351
370
  now: () => Date.now(),
352
371
  sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
@@ -354,6 +373,26 @@ const realClock = {
354
373
  export function createFactory(config, ports) {
355
374
  return new FactoryLoop(FactoryConfigSchema.parse(config), ports);
356
375
  }
376
+ /**
377
+ * The discovery pass a tree read was issued by, carried per async call (#363
378
+ * review, CodeRabbit).
379
+ *
380
+ * `#discoverySweepEpoch` says a sweep is in flight; it cannot say the read in
381
+ * hand belongs to it. In live mode an event drain reaches the very same
382
+ * enumeration helpers — `#handlePrChange` -> `#advanceMergedPrToDone` ->
383
+ * `#findMergeAdvanceIssueForPr` -> `#githubIssuePaths()` is the concrete path —
384
+ * so a drain's full-root walk would land in a concurrent sweep's ratio while
385
+ * measuring a different instant. That is enough to hide a mount that went
386
+ * silent mid-sweep.
387
+ *
388
+ * A drain does not inherit this store: its continuation begins at the
389
+ * subscription callback, outside the `run()` below. Threading an epoch argument
390
+ * through `#ingestGithubIssues` / `#handleGithubIssueChange` /
391
+ * `#findGithubIssueMirror` / `#loadLinearMirrorCandidates` would express the
392
+ * same fact and put a parameter on every hot path that must never be passed
393
+ * wrong.
394
+ */
395
+ const discoveryEnumerationPass = new AsyncLocalStorage();
357
396
  export class FactoryLoop {
358
397
  #config;
359
398
  #mount;
@@ -553,18 +592,30 @@ export class FactoryLoop {
553
592
  #readinessReconcileLastError;
554
593
  #readinessReconcileLastErrorClass;
555
594
  /**
556
- * The last *completed* sweep's arithmetic (#355).
595
+ * The last *enumerating* sweep's arithmetic (#355).
557
596
  *
558
- * Held as one record rather than four fields so it can only ever be replaced
597
+ * Held as one record rather than three fields so it can only ever be replaced
559
598
  * whole: publishing a `dispatched` from one pass beside a `candidates` from
560
599
  * another would be worse than publishing neither, since the whole use of
561
600
  * these numbers is comparing them to each other.
562
601
  *
563
- * `undefined` until a sweep completes, and never initialised to zeroes —
602
+ * `undefined` until a sweep enumerates, and never initialised to zeroes —
564
603
  * "this daemon has not finished a sweep" and "a sweep finished and found
565
604
  * nothing" are the two readings #355 has to tell apart.
566
605
  */
567
606
  #readinessReconcileLastSweep;
607
+ /**
608
+ * Whether the MOST RECENT pass deferred, tracked apart from the counts above
609
+ * (#358 review, CodeRabbit — Major, and right).
610
+ *
611
+ * A deferred pass enumerates nothing and settles in milliseconds, so folding
612
+ * it into the snapshot overwrote the last real sweep's numbers with zeroes.
613
+ * On a container where another process holds the lease for any length of time
614
+ * — the #347/#349 condition — every pass would publish `candidates: 0` and the
615
+ * last actual enumeration would be unrecoverable, destroying the measurement
616
+ * this whole change exists to provide.
617
+ */
618
+ #readinessReconcileLastSweepDeferred;
568
619
  #liveEventQueue = [];
569
620
  #liveEventDrainScheduled = false;
570
621
  #liveEventDrainActive = false;
@@ -577,6 +628,29 @@ export class FactoryLoop {
577
628
  #previewSweepTimer;
578
629
  #previewSweepInFlight;
579
630
  #completionInFlight = new Set();
631
+ #issueWritebackInFlight = new Map();
632
+ // A fast completion can make terminal issue state visible while dispatch is
633
+ // still performing its post-spawn readiness read. Completion publishes its
634
+ // provider receipt, then waits for that read to classify the state as owned
635
+ // or foreign before it releases agents under a terminal success reason.
636
+ #postSpawnIssueObservations = new Map();
637
+ // The post-spawn ready read and following dispatch claim form one local
638
+ // write boundary. A completion arriving first makes dispatch wait and
639
+ // re-read; one arriving after claim entry waits for the claim to finish.
640
+ #postSpawnDispatchClaimFences = new Map();
641
+ // Shutdown must not lose a local placement merely because a rejected
642
+ // provider claim unwinds before #releaseInFlightAgents snapshots the batch.
643
+ // The capture set makes both interleavings explicit: the dispatch catch may
644
+ // remove a record only after shutdown has copied its agents, otherwise the
645
+ // release sweep removes it after the copy is safe.
646
+ #shutdownReleaseCapturedDispatches = new Set();
647
+ #postSpawnDispatchesRejectedDuringStop = new Set();
648
+ // A rejected provider claim is unsafe to release until its conditional
649
+ // rollback succeeds or a newer provider status supersedes it. This runtime
650
+ // set closes the race before the durable dispatchClaim flag is persisted.
651
+ #uncompensatedDispatchClaims = new Set();
652
+ #pendingDispatchClaims = new Set();
653
+ #dispatchClaimSettlementsInFlight = new Set();
580
654
  #agentExitsInFlight = new Map();
581
655
  #reconciledAgentExitsActive = 0;
582
656
  #reconciledAgentExitWaiters = [];
@@ -665,6 +739,10 @@ export class FactoryLoop {
665
739
  #discoveryOverloadError;
666
740
  /** Relayfile operations this sweep has been shed on. */
667
741
  #discoverySweepOverloads = 0;
742
+ /** Tree reads this sweep issued that the backend served (#351 follow-up). */
743
+ #discoverySweepTreeReads = 0;
744
+ /** How many of those were served with zero entries. */
745
+ #discoverySweepEmptyTreeReads = 0;
668
746
  /**
669
747
  * The longest `Retry-After` any operation in this sweep advertised.
670
748
  *
@@ -971,6 +1049,15 @@ export class FactoryLoop {
971
1049
  }
972
1050
  }
973
1051
  async #start(opts) {
1052
+ // These sets coordinate one stop snapshot with dispatches unwinding during
1053
+ // that same stop. They are not durable lifecycle state: retaining their
1054
+ // keys across a later start would make the next stop mistake a fresh
1055
+ // dispatch for one already captured by the previous shutdown.
1056
+ this.#shutdownReleaseCapturedDispatches.clear();
1057
+ this.#postSpawnDispatchesRejectedDuringStop.clear();
1058
+ this.#uncompensatedDispatchClaims.clear();
1059
+ this.#pendingDispatchClaims.clear();
1060
+ this.#dispatchClaimSettlementsInFlight.clear();
974
1061
  this.#stopping = false;
975
1062
  this.#startMode = opts.mode ?? 'live';
976
1063
  const issueSource = await this.#issueSource();
@@ -1085,15 +1172,32 @@ export class FactoryLoop {
1085
1172
  async stop() {
1086
1173
  this.#started = false;
1087
1174
  this.#stopping = true;
1175
+ // These waits are armed before every planned spawn returns. Resolve them
1176
+ // before the first shutdown await: a fast first agent may be completing
1177
+ // while a later spawn is hung, and #drainAgentExitsInFlight would otherwise
1178
+ // wait forever on a resolver stranded inside that dispatch stack.
1179
+ const postSpawnWaitKeys = new Set([
1180
+ ...this.#postSpawnIssueObservations.keys(),
1181
+ ...this.#postSpawnDispatchClaimFences.keys(),
1182
+ ]);
1183
+ const rejectedClaimDispatches = new Set();
1184
+ for (const [key, fence] of this.#postSpawnDispatchClaimFences) {
1185
+ if (!fence.claimStarted)
1186
+ continue;
1187
+ const dispatchKeyPrefix = `${key}:live:`;
1188
+ for (const [dispatchKey, dispatch] of this.#dispatchInFlight) {
1189
+ if (dispatchKey.startsWith(dispatchKeyPrefix))
1190
+ rejectedClaimDispatches.add(dispatch);
1191
+ }
1192
+ }
1193
+ for (const key of postSpawnWaitKeys)
1194
+ this.#settlePostSpawnDispatchWaits(key, false, 'stop');
1088
1195
  if (this.#babysitterResourceDeliveryRetryTimer)
1089
1196
  clearTimeout(this.#babysitterResourceDeliveryRetryTimer);
1090
1197
  this.#babysitterResourceDeliveryRetryTimer = undefined;
1091
1198
  if (this.#babysitterResourceSubscriptionRenewTimer)
1092
1199
  clearTimeout(this.#babysitterResourceSubscriptionRenewTimer);
1093
1200
  this.#babysitterResourceSubscriptionRenewTimer = undefined;
1094
- if (this.#dispatchLifecycleRenewTimer)
1095
- clearInterval(this.#dispatchLifecycleRenewTimer);
1096
- this.#dispatchLifecycleRenewTimer = undefined;
1097
1201
  if (this.#heldAgentDeadlineTimer)
1098
1202
  clearTimeout(this.#heldAgentDeadlineTimer);
1099
1203
  this.#heldAgentDeadlineTimer = undefined;
@@ -1126,12 +1230,25 @@ export class FactoryLoop {
1126
1230
  await this.#previewSweepInFlight;
1127
1231
  this.#stoppingHeartbeatRefreshActive = await this.#stopLiveHeartbeat('stopping');
1128
1232
  try {
1233
+ // Only dispatches that had entered the provider-claim boundary can leave
1234
+ // an external claim behind. Drain those rejected promises before
1235
+ // relinquishing their lifecycle leases or tearing down the fleet/mount.
1236
+ // A pre-claim spawn hang is deliberately absent from this set, so it
1237
+ // cannot hold shutdown open.
1238
+ await this.#drainRejectedClaimDispatchesForStop(rejectedClaimDispatches);
1239
+ await this.#persistOutstandingClaimCancellationBlocks();
1240
+ // Keep renewing ownership through the potentially slow provider drain
1241
+ // and cancellation-fence handoff. Stopping renewal earlier can let the
1242
+ // lease expire before the fence is saved, allowing a successor to reap
1243
+ // agents while the old provider write can still land.
1129
1244
  // Relinquish durable ownership before waiting on mount-backed lifecycle
1130
1245
  // drives. A slow Relayfile scan must not consume the shutdown deadline
1131
1246
  // while every issue remains fenced to a publisher that is already
1132
- // stopping. The owner/epoch fence makes any late completion from those
1133
- // drives harmless; a second sweep below catches claims racing this one.
1134
- await this.#releaseOwnedDispatchLifecycleLeases();
1247
+ // stopping. Preserve leases for provider claims whose bounded drain
1248
+ // expired: the original caller still needs its epoch to persist a late
1249
+ // conclusive settlement. A second sweep below catches claims racing this
1250
+ // one.
1251
+ await this.#releaseOwnedDispatchLifecycleLeases((key) => this.#pendingDispatchClaims.has(key) || this.#dispatchClaimSettlementsInFlight.has(key));
1135
1252
  await Promise.allSettled([...this.#dispatchLifecycleDrives]);
1136
1253
  // Fence every source of new clarification work before touching the fleet.
1137
1254
  // A wake already past the fence is allowed to unwind, and is awaited
@@ -1154,13 +1271,18 @@ export class FactoryLoop {
1154
1271
  // non-durable (local/internal) records; terminal completion performs the
1155
1272
  // normal remote release before clearing the lifecycle.
1156
1273
  await this.#releaseInFlightAgents('factory-stopped', { preserveDurable: true });
1157
- await this.#releaseOwnedDispatchLifecycleLeases();
1274
+ if (this.#dispatchLifecycleRenewTimer)
1275
+ clearInterval(this.#dispatchLifecycleRenewTimer);
1276
+ this.#dispatchLifecycleRenewTimer = undefined;
1277
+ await this.#releaseOwnedDispatchLifecycleLeases((key) => this.#pendingDispatchClaims.has(key) || this.#dispatchClaimSettlementsInFlight.has(key));
1158
1278
  if (this.#livePollTimer)
1159
1279
  clearTimeout(this.#livePollTimer);
1160
1280
  this.#livePollTimer = undefined;
1161
1281
  this.#livePollInFlight = false;
1162
1282
  this.#liveEventQueue.length = 0;
1163
1283
  this.#completionInFlight.clear();
1284
+ this.#postSpawnIssueObservations.clear();
1285
+ this.#postSpawnDispatchClaimFences.clear();
1164
1286
  this.#babysitterSpawned.clear();
1165
1287
  this.#babysitterPr.clear();
1166
1288
  this.#babysitterIssueRefs.clear();
@@ -1207,10 +1329,13 @@ export class FactoryLoop {
1207
1329
  await this.#fleet.dispose();
1208
1330
  }
1209
1331
  finally {
1332
+ if (this.#dispatchLifecycleRenewTimer)
1333
+ clearInterval(this.#dispatchLifecycleRenewTimer);
1334
+ this.#dispatchLifecycleRenewTimer = undefined;
1210
1335
  this.#stoppingHeartbeatRefreshActive = false;
1211
1336
  }
1212
1337
  }
1213
- async #releaseOwnedDispatchLifecycleLeases() {
1338
+ async #releaseOwnedDispatchLifecycleLeases(preserve = () => false) {
1214
1339
  // The epoch cache is an execution optimization, not the durable ownership
1215
1340
  // authority. Error/fence paths may evict a cached epoch while its persisted
1216
1341
  // lease is still ours, so enumerate state before shutdown relinquishment.
@@ -1221,6 +1346,8 @@ export class FactoryLoop {
1221
1346
  }
1222
1347
  }
1223
1348
  for (const [key, epoch] of owned) {
1349
+ if (preserve(key))
1350
+ continue;
1224
1351
  await this.#state.releaseDispatchLifecycleLease(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch);
1225
1352
  if (this.#dispatchLifecycleEpochs.get(key) === epoch) {
1226
1353
  this.#dispatchLifecycleEpochs.delete(key);
@@ -1244,6 +1371,215 @@ export class FactoryLoop {
1244
1371
  await Promise.allSettled([...this.#clarificationQuestionDeliveryInFlight.values()]);
1245
1372
  }
1246
1373
  }
1374
+ async #drainRejectedClaimDispatchesForStop(dispatches) {
1375
+ if (dispatches.size === 0)
1376
+ return;
1377
+ let timer;
1378
+ const drained = Promise.allSettled([...dispatches]).then(() => true);
1379
+ // A durable fleet can recover ownership after this process exits. Local
1380
+ // and other non-durable fleets cannot: releasing their placements before
1381
+ // the provider claim has compensated would leave no successor able to
1382
+ // finish the lifecycle, and a late write could strand the issue in its
1383
+ // claimed state. Keep draining those fleets without the shutdown escape
1384
+ // hatch.
1385
+ if (this.#fleet.durableOwnership !== true) {
1386
+ await drained;
1387
+ return;
1388
+ }
1389
+ const timedOut = new Promise((resolve) => {
1390
+ timer = setTimeout(() => resolve(false), STOP_REJECTED_DISPATCH_DRAIN_TIMEOUT_MS);
1391
+ timer.unref?.();
1392
+ });
1393
+ const completed = await Promise.race([drained, timedOut]);
1394
+ if (timer)
1395
+ clearTimeout(timer);
1396
+ if (completed)
1397
+ return;
1398
+ this.#increment('postSpawnDispatchClaimDrainTimeouts');
1399
+ this.#logger.warn?.('[factory] rejected post-spawn dispatch compensation timed out; continuing shutdown', {
1400
+ dispatches: dispatches.size,
1401
+ timeoutMs: STOP_REJECTED_DISPATCH_DRAIN_TIMEOUT_MS,
1402
+ });
1403
+ }
1404
+ async #persistOutstandingClaimCancellationBlocks() {
1405
+ const batch = await this.#batch();
1406
+ for (const record of batch.inFlight) {
1407
+ const key = dispatchLifecycleKey(record.issue);
1408
+ if (!this.#pendingDispatchClaims.has(key))
1409
+ continue;
1410
+ // The bounded durable drain expired before the provider operation could
1411
+ // report whether rollback was safe. Persist the provisional release
1412
+ // fence while this publisher still owns the lifecycle lease, so a
1413
+ // successor cannot mistake the unresolved claim for abandonable work.
1414
+ const persisted = await this.#retainUncompensatedDispatchClaim(record, new Error('Rejected dispatch claim compensation did not settle before shutdown'), { preferExistingError: true, pending: true });
1415
+ if (!persisted) {
1416
+ const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
1417
+ if (durable?.dispatchClaim?.cancellationPending !== true) {
1418
+ throw new Error(`Unable to persist unresolved dispatch-claim fence for ${record.issue.key}; refusing shutdown handoff`);
1419
+ }
1420
+ }
1421
+ }
1422
+ }
1423
+ async #awaitRejectedClaimCompensation(settled) {
1424
+ let timer;
1425
+ const timedOut = new Promise((resolve) => {
1426
+ timer = setTimeout(() => resolve(false), STOP_REJECTED_DISPATCH_DRAIN_TIMEOUT_MS);
1427
+ timer.unref?.();
1428
+ });
1429
+ const compensated = await Promise.race([settled, timedOut]);
1430
+ if (timer)
1431
+ clearTimeout(timer);
1432
+ return compensated;
1433
+ }
1434
+ /**
1435
+ * Re-adopt this publisher's own lease epoch after a refused lifecycle write.
1436
+ *
1437
+ * `#saveDispatchLifecycle` cannot tell "the store rejected our fence" from
1438
+ * "the store declined this row's contents", so it evicts the cached epoch for
1439
+ * both. Only the first is a loss of ownership. Re-read the durable row and
1440
+ * take the epoch back when the lease is still ours and still live; a row that
1441
+ * is gone, terminal, expired, or owned by someone else keeps the eviction,
1442
+ * because for those the eviction was correct.
1443
+ */
1444
+ async #recoverDispatchLifecycleLeaseEpoch(key) {
1445
+ if (this.#dispatchLifecycleEpochs.has(key))
1446
+ return true;
1447
+ const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
1448
+ const lease = lifecycle?.lease;
1449
+ if (!lifecycle || !lease || isTerminalDispatchLifecycle(lifecycle))
1450
+ return false;
1451
+ if (lease.owner !== this.#dispatchLifecycleOwner)
1452
+ return false;
1453
+ if (lease.leaseUntilMs <= this.#clock.now())
1454
+ return false;
1455
+ this.#dispatchLifecycleEpochs.set(key, lease.epoch);
1456
+ this.#increment('dispatchLifecycleLeaseEpochsRecovered');
1457
+ return true;
1458
+ }
1459
+ /**
1460
+ * Persist a dispatch-claim fence without letting a refused write disarm the
1461
+ * lease the next one needs.
1462
+ *
1463
+ * The shutdown handoff deliberately probes with a provisional
1464
+ * `cancellationPending` fence, and a store may refuse that row on its
1465
+ * contents while this process still holds the lease. Losing the epoch there
1466
+ * would make every later write for this lifecycle short-circuit on
1467
+ * `epoch === undefined` — including the conclusive compensated settlement
1468
+ * that clears the block and frees the agents and the batch slot. That turns
1469
+ * one refused probe into a lifecycle retained until restart, so bracket the
1470
+ * write with epoch recovery instead.
1471
+ */
1472
+ async #persistDispatchClaimFence(record) {
1473
+ const key = dispatchLifecycleKey(record.issue);
1474
+ await this.#recoverDispatchLifecycleLeaseEpoch(key);
1475
+ const saved = await this.#saveDispatchLifecycle(record, 'running');
1476
+ if (!saved)
1477
+ await this.#recoverDispatchLifecycleLeaseEpoch(key);
1478
+ return saved;
1479
+ }
1480
+ async #retainUncompensatedDispatchClaim(record, error, opts = {}) {
1481
+ const key = dispatchLifecycleKey(record.issue);
1482
+ const alreadyBlocked = record.dispatchClaim?.cancellationBlocked === true;
1483
+ const errorMessage = describeError(error).errorMessage;
1484
+ this.#uncompensatedDispatchClaims.add(key);
1485
+ const { cancellationPending: _pending, ...existingClaim } = record.dispatchClaim ?? { state: 'degraded' };
1486
+ record.dispatchClaim = {
1487
+ ...existingClaim,
1488
+ state: 'degraded',
1489
+ cancellationBlocked: true,
1490
+ ...(opts.pending ? { cancellationPending: true } : {}),
1491
+ write: 'rejected dispatch claim rollback',
1492
+ error: opts.preferExistingError ? record.dispatchClaim?.error ?? errorMessage : errorMessage,
1493
+ deadLettered: true,
1494
+ updatedAtMs: this.#clock.now(),
1495
+ };
1496
+ this.#dispatchClaimStatuses.set(key, record.dispatchClaim);
1497
+ if (!alreadyBlocked) {
1498
+ this.#increment('postSpawnDispatchClaimRecoveryRetentions');
1499
+ this.#logger.warn?.('[factory] retaining dispatch lifecycle because rejected claim compensation is unproven', {
1500
+ issue: record.issue.key,
1501
+ });
1502
+ }
1503
+ if (this.#usesDurableDispatchLifecycle()) {
1504
+ return await this.#persistDispatchClaimFence(record);
1505
+ }
1506
+ else {
1507
+ await this.#writeDispatchClaimRegistry(record.issue);
1508
+ return true;
1509
+ }
1510
+ }
1511
+ async #clearDispatchCancellationBlock(record) {
1512
+ const key = dispatchLifecycleKey(record.issue);
1513
+ if (record.dispatchClaim?.cancellationBlocked !== true) {
1514
+ this.#uncompensatedDispatchClaims.delete(key);
1515
+ return true;
1516
+ }
1517
+ const previousClaim = record.dispatchClaim;
1518
+ const { cancellationBlocked: _blocked, cancellationPending: _pending, ...claim } = previousClaim;
1519
+ record.dispatchClaim = claim;
1520
+ this.#dispatchClaimStatuses.set(key, claim);
1521
+ try {
1522
+ const persisted = this.#usesDurableDispatchLifecycle()
1523
+ ? await this.#persistDispatchClaimFence(record)
1524
+ : await this.#writeDispatchClaimRegistry(record.issue).then(() => true);
1525
+ if (!persisted) {
1526
+ record.dispatchClaim = previousClaim;
1527
+ this.#dispatchClaimStatuses.set(key, previousClaim);
1528
+ return false;
1529
+ }
1530
+ this.#uncompensatedDispatchClaims.delete(key);
1531
+ return true;
1532
+ }
1533
+ catch (error) {
1534
+ record.dispatchClaim = previousClaim;
1535
+ this.#dispatchClaimStatuses.set(key, previousClaim);
1536
+ throw error;
1537
+ }
1538
+ }
1539
+ async #dispatchClaimBlocksAbandonment(record) {
1540
+ const key = dispatchLifecycleKey(record.issue);
1541
+ if (!this.#uncompensatedDispatchClaims.has(key) && record.dispatchClaim?.cancellationBlocked !== true) {
1542
+ return false;
1543
+ }
1544
+ // A provider request that was still unresolved at handoff can land after
1545
+ // any current-state read. No observation is proof of supersession until
1546
+ // the original caller records that the operation conclusively settled.
1547
+ if (record.dispatchClaim?.cancellationPending === true)
1548
+ return true;
1549
+ try {
1550
+ const issue = await this.#readIssue(record.issue.path);
1551
+ if (!issue)
1552
+ return true;
1553
+ if (isGithubIssue(issue)) {
1554
+ if (!this.#githubWriteback.getIssueStatus)
1555
+ return true;
1556
+ const status = await this.#githubWriteback.getIssueStatus(issue, {
1557
+ requireFresh: true,
1558
+ freshAfterMs: record.dispatchClaim?.claimStartedAtMs,
1559
+ });
1560
+ if (!status || status === 'in-progress')
1561
+ return true;
1562
+ }
1563
+ else {
1564
+ if (!this.#linear.getIssueStateId)
1565
+ return true;
1566
+ const implementingStateId = this.#states.idFor(issue.team, 'agentImplementing');
1567
+ const currentStateId = await this.#linear.getIssueStateId(issue);
1568
+ if (!implementingStateId || !currentStateId || currentStateId === implementingStateId)
1569
+ return true;
1570
+ }
1571
+ if (!await this.#clearDispatchCancellationBlock(record))
1572
+ return true;
1573
+ return false;
1574
+ }
1575
+ catch (error) {
1576
+ this.#logger.warn?.('[factory] unable to verify whether a blocked dispatch claim was superseded', {
1577
+ issue: record.issue.key,
1578
+ error: describeError(error).errorMessage,
1579
+ });
1580
+ return true;
1581
+ }
1582
+ }
1247
1583
  async #boundedStopTeardown(label, teardown) {
1248
1584
  let timer;
1249
1585
  const action = Promise.resolve()
@@ -1331,9 +1667,11 @@ export class FactoryLoop {
1331
1667
  // cold container it is the first — and for the next interval, only —
1332
1668
  // sweep whose counts exist. Leaving it unrecorded would make a daemon
1333
1669
  // that has completed a full pass still read as "never ran" (#355).
1334
- this.#recordReadinessSweepOutcome(await this.runOnce());
1670
+ const report = await this.runOnce();
1335
1671
  this.#readinessReconcileLastDurationMs = this.#elapsedSince(backfillStartedAtMs);
1336
- this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
1672
+ const completedAtMs = this.#clock.now();
1673
+ this.#readinessReconcileLastCompletedAtMs = completedAtMs;
1674
+ this.#recordReadinessSweepOutcome(report, completedAtMs);
1337
1675
  }
1338
1676
  catch (error) {
1339
1677
  this.#readinessReconcileLastDurationMs = this.#elapsedSince(backfillStartedAtMs);
@@ -1559,20 +1897,28 @@ export class FactoryLoop {
1559
1897
  const report = await this.#runOnceWithReadinessDeadline();
1560
1898
  this.#readinessReconcileConsecutiveFailures = 0;
1561
1899
  this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
1562
- this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
1900
+ const completedAtMs = this.#clock.now();
1901
+ this.#readinessReconcileLastCompletedAtMs = completedAtMs;
1563
1902
  this.#readinessReconcileLastError = undefined;
1564
1903
  this.#readinessReconcileLastErrorClass = undefined;
1565
1904
  // The three integers below have gone to stdout since this loop existed,
1566
1905
  // and stdout does not reach the deployed container's operator (#355).
1567
1906
  // Publishing them is what lets a reader tell a sweep that saw eligible
1568
1907
  // work and rejected it from one that never pulled it at all.
1569
- this.#recordReadinessSweepOutcome(report);
1908
+ this.#recordReadinessSweepOutcome(report, completedAtMs);
1570
1909
  this.#logger.info?.('[factory] periodic readiness reconciliation completed', {
1571
1910
  durationMs: this.#readinessReconcileLastDurationMs,
1572
1911
  candidates: report.pulled.length,
1573
1912
  dispatched: report.dispatched.length,
1574
1913
  skipped: report.skipped.length,
1575
- skipReasons: this.#readinessReconcileLastSweep?.skipReasons,
1914
+ // THIS pass's breakdown, never the retained snapshot (#359 review,
1915
+ // codex P2). Logging the retained one beside a deferred pass's zeroes
1916
+ // produced a line that contradicted its own arithmetic —
1917
+ // `skipped: 0` next to a non-empty breakdown — and this log is what a
1918
+ // local operator reads.
1919
+ skipReasons: factorySweepSkipReasonCounts(report.skipped),
1920
+ dispatchFailures: report.skipped.filter((entry) => entry.code === 'dispatch-failed').length,
1921
+ dispatchFailureReasons: factoryDispatchFailureReasonCounts(report.skipped),
1576
1922
  discoveryDeferred: report.discoveryDeferred,
1577
1923
  });
1578
1924
  }
@@ -1596,6 +1942,10 @@ export class FactoryLoop {
1596
1942
  this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
1597
1943
  this.#readinessReconcileLastFailureAtMs = this.#clock.now();
1598
1944
  this.#readinessReconcileLastError = errorMessage;
1945
+ // This failure is now the latest settled pass. A deferral marker left by
1946
+ // an older pass would falsely describe this one as lease contention when
1947
+ // the timestamps/error below prove that it acquired the lease and failed.
1948
+ this.#readinessReconcileLastSweepDeferred = undefined;
1599
1949
  // The class, unlike the message, is publishable: #295 puts it on the
1600
1950
  // unauthenticated health surface through the same allowlist.
1601
1951
  this.#readinessReconcileLastErrorClass = telemetryErrorClass(error);
@@ -2341,6 +2691,8 @@ export class FactoryLoop {
2341
2691
  this.#discoverySweepLeaseLost = false;
2342
2692
  this.#discoveryOverloadError = undefined;
2343
2693
  this.#discoverySweepOverloads = 0;
2694
+ this.#discoverySweepTreeReads = 0;
2695
+ this.#discoverySweepEmptyTreeReads = 0;
2344
2696
  this.#discoverySweepRetryAfterSeconds = undefined;
2345
2697
  this.#discoverySweepProgress = false;
2346
2698
  this.#startDiscoverySweepRenewal(claim.lease.epoch);
@@ -2421,6 +2773,8 @@ export class FactoryLoop {
2421
2773
  this.#discoverySweepStartedAtMs = undefined;
2422
2774
  this.#discoveryOverloadError = undefined;
2423
2775
  this.#discoverySweepOverloads = 0;
2776
+ this.#discoverySweepTreeReads = 0;
2777
+ this.#discoverySweepEmptyTreeReads = 0;
2424
2778
  this.#discoverySweepRetryAfterSeconds = undefined;
2425
2779
  this.#discoverySweepProgress = false;
2426
2780
  // This sweep is over either way (committed, deferred, or lease lost) —
@@ -2539,13 +2893,19 @@ export class FactoryLoop {
2539
2893
  this.#dependencyGithubPathsByIdentity = undefined;
2540
2894
  this.#dependencyLinearTreeLoaded = false;
2541
2895
  const issueSource = await this.#issueSource();
2896
+ // The sweep's own discovery pass, and the only region whose full-root
2897
+ // walks may enter the tree-read ratio (#363 review). Everything reached
2898
+ // from here inherits the marker; a concurrently running drain does not.
2899
+ const enumerate = (fn) => this.#discoverySweepEpoch === undefined
2900
+ ? fn()
2901
+ : discoveryEnumerationPass.run({ epoch: this.#discoverySweepEpoch }, fn);
2542
2902
  if (issueSource === 'linear') {
2543
- await this.#ingestGithubIssues({ dryRun });
2903
+ await enumerate(() => this.#ingestGithubIssues({ dryRun }));
2544
2904
  }
2545
2905
  else {
2546
2906
  await this.#ensureGithubIngestionReady();
2547
2907
  }
2548
- const paths = await this.#readyIssuePaths();
2908
+ const paths = await enumerate(() => this.#readyIssuePaths());
2549
2909
  const orphanRecovery = issueSource === 'github'
2550
2910
  ? await this.#githubOrphanRecoveryContext(dryRun)
2551
2911
  : undefined;
@@ -2703,6 +3063,10 @@ export class FactoryLoop {
2703
3063
  continue;
2704
3064
  }
2705
3065
  const recoveredIdentity = recoveredOrphan ? githubIssueRefIdentity(issueRef(issue)) : undefined;
3066
+ // Reset per work unit, and advanced by assignment immediately before
3067
+ // each stage rather than inferred in the catch: the whole value of the
3068
+ // `unclassified-*` codes is that they name the stage honestly (#355).
3069
+ let attemptPhase = 'gate';
2706
3070
  try {
2707
3071
  if (recoveredOrphan) {
2708
3072
  const dispatchBlock = await this.#dispatchBlockReason(issue);
@@ -2723,8 +3087,10 @@ export class FactoryLoop {
2723
3087
  });
2724
3088
  continue;
2725
3089
  }
3090
+ attemptPhase = 'triage';
2726
3091
  const decision = await this.triageIssue(issue);
2727
3092
  triaged.push(decision);
3093
+ attemptPhase = 'dispatch';
2728
3094
  const result = await this.dispatch(decision, { dryRun });
2729
3095
  // A completed dispatch — even one that parks or escalates the issue —
2730
3096
  // proves the pipeline still works, so the fuse below starts over.
@@ -2825,6 +3191,11 @@ export class FactoryLoop {
2825
3191
  issue: issueRef(issue),
2826
3192
  reason: perItemDispatchSkipReason(error),
2827
3193
  code: 'dispatch-failed',
3194
+ // The publishable half of the same classification. `reason` is the
3195
+ // operator's sentence and stays off the health surface; this token
3196
+ // is what tells a reader watching `dispatch-failed: 5` which of
3197
+ // five very different bugs they are looking at (#355).
3198
+ failureCode: perItemDispatchFailureCode(error, attemptPhase),
2828
3199
  });
2829
3200
  continue;
2830
3201
  }
@@ -2839,6 +3210,10 @@ export class FactoryLoop {
2839
3210
  dispatched,
2840
3211
  skipped,
2841
3212
  dryRun,
3213
+ // Read before `#runOnceWithDiscoveryFence`'s finally resets them: this
3214
+ // is still inside that try, so the counts are this sweep's own.
3215
+ treeReads: this.#discoverySweepTreeReads,
3216
+ emptyTreeReads: this.#discoverySweepEmptyTreeReads,
2842
3217
  slackDegraded: this.#slackDegraded,
2843
3218
  ...(orphanRecoveryDegraded ? { orphanRecoveryDegraded } : {}),
2844
3219
  };
@@ -3744,6 +4119,10 @@ export class FactoryLoop {
3744
4119
  // that expects a specific PR to appear, or an escalation/comment-replay
3745
4120
  // scan that must not miss a marker or reply that landed after the cache
3746
4121
  // was populated. Those callers must omit `cache` and pay for a fresh list.
4122
+ //
4123
+ // `enumeration` marks the call sites that ARE the readiness sweep's
4124
+ // discovery pass — a full-root walk looking for candidate work. Only those
4125
+ // feed the tree-read pair below, and only while a sweep holds the lease.
3747
4126
  async #listRelayfileTree(prefix, phase, opts = {}) {
3748
4127
  if (this.#discoverySweepLeaseLost) {
3749
4128
  throw new Error('discovery sweep lease was lost; refusing another tree request');
@@ -3762,6 +4141,46 @@ export class FactoryLoop {
3762
4141
  logStart: true,
3763
4142
  logComplete: true,
3764
4143
  });
4144
+ // #351 follow-up: tree reads the backend SERVED, and how many of them it
4145
+ // answered with nothing.
4146
+ //
4147
+ // The per-call deadline made a hung dependency loud. This is its
4148
+ // companion: a mount that starts returning empty trees instead of hanging
4149
+ // raises no timeout, no failure and no `lastError`, and produces a sweep
4150
+ // that completes `healthy` having dispatched nothing.
4151
+ //
4152
+ // BOTH numbers, because one is not a signal. A healthy sweep lists two
4153
+ // path forms per repo and only one of them exists, so an empty read is
4154
+ // ordinary and a bare count of them fires constantly. What separates the
4155
+ // fault is the RATIO: `emptyTreeReads === treeReads` means the mount
4156
+ // served nothing at all, which `candidates: 0` cannot distinguish from a
4157
+ // workspace that simply has no ready work.
4158
+ //
4159
+ // SCOPED TWICE, and both guards are load-bearing (#363 review, codex P2).
4160
+ // The ratio is only readable if every read in it came from the same
4161
+ // question. In live mode a Slack identity lookup, a PR-confirmation poll
4162
+ // or a comment-replay scan can run concurrently with `runOnce()` — each is
4163
+ // a `listTree`, none is discovery, and one populated result among them
4164
+ // makes `emptyTreeReads < treeReads` on an all-empty sweep, silently
4165
+ // erasing exactly the signal this pair exists to raise. `enumeration`
4166
+ // keeps the numerator and denominator to the discovery pass; the epoch
4167
+ // check keeps a discovery walk issued outside any sweep — startup
4168
+ // backfill, most obviously — out of a sweep's totals.
4169
+ // `enumeration` says this is a full-root discovery walk rather than a point
4170
+ // lookup; the context says THIS sweep's discovery pass is what issued it.
4171
+ // Both, because either alone admits a read the ratio cannot use. Compared
4172
+ // by value rather than presence: an absent store and an absent epoch are
4173
+ // both `undefined` and must not read as a match.
4174
+ const issuingPass = discoveryEnumerationPass.getStore();
4175
+ if (opts.enumeration &&
4176
+ issuingPass !== undefined &&
4177
+ issuingPass.epoch === this.#discoverySweepEpoch) {
4178
+ this.#discoverySweepTreeReads += 1;
4179
+ if (paths.length === 0) {
4180
+ this.#increment('relayfileEmptyTreeReads');
4181
+ this.#discoverySweepEmptyTreeReads += 1;
4182
+ }
4183
+ }
3765
4184
  if (opts.cache)
3766
4185
  await this.#rememberDiscoveryTree(prefix, paths);
3767
4186
  return paths;
@@ -4277,6 +4696,66 @@ export class FactoryLoop {
4277
4696
  if (!dryRun)
4278
4697
  await this.#ensureGithubAgentQuestionWatch(record, liveIssue);
4279
4698
  const spawnedForReaperHandoff = [];
4699
+ // These waits belong to the durable work unit, not the ingestion surface.
4700
+ // Completion and lifecycle ownership use the same identity (#211); using
4701
+ // issueKey here would make every GitHub-native dispatch miss its own fence.
4702
+ const postSpawnKey = dispatchLifecycleKey(record.issue);
4703
+ let resolvePostSpawnIssueObservation;
4704
+ let postSpawnIssueObservationDidSettle = false;
4705
+ const postSpawnIssueObservationSettled = new Promise((resolve) => {
4706
+ resolvePostSpawnIssueObservation = resolve;
4707
+ });
4708
+ let postSpawnIssueObservation;
4709
+ postSpawnIssueObservation = {
4710
+ settled: postSpawnIssueObservationSettled,
4711
+ settle: (accepted) => {
4712
+ if (postSpawnIssueObservationDidSettle)
4713
+ return;
4714
+ postSpawnIssueObservationDidSettle = true;
4715
+ resolvePostSpawnIssueObservation(accepted);
4716
+ if (this.#postSpawnIssueObservations.get(postSpawnKey) === postSpawnIssueObservation) {
4717
+ this.#postSpawnIssueObservations.delete(postSpawnKey);
4718
+ }
4719
+ },
4720
+ };
4721
+ this.#postSpawnIssueObservations.set(postSpawnKey, postSpawnIssueObservation);
4722
+ let resolvePostSpawnDispatchClaim;
4723
+ let postSpawnDispatchClaimSettled = false;
4724
+ const postSpawnDispatchClaim = new Promise((resolve) => {
4725
+ resolvePostSpawnDispatchClaim = resolve;
4726
+ });
4727
+ let resolveRejectedClaimCompensation;
4728
+ let rejectedClaimCompensationSettled = false;
4729
+ const rejectedClaimCompensation = new Promise((resolve) => {
4730
+ resolveRejectedClaimCompensation = resolve;
4731
+ });
4732
+ let postSpawnDispatchClaimFence;
4733
+ postSpawnDispatchClaimFence = {
4734
+ completionAtWriteBoundary: false,
4735
+ claimStarted: false,
4736
+ settled: postSpawnDispatchClaim,
4737
+ rejectionSettled: rejectedClaimCompensation,
4738
+ settle: (accepted) => {
4739
+ if (postSpawnDispatchClaimSettled)
4740
+ return;
4741
+ postSpawnDispatchClaimSettled = true;
4742
+ postSpawnDispatchClaimFence.accepted = accepted;
4743
+ resolvePostSpawnDispatchClaim(accepted);
4744
+ if (this.#postSpawnDispatchClaimFences.get(postSpawnKey) === postSpawnDispatchClaimFence) {
4745
+ this.#postSpawnDispatchClaimFences.delete(postSpawnKey);
4746
+ }
4747
+ },
4748
+ settleRejection: (compensated) => {
4749
+ if (rejectedClaimCompensationSettled)
4750
+ return;
4751
+ rejectedClaimCompensationSettled = true;
4752
+ resolveRejectedClaimCompensation(compensated);
4753
+ },
4754
+ };
4755
+ this.#postSpawnDispatchClaimFences.set(postSpawnKey, postSpawnDispatchClaimFence);
4756
+ const settlePostSpawnIssueObservation = postSpawnIssueObservation.settle;
4757
+ const settlePostSpawnDispatchClaim = postSpawnDispatchClaimFence.settle;
4758
+ let rejectDispatchClaim;
4280
4759
  try {
4281
4760
  if (!dryRun) {
4282
4761
  const issue = await this.#readIssue(dispatchDecision.issue.path);
@@ -4301,47 +4780,92 @@ export class FactoryLoop {
4301
4780
  agents.push({ name: spawned.name, role: spec.role });
4302
4781
  }
4303
4782
  if (!dryRun) {
4783
+ const claimStartedAtMs = this.#clock.now();
4304
4784
  record.dispatchClaim = {
4305
4785
  state: 'pending',
4306
- updatedAtMs: this.#clock.now(),
4786
+ claimStartedAtMs,
4787
+ updatedAtMs: claimStartedAtMs,
4307
4788
  };
4308
4789
  this.#dispatchClaimStatuses.set(dispatchLifecycleKey(record.issue), record.dispatchClaim);
4309
4790
  }
4310
4791
  await this.#writeInFlightRegistry();
4311
4792
  const comment = dispatchComment(dispatchDecision, agents);
4312
4793
  let implementingStateId;
4794
+ const rejectDispatchClaimIfCancelled = async () => {
4795
+ if (postSpawnDispatchClaimFence.accepted !== false)
4796
+ return;
4797
+ if (rejectDispatchClaim)
4798
+ await rejectDispatchClaim();
4799
+ throw new PostSpawnDispatchWaitRejectedError(dispatchDecision.issue.key);
4800
+ };
4313
4801
  if (!dryRun) {
4314
- const issue = await this.#readIssue(dispatchDecision.issue.path);
4315
- if (!issue || !this.#isIssueReady(issue)) {
4316
- // The agents spawned a few lines above can reach terminal before we
4317
- // get here. Their completion writeback parks the issue — Linear
4318
- // `humanReview`, or the GitHub human-review label and stamps
4319
- // *this same record's* lifecycle on the way past. Re-reading the
4320
- // issue then shows "not ready", but the writer was us.
4321
- //
4322
- // Treating that as a foreign change is not a cosmetic misreport: the
4323
- // catch below classifies LiveDispatchStateChangedError as terminal
4324
- // and calls #releaseAndTerminateAgents, so a dispatch whose agents
4325
- // finished quickly tore down its own completed work and reported
4326
- // RETRYABLE to its supervisor (factory#319).
4802
+ let issue = await this.#readIssue(dispatchDecision.issue.path);
4803
+ if (postSpawnDispatchClaimFence.completionAtWriteBoundary) {
4804
+ // Completion reached its terminal provider-write boundary before
4805
+ // dispatch entered the claim boundary. Always re-read after its
4806
+ // receipt, even when the first snapshot was ready: the terminal
4807
+ // write may have started immediately after that snapshot.
4808
+ const issueWriteback = this.#issueWritebackInFlight.get(postSpawnKey);
4809
+ if (issueWriteback)
4810
+ await issueWriteback;
4811
+ issue = await this.#readIssue(dispatchDecision.issue.path);
4812
+ }
4813
+ else if ((!issue || !this.#isIssueReady(issue)) && record.issueWritebackConfirmedAtMs === undefined) {
4814
+ // A very fast agent can finish while its own dispatch is still
4815
+ // performing this post-spawn read. Do not guess authorship from the
4816
+ // first visible state: an idempotent provider mutation cannot tell us
4817
+ // whether Factory or another actor won the race. Instead, wait for
4818
+ // this record's concurrent completion writeback to settle, then use
4819
+ // only its provider-confirmed outcome (factory#319).
4327
4820
  //
4328
- // Only THIS dispatch's own confirmed issue writeback may excuse the
4329
- // change. Lifecycle phases are not enough: `publishing` is entered
4330
- // before the PR is published and `parking` before anything touches
4331
- // the issue, so a phase records local progress, not authorship — and
4332
- // a foreign park landing during those awaits would be misread as
4333
- // ours (codex review on #321). Losing the row to another owner is a
4334
- // different condition and is still caught where it always was, by
4335
- // #saveDispatchLifecycle returning false.
4821
+ // The completion promise settles as soon as terminal issue writeback
4822
+ // succeeds or fails, before Slack/release cleanup. If there is no
4823
+ // completion in flight, this is an ordinary foreign state change and
4824
+ // the existing abort remains immediate.
4825
+ const issueWriteback = this.#issueWritebackInFlight.get(postSpawnKey);
4826
+ if (issueWriteback) {
4827
+ await issueWriteback;
4828
+ issue = await this.#readIssue(dispatchDecision.issue.path);
4829
+ }
4830
+ }
4831
+ if (!issue || !this.#isIssueReady(issue)) {
4832
+ // Only a terminal writeback applied and confirmed by the completion
4833
+ // already in flight for this exact record may excuse the change. The
4834
+ // marker is stamped after provider acknowledgement/readback, never at
4835
+ // the earlier locally-visible write boundary. A confirmed no-op does
4836
+ // not establish authorship: another actor may have parked the issue
4837
+ // before our adapter's first provider read.
4336
4838
  if (record.issueWritebackConfirmedAtMs === undefined) {
4337
4839
  throw new LiveDispatchStateChangedError(dispatchDecision.issue.key);
4338
4840
  }
4339
4841
  // The claim is moot and would be wrong to write: it would drag an
4340
4842
  // issue our own lifecycle has already parked back to `implementing`.
4341
4843
  }
4844
+ else if (this.#stopping) {
4845
+ // Shutdown snapshots the post-spawn fence maps once and settles only
4846
+ // what it finds there. A dispatch that armed its fence after that
4847
+ // snapshot — it can still be inside the preview render, the
4848
+ // `dispatching` save or the question-watch setup when `stop()`
4849
+ // begins — would otherwise enter the claim boundary below against a
4850
+ // fleet already being disposed and leases already relinquished,
4851
+ // leaving a claimed issue with no lifecycle to recover it. Reject
4852
+ // here instead. No claim has started, so there is nothing to
4853
+ // compensate, and this stays synchronous so the completion-race
4854
+ // invariant documented below is untouched (#346 review, cubic).
4855
+ this.#increment('postSpawnDispatchClaimsRefusedDuringStop');
4856
+ postSpawnDispatchClaimFence.settle(false);
4857
+ throw new PostSpawnDispatchWaitRejectedError(dispatchDecision.issue.key);
4858
+ }
4342
4859
  else {
4343
- implementingStateId = await this.#applyDispatchClaim(record, issue, comment);
4344
- record.issueWritebackConfirmedAtMs = this.#clock.now();
4860
+ // This assignment is synchronous with the preceding completion flag
4861
+ // check. Completion either arrived first (the branch above) or will
4862
+ // now observe claimStarted and wait; no await-sized gap remains.
4863
+ postSpawnDispatchClaimFence.claimStarted = true;
4864
+ const appliedClaim = await this.#applyDispatchClaim(record, issue, comment, postSpawnDispatchClaimFence);
4865
+ implementingStateId = appliedClaim.implementingStateId;
4866
+ rejectDispatchClaim = appliedClaim.reject;
4867
+ await rejectDispatchClaimIfCancelled();
4868
+ record.issueWritebackConfirmedAtMs ??= this.#clock.now();
4345
4869
  this.#emit('writeback-verified', { issue: dispatchDecision.issue, path: issue.path });
4346
4870
  }
4347
4871
  }
@@ -4356,19 +4880,110 @@ export class FactoryLoop {
4356
4880
  dryRun,
4357
4881
  };
4358
4882
  record.result = result;
4359
- if (!await this.#saveDispatchLifecycle(record, 'running'))
4883
+ const runningSaved = await this.#saveDispatchLifecycle(record, 'running');
4884
+ await rejectDispatchClaimIfCancelled();
4885
+ if (!runningSaved) {
4886
+ settlePostSpawnDispatchClaim(true);
4887
+ settlePostSpawnIssueObservation(true);
4360
4888
  return result;
4361
- this.#increment('dispatched');
4362
- this.#emit('dispatched', { issue: dispatchDecision.issue, result });
4889
+ }
4363
4890
  if (!dryRun && this.#config.hooks?.onTicketDispatch) {
4364
4891
  await this.#notifyTicketDispatch(dispatchDecision, liveIssue, record, result);
4892
+ await rejectDispatchClaimIfCancelled();
4365
4893
  }
4366
4894
  if (!dryRun) {
4367
4895
  await this.#ensureSlackDispatchThread(record, result, liveIssue);
4896
+ await rejectDispatchClaimIfCancelled();
4368
4897
  }
4898
+ // Accept without an intervening await only after every durable
4899
+ // publication and post-dispatch side effect has completed. Stop or held
4900
+ // abandonment can therefore reject any still-publishing dispatch.
4901
+ settlePostSpawnDispatchClaim(true);
4902
+ settlePostSpawnIssueObservation(true);
4903
+ this.#increment('dispatched');
4904
+ this.#emit('dispatched', { issue: dispatchDecision.issue, result });
4369
4905
  return result;
4370
4906
  }
4371
- catch (error) {
4907
+ catch (caughtError) {
4908
+ // Stop/deadline may reject the fence while any awaited provider or
4909
+ // lifecycle operation is still in flight. If that operation then fails,
4910
+ // its raw error must not fall into ordinary dispatch cleanup: shutdown
4911
+ // still owns the already-spawned placements and needs them retained
4912
+ // until its release snapshot. If the claim receipt is already available,
4913
+ // conditionally reject it before releasing those placements. Without a
4914
+ // receipt the provider outcome is ambiguous, so preserve the original
4915
+ // failure as the compensation detail after the claim boundary began.
4916
+ let error = caughtError;
4917
+ if (postSpawnDispatchClaimFence.accepted === false
4918
+ && !(caughtError instanceof PostSpawnDispatchWaitRejectedError)) {
4919
+ if (rejectDispatchClaim) {
4920
+ try {
4921
+ await this.#rejectDispatchClaimAfterOperationError(dispatchDecision.issue.key, caughtError, rejectDispatchClaim);
4922
+ }
4923
+ catch (rejectionError) {
4924
+ error = rejectionError;
4925
+ }
4926
+ }
4927
+ else {
4928
+ error = new PostSpawnDispatchWaitRejectedError(dispatchDecision.issue.key, postSpawnDispatchClaimFence.claimStarted ? caughtError : undefined);
4929
+ }
4930
+ }
4931
+ settlePostSpawnDispatchClaim(false);
4932
+ if (error instanceof PostSpawnDispatchWaitRejectedError) {
4933
+ settlePostSpawnIssueObservation(false);
4934
+ const compensated = error.compensationError === undefined;
4935
+ // The original provider call has conclusively returned. Stop the
4936
+ // timeout handoff from writing a provisional pending fence, but keep a
4937
+ // separate lease fence until the definitive result is durable. Without
4938
+ // that second set, shutdown can relinquish this epoch in the await
4939
+ // below and make the late save fail after the provider compensated.
4940
+ this.#dispatchClaimSettlementsInFlight.add(postSpawnKey);
4941
+ this.#pendingDispatchClaims.delete(postSpawnKey);
4942
+ let settlementPersisted = false;
4943
+ try {
4944
+ settlementPersisted = compensated
4945
+ ? await this.#clearDispatchCancellationBlock(record)
4946
+ : await this.#retainUncompensatedDispatchClaim(record, error.compensationError);
4947
+ if (!settlementPersisted) {
4948
+ throw new Error(`Unable to persist settled dispatch-claim compensation for ${decision.issue.key}`);
4949
+ }
4950
+ postSpawnDispatchClaimFence.settleRejection(compensated);
4951
+ if (error.compensationError !== undefined) {
4952
+ this.#increment('postSpawnDispatchClaimCompensationFailures');
4953
+ this.#logger.error?.('[factory] rejected dispatch claim could not be safely compensated', {
4954
+ issue: decision.issue.key,
4955
+ error: describeError(error.compensationError).errorMessage,
4956
+ });
4957
+ }
4958
+ // External stop/abandonment owns agent and durable-lifecycle cleanup.
4959
+ // Remove only the process-local slot here; running the ordinary
4960
+ // dispatch-failure path would race or duplicate that cleanup. During
4961
+ // stop, however, retain local placements until the shutdown release
4962
+ // sweep has captured them. If it already has, removal is safe now.
4963
+ if (!compensated) {
4964
+ // The provider still exposes a claim this lifecycle authored. Keep
4965
+ // the process-local slot and durable row recoverable; releasing its
4966
+ // agents would leave the claimed issue with no successor lifecycle.
4967
+ }
4968
+ else if (this.#stopping) {
4969
+ this.#postSpawnDispatchesRejectedDuringStop.add(postSpawnKey);
4970
+ if (this.#shutdownReleaseCapturedDispatches.has(postSpawnKey)) {
4971
+ batch.abandon(decision.issue);
4972
+ this.#postSpawnDispatchesRejectedDuringStop.delete(postSpawnKey);
4973
+ }
4974
+ }
4975
+ else {
4976
+ batch.abandon(decision.issue);
4977
+ }
4978
+ }
4979
+ finally {
4980
+ if (!settlementPersisted)
4981
+ this.#pendingDispatchClaims.add(postSpawnKey);
4982
+ this.#dispatchClaimSettlementsInFlight.delete(postSpawnKey);
4983
+ }
4984
+ throw error;
4985
+ }
4986
+ settlePostSpawnIssueObservation(!(error instanceof LiveDispatchStateChangedError));
4372
4987
  // A spawn can fail after the broker accepted it but before its ack
4373
4988
  // reached Factory. Include every planned worktree agent, not only the
4374
4989
  // acknowledged spawns, so cleanup never races a name-only survivor.
@@ -4488,6 +5103,9 @@ export class FactoryLoop {
4488
5103
  })) ?? [],
4489
5104
  counters: { ...this.#counters },
4490
5105
  fleetControlPlane: this.#fleetControlPlane.status(),
5106
+ // Optional on the port: a backend with no socket omits it, and an absent
5107
+ // value stays absent rather than being invented as healthy.
5108
+ ...(this.#fleet.fleetConnectStatus ? { fleetConnect: this.#fleet.fleetConnectStatus() } : {}),
4491
5109
  slackDegraded: this.#slackDegraded,
4492
5110
  slackDegradedReason: this.#slackDegradedReason,
4493
5111
  eventListener: this.#eventListenerStatus(),
@@ -4519,16 +5137,39 @@ export class FactoryLoop {
4519
5137
  *
4520
5138
  * Only successful passes reach here: a pass that threw has no report, and
4521
5139
  * inventing zeroes for it would publish "found nothing" for a sweep that
4522
- * never got to look. The previous pass's numbers stay put instead, dated by
4523
- * `lastCompletedAtMs`, which is the honest reading.
5140
+ * never got to look. The previous enumerating pass's numbers stay put
5141
+ * instead, dated by `lastEnumeratedAtMs`, which is the honest reading.
5142
+ *
5143
+ * A deferred pass gets the same treatment for the same reason. It settles
5144
+ * successfully, and `lastCompletedAtMs` moves — deliberately, because the
5145
+ * #295/#296 stall derivation reads that timestamp against `lastStartedAtMs`,
5146
+ * and freezing it would report a functioning daemon as a hung one after ten
5147
+ * intervals of deferring correctly to another owner. But it enumerated
5148
+ * nothing, so its zeroes are not a measurement of anything and must not
5149
+ * replace one. Only the marker is recorded.
4524
5150
  */
4525
- #recordReadinessSweepOutcome(report) {
5151
+ #recordReadinessSweepOutcome(report, completedAtMs) {
5152
+ if (report.discoveryDeferred) {
5153
+ this.#readinessReconcileLastSweepDeferred = report.discoveryDeferred;
5154
+ return;
5155
+ }
5156
+ this.#readinessReconcileLastSweepDeferred = undefined;
4526
5157
  this.#readinessReconcileLastSweep = {
4527
5158
  candidates: report.pulled.length,
5159
+ treeReads: report.treeReads ?? 0,
5160
+ emptyTreeReads: report.emptyTreeReads ?? 0,
4528
5161
  dispatched: report.dispatched.length,
4529
5162
  skipped: report.skipped.length,
4530
5163
  skipReasons: factorySweepSkipReasonCounts(report.skipped),
4531
- ...(report.discoveryDeferred ? { discoveryDeferred: report.discoveryDeferred } : {}),
5164
+ // Counted from the same entries `skipReasons` counts, so the parts sum to
5165
+ // `skipReasons['dispatch-failed']` by construction rather than by a
5166
+ // second traversal agreeing with the first.
5167
+ dispatchFailures: report.skipped.filter((entry) => entry.code === 'dispatch-failed').length,
5168
+ dispatchFailureReasons: factoryDispatchFailureReasonCounts(report.skipped),
5169
+ // The caller's completion stamp, not a fresh clock read: on a pass that
5170
+ // enumerated, `lastEnumeratedAtMs` and `lastCompletedAtMs` describe the
5171
+ // same instant and must not drift apart by a tick.
5172
+ enumeratedAtMs: completedAtMs,
4532
5173
  };
4533
5174
  }
4534
5175
  #readinessReconcileStatus() {
@@ -4611,16 +5252,33 @@ export class FactoryLoop {
4611
5252
  ...(this.#readinessReconcileLastSweep
4612
5253
  ? {
4613
5254
  candidates: this.#readinessReconcileLastSweep.candidates,
5255
+ // Unconditional for the same reason `dispatchFailures` is: these
5256
+ // are only meaningful as a pair, and only a published zero lets a
5257
+ // reader see that `emptyTreeReads < treeReads` — i.e. that the
5258
+ // mount served real content and a zero `candidates` beside it means
5259
+ // an empty workspace, not a silent mount (#351 follow-up).
5260
+ treeReads: this.#readinessReconcileLastSweep.treeReads,
5261
+ emptyTreeReads: this.#readinessReconcileLastSweep.emptyTreeReads,
4614
5262
  dispatched: this.#readinessReconcileLastSweep.dispatched,
4615
5263
  skipped: this.#readinessReconcileLastSweep.skipped,
4616
5264
  ...(Object.keys(this.#readinessReconcileLastSweep.skipReasons).length > 0
4617
5265
  ? { skipReasons: { ...this.#readinessReconcileLastSweep.skipReasons } }
4618
5266
  : {}),
4619
- ...(this.#readinessReconcileLastSweep.discoveryDeferred
4620
- ? { discoveryDeferred: this.#readinessReconcileLastSweep.discoveryDeferred }
5267
+ // Unconditional, unlike the breakdown below it: a zero here is the
5268
+ // fact "this sweep attempted dispatches and none of them failed",
5269
+ // which no other field on this surface can express.
5270
+ dispatchFailures: this.#readinessReconcileLastSweep.dispatchFailures,
5271
+ ...(Object.keys(this.#readinessReconcileLastSweep.dispatchFailureReasons).length > 0
5272
+ ? { dispatchFailureReasons: { ...this.#readinessReconcileLastSweep.dispatchFailureReasons } }
4621
5273
  : {}),
5274
+ lastEnumeratedAtMs: this.#readinessReconcileLastSweep.enumeratedAtMs,
4622
5275
  }
4623
5276
  : {}),
5277
+ // Independent of the trio: a daemon whose FIRST pass deferred has no
5278
+ // counts to publish and still needs to say why.
5279
+ ...(this.#readinessReconcileLastSweepDeferred
5280
+ ? { discoveryDeferred: this.#readinessReconcileLastSweepDeferred }
5281
+ : {}),
4624
5282
  ...(this.#readinessReconcileLastError ? { lastError: this.#readinessReconcileLastError } : {}),
4625
5283
  ...(this.#readinessReconcileLastErrorClass
4626
5284
  ? { lastErrorClass: this.#readinessReconcileLastErrorClass }
@@ -6187,9 +6845,11 @@ export class FactoryLoop {
6187
6845
  const comment = dispatchComment(record.decision, agents);
6188
6846
  let implementingStateId;
6189
6847
  if (!record.dryRun) {
6848
+ const claimStartedAtMs = this.#clock.now();
6190
6849
  record.dispatchClaim = {
6191
6850
  state: 'pending',
6192
- updatedAtMs: this.#clock.now(),
6851
+ claimStartedAtMs,
6852
+ updatedAtMs: claimStartedAtMs,
6193
6853
  };
6194
6854
  this.#dispatchClaimStatuses.set(dispatchLifecycleKey(record.issue), record.dispatchClaim);
6195
6855
  }
@@ -6199,7 +6859,7 @@ export class FactoryLoop {
6199
6859
  if (!issue)
6200
6860
  throw new Error(`Unable to recover durable dispatch ${record.issue.key}: issue is no longer readable`);
6201
6861
  await this.#ensureGithubAgentQuestionWatch(record, issue);
6202
- implementingStateId = await this.#applyDispatchClaim(record, issue, comment);
6862
+ implementingStateId = (await this.#applyDispatchClaim(record, issue, comment)).implementingStateId;
6203
6863
  }
6204
6864
  const recoveredPreviews = uniquePreviewReferences([
6205
6865
  ...dispatchSpecs(record.decision).map((spec) => spec.preview),
@@ -6508,33 +7168,161 @@ export class FactoryLoop {
6508
7168
  }
6509
7169
  await this.#linear.postComment(issue, body);
6510
7170
  }
6511
- async #applyDispatchClaim(record, issue, comment) {
7171
+ async #applyDispatchClaim(record, issue, comment, claimFence) {
7172
+ const claimWasRejected = () => claimFence?.accepted === false;
7173
+ // stop() can reject this fence after the post-spawn readiness read but
7174
+ // before this method starts. Never begin a provider claim in that state;
7175
+ // shutdown intentionally does not drain dispatches that never crossed the
7176
+ // claim boundary.
7177
+ if (claimWasRejected()) {
7178
+ throw new PostSpawnDispatchWaitRejectedError(record.issue.key);
7179
+ }
6512
7180
  let implementingStateId;
7181
+ let rejectClaim;
6513
7182
  if (isGithubIssue(issue)) {
7183
+ const statusClaim = {
7184
+ value: { result: undefined },
7185
+ };
6514
7186
  await this.#retryDispatchWriteback(record, issue, 'GitHub label factory:in-progress', async () => {
6515
- await this.#githubWriteback.setStatus(issue, 'in-progress');
7187
+ statusClaim.value = this.#githubWriteback.claimStatus
7188
+ ? await this.#githubWriteback.claimStatus(issue, 'in-progress')
7189
+ : { result: await this.#githubWriteback.setStatus(issue, 'in-progress') };
6516
7190
  });
7191
+ rejectClaim = async () => await this.#rejectGithubDispatchClaim(record, issue, statusClaim.value);
7192
+ if (claimWasRejected()) {
7193
+ await rejectClaim();
7194
+ }
6517
7195
  const commentApplied = this.#githubWriteback.hasCommentMarker
6518
7196
  ? async () => this.#githubWriteback.hasCommentMarker(issue, comment)
6519
7197
  : undefined;
6520
- await this.#retryDispatchWriteback(record, issue, 'GitHub dispatch comment', async () => this.#githubWriteback.postComment(issue, comment), commentApplied);
7198
+ try {
7199
+ await this.#retryDispatchWriteback(record, issue, 'GitHub dispatch comment', async () => {
7200
+ if (claimWasRejected())
7201
+ await rejectClaim();
7202
+ await this.#githubWriteback.postComment(issue, comment);
7203
+ }, commentApplied);
7204
+ }
7205
+ catch (error) {
7206
+ if (claimWasRejected() && !(error instanceof PostSpawnDispatchWaitRejectedError)) {
7207
+ await this.#rejectDispatchClaimAfterOperationError(record.issue.key, error, rejectClaim);
7208
+ }
7209
+ throw error;
7210
+ }
7211
+ if (claimWasRejected())
7212
+ await rejectClaim();
6521
7213
  }
6522
7214
  else {
6523
7215
  implementingStateId = this.#states.idFor(issue.team, 'agentImplementing');
7216
+ const readyStateId = issue.stateId;
7217
+ let claimToken;
6524
7218
  await this.#retryDispatchWriteback(record, issue, `Linear state ${implementingStateId}`, async () => {
6525
- await this.#linear.setState(issue, implementingStateId);
6526
- });
6527
- await this.#retryDispatchWriteback(record, issue, 'Linear dispatch comment', async () => {
6528
- await this.#linear.postComment(issue, comment);
7219
+ claimToken = (await this.#linear.setState(issue, implementingStateId))?.claimToken;
6529
7220
  });
7221
+ rejectClaim = async () => await this.#rejectLinearDispatchClaim(record, issue, implementingStateId, readyStateId, claimToken);
7222
+ if (claimWasRejected()) {
7223
+ await rejectClaim();
7224
+ }
7225
+ try {
7226
+ await this.#retryDispatchWriteback(record, issue, 'Linear dispatch comment', async () => {
7227
+ if (claimWasRejected())
7228
+ await rejectClaim();
7229
+ await this.#linear.postComment(issue, comment);
7230
+ });
7231
+ }
7232
+ catch (error) {
7233
+ if (claimWasRejected() && !(error instanceof PostSpawnDispatchWaitRejectedError)) {
7234
+ await this.#rejectDispatchClaimAfterOperationError(record.issue.key, error, rejectClaim);
7235
+ }
7236
+ throw error;
7237
+ }
7238
+ if (claimWasRejected())
7239
+ await rejectClaim();
6530
7240
  }
6531
7241
  record.dispatchClaim = {
6532
7242
  state: 'verified',
7243
+ // Cancellation can still arrive after this returns — while the running
7244
+ // lifecycle save, the dispatch hook or the Slack publication is pending.
7245
+ // The rejection fence rebuilds the claim from this record, so a verified
7246
+ // state that drops the claim-start stamp leaves `cancellationBlocked`
7247
+ // with no watermark for `getIssueStatus(..., { requireFresh: true })`.
7248
+ // A private repository has no unauthenticated fallback to fall back on,
7249
+ // so that lifecycle, its agents and its batch slot would then be retained
7250
+ // for good, even once a human superseded the claim (#346 review, codex).
7251
+ ...(record.dispatchClaim?.claimStartedAtMs === undefined
7252
+ ? {}
7253
+ : { claimStartedAtMs: record.dispatchClaim.claimStartedAtMs }),
6533
7254
  updatedAtMs: this.#clock.now(),
6534
7255
  };
6535
7256
  this.#dispatchClaimStatuses.set(dispatchLifecycleKey(record.issue), record.dispatchClaim);
6536
7257
  await this.#writeDispatchClaimRegistry(record.issue);
6537
- return implementingStateId;
7258
+ if (claimWasRejected())
7259
+ await rejectClaim?.();
7260
+ return {
7261
+ ...(implementingStateId ? { implementingStateId } : {}),
7262
+ reject: rejectClaim,
7263
+ };
7264
+ }
7265
+ async #rejectGithubDispatchClaim(record, issue, claim) {
7266
+ let compensationError;
7267
+ if (claim.result !== 'already-matched') {
7268
+ try {
7269
+ const claimToken = claim.claimToken;
7270
+ if (!claimToken) {
7271
+ throw new Error('GitHub rejected dispatch claim has no immutable ownership token');
7272
+ }
7273
+ const rollbackStatusClaim = this.#githubWriteback.rollbackStatusClaim?.bind(this.#githubWriteback);
7274
+ if (!rollbackStatusClaim) {
7275
+ throw new Error('GitHub writeback cannot conditionally roll back a rejected dispatch claim');
7276
+ }
7277
+ let outcome = 'unproven';
7278
+ await this.#retryDispatchWriteback(record, issue, 'GitHub rejected dispatch claim rollback', async () => {
7279
+ outcome = await rollbackStatusClaim(issue, 'in-progress', claimToken);
7280
+ if (outcome === 'unproven') {
7281
+ throw new Error('GitHub rejected dispatch claim rollback could not prove ownership');
7282
+ }
7283
+ });
7284
+ }
7285
+ catch (error) {
7286
+ compensationError = error;
7287
+ }
7288
+ }
7289
+ throw new PostSpawnDispatchWaitRejectedError(record.issue.key, compensationError);
7290
+ }
7291
+ async #rejectDispatchClaimAfterOperationError(issueKey, operationError, rejectClaim) {
7292
+ try {
7293
+ await rejectClaim();
7294
+ }
7295
+ catch (rejectionError) {
7296
+ if (rejectionError instanceof PostSpawnDispatchWaitRejectedError) {
7297
+ if (rejectionError.compensationError === undefined)
7298
+ throw rejectionError;
7299
+ throw new PostSpawnDispatchWaitRejectedError(issueKey, new AggregateError([operationError, rejectionError.compensationError], 'Cancelled dispatch operation failed and its claim could not be safely compensated'));
7300
+ }
7301
+ throw new PostSpawnDispatchWaitRejectedError(issueKey, new AggregateError([operationError, rejectionError], 'Cancelled dispatch operation failed and its claim rejection threw unexpectedly'));
7302
+ }
7303
+ throw new PostSpawnDispatchWaitRejectedError(issueKey);
7304
+ }
7305
+ async #rejectLinearDispatchClaim(record, issue, implementingStateId, readyStateId, claimToken) {
7306
+ let compensationError;
7307
+ try {
7308
+ const compareAndSetState = this.#linear.compareAndSetState?.bind(this.#linear);
7309
+ if (!compareAndSetState) {
7310
+ throw new Error('Linear writeback cannot conditionally roll back a rejected dispatch claim');
7311
+ }
7312
+ if (!claimToken) {
7313
+ throw new Error('Linear rejected dispatch claim has no immutable ownership token');
7314
+ }
7315
+ await this.#retryDispatchWriteback(record, issue, `Linear rejected dispatch claim rollback to ${readyStateId}`, async () => {
7316
+ const outcome = await compareAndSetState(issue, implementingStateId, claimToken, readyStateId);
7317
+ if (outcome === 'unproven') {
7318
+ throw new Error('Linear rejected dispatch claim rollback could not prove an atomic provider transition');
7319
+ }
7320
+ });
7321
+ }
7322
+ catch (error) {
7323
+ compensationError = error;
7324
+ }
7325
+ throw new PostSpawnDispatchWaitRejectedError(record.issue.key, compensationError);
6538
7326
  }
6539
7327
  async #retryDispatchWriteback(record, issue, write, apply, isApplied) {
6540
7328
  let lastError;
@@ -6549,11 +7337,23 @@ export class FactoryLoop {
6549
7337
  return;
6550
7338
  }
6551
7339
  catch (error) {
7340
+ // Claim rejection is terminal for this dispatch. In particular, do not
7341
+ // turn a cancellation discovered between comment attempts into another
7342
+ // provider retry that can publish after shutdown began.
7343
+ if (error instanceof PostSpawnDispatchWaitRejectedError) {
7344
+ if (lastError !== undefined && error.compensationError !== undefined) {
7345
+ throw new PostSpawnDispatchWaitRejectedError(record.issue.key, new AggregateError([lastError, error.compensationError], 'Dispatch writeback failed before cancellation and its claim could not be safely compensated'));
7346
+ }
7347
+ throw error;
7348
+ }
6552
7349
  lastError = error;
6553
7350
  const deadLettered = attempt === DISPATCH_WRITEBACK_MAX_ATTEMPTS;
6554
7351
  this.#increment('dispatchWritebackFailures');
6555
7352
  record.dispatchClaim = {
6556
7353
  state: 'degraded',
7354
+ ...(record.dispatchClaim?.claimStartedAtMs === undefined
7355
+ ? {}
7356
+ : { claimStartedAtMs: record.dispatchClaim.claimStartedAtMs }),
6557
7357
  write,
6558
7358
  attempts: attempt,
6559
7359
  maxAttempts: DISPATCH_WRITEBACK_MAX_ATTEMPTS,
@@ -6656,7 +7456,7 @@ export class FactoryLoop {
6656
7456
  const candidates = [];
6657
7457
  let scanned = 0;
6658
7458
  let lastProgressAtMs = startedAtMs;
6659
- for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'GitHub mirror candidate loading', { cache: true })) {
7459
+ for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'GitHub mirror candidate loading', { cache: true, enumeration: true })) {
6660
7460
  await this.#refreshLiveHeartbeatIfDue();
6661
7461
  if (!isLinearIssueMirrorCandidatePath(path)) {
6662
7462
  continue;
@@ -6706,7 +7506,7 @@ export class FactoryLoop {
6706
7506
  else {
6707
7507
  pathBatches = [];
6708
7508
  for (const root of roots) {
6709
- pathBatches.push(await this.#listRelayfileTree(root, 'GitHub issue ingestion', { cache: true }));
7509
+ pathBatches.push(await this.#listRelayfileTree(root, 'GitHub issue ingestion', { cache: true, enumeration: true }));
6710
7510
  }
6711
7511
  this.#increment('githubIssueIndexFallbacks');
6712
7512
  }
@@ -7184,6 +7984,10 @@ export class FactoryLoop {
7184
7984
  // The pass-scoped flag also lets every later blocked issue reuse the
7185
7985
  // resulting dependency index instead of rescanning the full tree.
7186
7986
  this.#dependencyLinearTreeLoaded = true;
7987
+ // Cached like the enumeration walks, but deliberately NOT `enumeration`:
7988
+ // this is a lookup for named blocker identities, not the sweep asking
7989
+ // what work exists, and mixing a lookup's result into the ratio is the
7990
+ // masking the pair's scoping exists to prevent (#363 review).
7187
7991
  for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'dependency blocker discovery', { cache: true })) {
7188
7992
  if (isIssueFilePath(path))
7189
7993
  await this.#readIssue(path);
@@ -7453,6 +8257,9 @@ export class FactoryLoop {
7453
8257
  readinessReconcile: this.#readinessReconcileStatus(),
7454
8258
  dispatchCapacity: this.#dispatchCapacityStatus(),
7455
8259
  fleetControlPlane: this.#fleetControlPlane.status(),
8260
+ // Optional on the port: a backend with no socket omits it, and an absent
8261
+ // value stays absent rather than being invented as healthy.
8262
+ ...(this.#fleet.fleetConnectStatus ? { fleetConnect: this.#fleet.fleetConnectStatus() } : {}),
7456
8263
  };
7457
8264
  // The deployed container serves `/healthz` straight out of this file and
7458
8265
  // has no redaction logic of its own, so publish the already-safe view here
@@ -7631,14 +8438,14 @@ export class FactoryLoop {
7631
8438
  }
7632
8439
  const pathsByKey = new Map();
7633
8440
  const canonicalPathsByKey = new Map();
7634
- for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'Linear ready issue canonical discovery', { cache: true })) {
8441
+ for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'Linear ready issue canonical discovery', { cache: true, enumeration: true })) {
7635
8442
  if (isIssueFilePath(path)) {
7636
8443
  const key = keyFromPath(path);
7637
8444
  canonicalPathsByKey.set(key, path);
7638
8445
  pathsByKey.set(key, path);
7639
8446
  }
7640
8447
  }
7641
- for (const path of await this.#listRelayfileTree(linearByStatePath('ready-for-agent'), 'Linear ready issue alias discovery', { cache: true })) {
8448
+ for (const path of await this.#listRelayfileTree(linearByStatePath('ready-for-agent'), 'Linear ready issue alias discovery', { cache: true, enumeration: true })) {
7642
8449
  if (isIssueAliasFilePath(path)) {
7643
8450
  const canonicalPath = canonicalPathsByKey.get(keyFromPath(path));
7644
8451
  if (canonicalPath) {
@@ -7692,10 +8499,20 @@ export class FactoryLoop {
7692
8499
  }
7693
8500
  async #releaseInFlightAgents(reason, opts = {}) {
7694
8501
  const agents = new Map();
7695
- for (const record of (await this.#batch()).inFlight) {
8502
+ const blockedDispatches = new Set();
8503
+ const batch = await this.#batch();
8504
+ const records = [...batch.inFlight];
8505
+ for (const record of records) {
8506
+ const key = dispatchLifecycleKey(record.issue);
7696
8507
  if (record.dryRun) {
8508
+ this.#shutdownReleaseCapturedDispatches.add(key);
8509
+ continue;
8510
+ }
8511
+ if (this.#uncompensatedDispatchClaims.has(key) || record.dispatchClaim?.cancellationBlocked === true) {
8512
+ blockedDispatches.add(key);
7697
8513
  continue;
7698
8514
  }
8515
+ this.#shutdownReleaseCapturedDispatches.add(key);
7699
8516
  if (opts.preserveDurable && [...record.agents.values()].some((tracked) => tracked.result?.locality === 'remote')) {
7700
8517
  continue;
7701
8518
  }
@@ -7704,7 +8521,15 @@ export class FactoryLoop {
7704
8521
  }
7705
8522
  }
7706
8523
  await this.#releaseAndTerminateAgents([...agents], reason, 'stop');
7707
- await this.#writeInFlightRegistry(undefined, undefined, true);
8524
+ for (const record of records) {
8525
+ const key = dispatchLifecycleKey(record.issue);
8526
+ if (blockedDispatches.has(key))
8527
+ continue;
8528
+ if (!this.#postSpawnDispatchesRejectedDuringStop.delete(key))
8529
+ continue;
8530
+ batch.abandon(record.issue);
8531
+ }
8532
+ await this.#writeInFlightRegistry(undefined, undefined, true, blockedDispatches);
7708
8533
  }
7709
8534
  async #releaseAndTerminateAgents(agents, reason, context) {
7710
8535
  const failed = [];
@@ -7763,17 +8588,40 @@ export class FactoryLoop {
7763
8588
  await this.#reportAgent(record, tracked, 'agent.released', { releaseReason: reason });
7764
8589
  }
7765
8590
  catch (error) {
7766
- failed.push(agentName);
7767
- this.#logger.warn?.(`[factory] failed to release ${agentName} during ${context}`, error);
7768
- if (record) {
7769
- const lifecycle = await this.#state
7770
- .getDispatchLifecycle(this.#workspaceId, dispatchLifecycleKey(record.issue))
7771
- .catch(() => undefined);
7772
- if (lifecycle) {
7773
- await this.#reportLifecycle(lifecycle, 'factory.failure', {
7774
- level: 'error',
7775
- errorCode: 'release_failed',
7776
- });
8591
+ if (isAgentAlreadyGoneOnRelease(error)) {
8592
+ // Do the same bookkeeping the success path does — the agent IS
8593
+ // gone. Skipping this would leave `batch.recordRelease` unset and
8594
+ // the next retry attempt would think the invocation is still
8595
+ // dispatchable.
8596
+ this.#increment('releaseAgentAlreadyGone');
8597
+ this.#logger.info?.(`[factory] release skipped: agent already gone during ${context}`, { agentName, reason });
8598
+ if (record && batch && context !== 'stop') {
8599
+ const releasedInvocationId = batch.recordRelease(record, agentName, this.#clock.now());
8600
+ if (releasedInvocationId) {
8601
+ this.#logger.debug?.('[factory] released agent invocation is no longer dispatchable', {
8602
+ issue: record.issue.key,
8603
+ agentName,
8604
+ reason,
8605
+ invocationId: releasedInvocationId,
8606
+ });
8607
+ }
8608
+ }
8609
+ if (record)
8610
+ await this.#reportAgent(record, tracked, 'agent.released', { releaseReason: reason });
8611
+ }
8612
+ else {
8613
+ failed.push(agentName);
8614
+ this.#logger.warn?.(`[factory] failed to release ${agentName} during ${context}`, error);
8615
+ if (record) {
8616
+ const lifecycle = await this.#state
8617
+ .getDispatchLifecycle(this.#workspaceId, dispatchLifecycleKey(record.issue))
8618
+ .catch(() => undefined);
8619
+ if (lifecycle) {
8620
+ await this.#reportLifecycle(lifecycle, 'factory.failure', {
8621
+ level: 'error',
8622
+ errorCode: 'release_failed',
8623
+ });
8624
+ }
7777
8625
  }
7778
8626
  }
7779
8627
  }
@@ -7923,7 +8771,7 @@ export class FactoryLoop {
7923
8771
  this.#increment('agentWorktreesCleaned');
7924
8772
  }
7925
8773
  }
7926
- async #writeInFlightRegistry(path = this.#config.loop.registryPath, heartbeatPath = this.#config.loop.heartbeatPath, empty = false) {
8774
+ async #writeInFlightRegistry(path = this.#config.loop.registryPath, heartbeatPath = this.#config.loop.heartbeatPath, empty = false, retainIssueKeys) {
7927
8775
  const updatedAtMs = this.#clock.now();
7928
8776
  const agents = [];
7929
8777
  const seenAgents = new Set();
@@ -7961,10 +8809,12 @@ export class FactoryLoop {
7961
8809
  } : {}),
7962
8810
  });
7963
8811
  };
7964
- if (!empty) {
8812
+ if (!empty || retainIssueKeys?.size) {
7965
8813
  for (const record of (await this.#batch()).inFlight) {
7966
8814
  if (record.dryRun)
7967
8815
  continue;
8816
+ if (empty && !retainIssueKeys?.has(dispatchLifecycleKey(record.issue)))
8817
+ continue;
7968
8818
  if (record.dispatchClaim) {
7969
8819
  this.#dispatchClaimStatuses.set(dispatchLifecycleKey(record.issue), record.dispatchClaim);
7970
8820
  }
@@ -9179,10 +10029,59 @@ export class FactoryLoop {
9179
10029
  // the release-driven exit event so it cannot re-trigger a resume before the
9180
10030
  // record leaves the batch.
9181
10031
  async #abandonStuckDispatch(record, reason) {
9182
- const key = issueKey(record.issue);
10032
+ const key = dispatchLifecycleKey(record.issue);
10033
+ // Fence the periodic held sweep before any await below. A blocked claim is
10034
+ // retried by the single keyed abandonment timer; letting the held sweep
10035
+ // enter concurrently can release the same placements twice when provider
10036
+ // supersession becomes visible.
10037
+ this.#abandonedDispatchReasons.set(key, reason);
10038
+ try {
10039
+ await this.#abandonStuckDispatchFenced(record, reason, key);
10040
+ }
10041
+ catch (error) {
10042
+ // The early fence above suppresses the periodic held sweep. Any
10043
+ // exceptional exit must therefore install the keyed retry before
10044
+ // propagating, or one transient persistence/read failure becomes a
10045
+ // permanent absorbing state.
10046
+ this.#increment('abandonedDispatchReleaseRetries');
10047
+ this.#logger.warn?.('[factory] abandoned dispatch cleanup failed before retry was armed', {
10048
+ issue: record.issue.key,
10049
+ error: describeError(error).errorMessage,
10050
+ });
10051
+ this.#scheduleAbandonedDispatchRetry(record, reason);
10052
+ throw error;
10053
+ }
10054
+ }
10055
+ async #abandonStuckDispatchFenced(record, reason, key) {
10056
+ const claimFence = this.#postSpawnDispatchClaimFences.get(key);
10057
+ const rejectedClaimCompensation = claimFence?.claimStarted
10058
+ ? claimFence.rejectionSettled
10059
+ : undefined;
10060
+ // A dispatch can be stuck inside a later spawn after an earlier agent has
10061
+ // already started completion. Release both post-spawn waits before the
10062
+ // first abandonment await so terminal processing cannot remain an
10063
+ // absorbing promise after this lifecycle is reaped.
10064
+ this.#settlePostSpawnDispatchWaits(key, false, 'abandonment');
10065
+ if (rejectedClaimCompensation) {
10066
+ const compensated = await this.#awaitRejectedClaimCompensation(rejectedClaimCompensation);
10067
+ if (!compensated) {
10068
+ if (!this.#pendingDispatchClaims.has(key)) {
10069
+ // The provider call settled while this bounded wait expired; its
10070
+ // caller is recording the definitive compensated/unproven result.
10071
+ this.#scheduleAbandonedDispatchRetry(record, reason);
10072
+ return;
10073
+ }
10074
+ await this.#retainUncompensatedDispatchClaim(record, new Error('Rejected dispatch claim compensation did not complete before abandonment'), { preferExistingError: true, pending: true });
10075
+ this.#scheduleAbandonedDispatchRetry(record, reason);
10076
+ return;
10077
+ }
10078
+ }
10079
+ if (await this.#dispatchClaimBlocksAbandonment(record)) {
10080
+ this.#scheduleAbandonedDispatchRetry(record, reason);
10081
+ return;
10082
+ }
9183
10083
  const heldPastDeadline = reason === HELD_PAST_DEADLINE_RELEASE_REASON;
9184
10084
  const agentReleaseReason = heldPastDeadline ? HELD_PAST_DEADLINE_RELEASE_REASON : 'issue-abandoned';
9185
- this.#abandonedDispatchReasons.set(key, reason);
9186
10085
  if (!await this.#saveDispatchLifecycle(record, 'abandoning', undefined, reason, new Set(), { cancellationReason: 'dispatch_failed' })) {
9187
10086
  this.#increment('abandonedDispatchReleaseRetries');
9188
10087
  // The generic durable retry can recover the in-memory reason in this
@@ -13210,6 +14109,22 @@ export class FactoryLoop {
13210
14109
  return;
13211
14110
  }
13212
14111
  this.#completionInFlight.add(completionKey);
14112
+ const postSpawnIssueObservation = this.#postSpawnIssueObservations.get(completionKey);
14113
+ const postSpawnDispatchClaimFence = this.#postSpawnDispatchClaimFences.get(completionKey);
14114
+ let settleIssueWriteback;
14115
+ const issueWritebackSettled = new Promise((resolve) => {
14116
+ settleIssueWriteback = resolve;
14117
+ });
14118
+ let issueWritebackDidSettle = false;
14119
+ const settleIssueWritebackOnce = () => {
14120
+ if (issueWritebackDidSettle)
14121
+ return;
14122
+ issueWritebackDidSettle = true;
14123
+ settleIssueWriteback();
14124
+ if (this.#issueWritebackInFlight.get(completionKey) === issueWritebackSettled) {
14125
+ this.#issueWritebackInFlight.delete(completionKey);
14126
+ }
14127
+ };
13213
14128
  let releaseReasonForRetry;
13214
14129
  try {
13215
14130
  if (!await this.#assertDispatchLifecycleOwner(record))
@@ -13259,13 +14174,59 @@ export class FactoryLoop {
13259
14174
  const humanReview = configuredHumanReview || (githubIssue && !githubMerged);
13260
14175
  const statusLabel = humanReview ? 'In Human Review' : 'Done';
13261
14176
  if (issue) {
14177
+ if (postSpawnDispatchClaimFence) {
14178
+ // Publish boundary arrival before any await. Dispatch performs the
14179
+ // paired synchronous claimStarted assignment after its ready read,
14180
+ // so exactly one side wins: an earlier completion is awaited and
14181
+ // re-read; a claim already entering its provider write finishes
14182
+ // before this terminal write begins.
14183
+ postSpawnDispatchClaimFence.completionAtWriteBoundary = true;
14184
+ if (postSpawnDispatchClaimFence.claimStarted || postSpawnDispatchClaimFence.accepted !== undefined) {
14185
+ let claimAccepted = postSpawnDispatchClaimFence.accepted;
14186
+ if (claimAccepted === undefined) {
14187
+ // This counter makes the otherwise internal promise boundary
14188
+ // observable in deterministic race regressions and in incident
14189
+ // telemetry. External abandonment/stop settlement must release
14190
+ // every completion counted here.
14191
+ this.#increment('postSpawnDispatchClaimFenceWaits');
14192
+ claimAccepted = await postSpawnDispatchClaimFence.settled;
14193
+ }
14194
+ if (!claimAccepted)
14195
+ return;
14196
+ }
14197
+ }
14198
+ // Register only at the provider-write boundary. Work before this point
14199
+ // (PR discovery/merge gating) has not changed the issue, so a foreign
14200
+ // park during it must still abort immediately rather than waiting on a
14201
+ // possibly long completion path.
14202
+ this.#issueWritebackInFlight.set(completionKey, issueWritebackSettled);
13262
14203
  if (githubIssue) {
13263
14204
  if (humanReview) {
13264
- await this.#githubWriteback.setStatus(issue, 'human-review');
14205
+ const statusWrite = await this.#githubWriteback.setStatus(issue, 'human-review');
14206
+ if (statusWrite === undefined)
14207
+ this.#recordMissingGithubWritebackReceipt('setStatus');
14208
+ // Only an explicit provider-proven transition establishes that
14209
+ // this dispatch owns the visible park. Legacy void adapters and
14210
+ // App acknowledgements remain deliberately untrusted.
14211
+ if (statusWrite === 'applied') {
14212
+ record.issueWritebackConfirmedAtMs ??= this.#clock.now();
14213
+ }
14214
+ // The lifecycle-state outcome is now known. Unblock the concurrent
14215
+ // post-spawn read before the separate completion comment write.
14216
+ settleIssueWritebackOnce();
13265
14217
  await this.#githubWriteback.postComment(issue, `Factory agents completed; this issue is awaiting human review. The pull request remains open.\n\nMerge policy: ${this.#config.mergePolicy}`);
13266
14218
  }
13267
14219
  else {
13268
- await this.#githubWriteback.closeIssue(issue, 'Factory observed the linked pull request merge and completed this issue.');
14220
+ const closeWrite = await this.#githubWriteback.closeIssue(issue, 'Factory observed the linked pull request merge and completed this issue.');
14221
+ if (closeWrite === undefined)
14222
+ this.#recordMissingGithubWritebackReceipt('closeIssue');
14223
+ // A provider-confirmed, actor-attributed close is the only safe
14224
+ // proof that this dispatch owns the visible terminal state. An
14225
+ // idempotent no-op, legacy void adapter, or App acknowledgement
14226
+ // may instead mean a third party won the close race.
14227
+ if (closeWrite === 'applied') {
14228
+ record.issueWritebackConfirmedAtMs ??= this.#clock.now();
14229
+ }
13269
14230
  }
13270
14231
  }
13271
14232
  else {
@@ -13273,13 +14234,28 @@ export class FactoryLoop {
13273
14234
  ? this.#states.idFor(issueTeam, 'humanReview')
13274
14235
  : this.#states.idFor(issueTeam, 'done');
13275
14236
  await this.#linear.setState(issue, targetState);
14237
+ record.issueWritebackConfirmedAtMs ??= this.#clock.now();
13276
14238
  await this.#recordCanonicalIssueState({ ...record.issue, stateId: targetState });
13277
14239
  }
13278
- record.issueWritebackConfirmedAtMs = this.#clock.now();
13279
- this.#emit('writeback-verified', { issue: record.issue, path: issue.path });
14240
+ if (record.issueWritebackConfirmedAtMs !== undefined) {
14241
+ this.#emit('writeback-verified', { issue: record.issue, path: issue.path });
14242
+ }
14243
+ // Unblock a concurrent post-spawn read as soon as the issue writeback
14244
+ // outcome is known. Completion still has dependency, Slack and release
14245
+ // work to do; none determines whether the observed issue state is safe.
14246
+ settleIssueWritebackOnce();
14247
+ // If completion raced dispatch's post-spawn read, let that read consume
14248
+ // the receipt before claiming terminal success. An unproven provider
14249
+ // transition is a foreign live-state change: dispatch owns the
14250
+ // abandonment and must release agents with that reason, not issue-done.
14251
+ if (postSpawnIssueObservation && !await postSpawnIssueObservation.settled)
14252
+ return;
13280
14253
  if (!humanReview)
13281
14254
  await this.#markDependencyTerminalAndReconcile(issue);
13282
14255
  }
14256
+ else {
14257
+ settleIssueWritebackOnce();
14258
+ }
13283
14259
  if (!await this.#saveDispatchLifecycle(record, 'writeback-applied'))
13284
14260
  return;
13285
14261
  if (issue && this.#slack && this.#config.slack && !await this.#shouldSkipSlackWriteback('completion-thread')) {
@@ -13344,6 +14320,10 @@ export class FactoryLoop {
13344
14320
  this.#scheduleDispatchLifecycleRetry(record);
13345
14321
  }
13346
14322
  finally {
14323
+ // Errors before or during provider confirmation leave the marker unset;
14324
+ // the waiting dispatch will re-read and preserve the foreign-change
14325
+ // abort. Always settle so a failed write cannot strand that dispatch.
14326
+ settleIssueWritebackOnce();
13347
14327
  this.#completionInFlight.delete(completionKey);
13348
14328
  const stateKey = issueStateKey(record.issue);
13349
14329
  this.#probePrGhBackoffUntilMs.delete(stateKey);
@@ -13360,6 +14340,35 @@ export class FactoryLoop {
13360
14340
  }
13361
14341
  }
13362
14342
  }
14343
+ #recordMissingGithubWritebackReceipt(method) {
14344
+ const implementation = this.#githubWriteback.constructor.name || 'anonymous GithubWriteback';
14345
+ this.#increment('githubWritebackReceiptMissing');
14346
+ this.#logger.warn?.('[factory] GitHub writeback returned no ownership receipt', {
14347
+ implementation,
14348
+ method,
14349
+ });
14350
+ }
14351
+ #settlePostSpawnDispatchWaits(key, accepted, source) {
14352
+ const observation = this.#postSpawnIssueObservations.get(key);
14353
+ const claimFence = this.#postSpawnDispatchClaimFences.get(key);
14354
+ if (!observation && !claimFence)
14355
+ return false;
14356
+ // Capture both before either settlement deletes its own map entry.
14357
+ if (!accepted && claimFence?.claimStarted) {
14358
+ // The rejection handler clears this provisional block only after a
14359
+ // conditional provider rollback succeeds. Until then, shutdown and held
14360
+ // abandonment must not release the lifecycle's agents.
14361
+ this.#uncompensatedDispatchClaims.add(key);
14362
+ this.#pendingDispatchClaims.add(key);
14363
+ }
14364
+ observation?.settle(accepted);
14365
+ claimFence?.settle(accepted);
14366
+ if (source === 'abandonment')
14367
+ this.#increment('postSpawnWaitsSettledByAbandonment');
14368
+ if (source === 'stop')
14369
+ this.#increment('postSpawnWaitsSettledByStop');
14370
+ return true;
14371
+ }
13363
14372
  #emit(event, payload) {
13364
14373
  for (const listener of this.#listeners.get(event) ?? []) {
13365
14374
  try {
@@ -16030,7 +17039,7 @@ const defaultGithubWriteback = (config, mount) => {
16030
17039
  if (!mount.githubWrite) {
16031
17040
  throw new Error('GitHub identity "app" requires a connected workspace GitHub App lifecycle write path; refusing to fall back to the local gh user');
16032
17041
  }
16033
- return new AppGithubWriteback(mount.githubWrite);
17042
+ return new AppGithubWriteback(mount.githubWrite, mount.githubRead);
16034
17043
  };
16035
17044
  export function parseLinearIssue(path, content) {
16036
17045
  const parsed = parseJsonContent(content);
@@ -18271,21 +19280,28 @@ const githubIssueIndexRepoRoots = (path) => {
18271
19280
  `/github/repos/${owner}__${repo}/issues`,
18272
19281
  ];
18273
19282
  };
18274
- const relayfileOverload = (error) => {
18275
- const flat = asRecord(error) ?? {};
19283
+ /** How far to follow wrapped provider failures without trusting an unbounded chain. */
19284
+ const RELAYFILE_OVERLOAD_CAUSE_DEPTH = 4;
19285
+ const relayfileOverload = (error, depth = 0) => {
19286
+ if (depth > RELAYFILE_OVERLOAD_CAUSE_DEPTH)
19287
+ return undefined;
19288
+ const flat = asRecord(error);
19289
+ if (!flat)
19290
+ return undefined;
18276
19291
  const response = asRecord(flat.response) ?? {};
18277
19292
  const data = asRecord(flat.data) ?? asRecord(response.data) ?? {};
18278
19293
  const details = asRecord(flat.details) ?? asRecord(data.details) ?? {};
18279
19294
  const statusValue = flat.status ?? flat.statusCode ?? response.status ?? response.statusCode;
18280
19295
  const status = typeof statusValue === 'number' ? statusValue : Number(statusValue);
18281
- if (status !== 429)
18282
- return undefined;
18283
- const retryValue = flat.retryAfterSeconds ?? details.retryAfterSeconds ?? data.retryAfterSeconds;
18284
- const parsedRetry = typeof retryValue === 'number' ? retryValue : Number(retryValue);
18285
- const retryAfterSeconds = Number.isFinite(parsedRetry) && parsedRetry >= 0 ? parsedRetry : undefined;
18286
- const reason = stringValue(flat.reason) ?? stringValue(details.reason) ?? stringValue(data.reason) ??
18287
- stringValue(flat.code) ?? stringValue(data.code) ?? 'rate_limited';
18288
- return { status, reason, ...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }) };
19296
+ if (status === 429) {
19297
+ const retryValue = flat.retryAfterSeconds ?? details.retryAfterSeconds ?? data.retryAfterSeconds;
19298
+ const parsedRetry = typeof retryValue === 'number' ? retryValue : Number(retryValue);
19299
+ const retryAfterSeconds = Number.isFinite(parsedRetry) && parsedRetry >= 0 ? parsedRetry : undefined;
19300
+ const reason = stringValue(flat.reason) ?? stringValue(details.reason) ?? stringValue(data.reason) ??
19301
+ stringValue(flat.code) ?? stringValue(data.code) ?? 'rate_limited';
19302
+ return { status, reason, ...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }) };
19303
+ }
19304
+ return relayfileOverload(flat.cause, depth + 1);
18289
19305
  };
18290
19306
  /**
18291
19307
  * relayfile's overload reason codes, allowlisted.
@@ -18804,13 +19820,14 @@ const PASS_FATAL_CAUSE_DEPTH = 4;
18804
19820
  * classification has to follow the cause chain rather than trust the outermost
18805
19821
  * type.
18806
19822
  */
18807
- const wrapsErrorOfType = (error, type, depth = 0) => {
19823
+ const findWrappedErrorOfType = (error, type, depth = 0) => {
18808
19824
  if (depth > PASS_FATAL_CAUSE_DEPTH || !(error instanceof Error))
18809
- return false;
19825
+ return undefined;
18810
19826
  if (error instanceof type)
18811
- return true;
18812
- return wrapsErrorOfType(error.cause, type, depth + 1);
19827
+ return error;
19828
+ return findWrappedErrorOfType(error.cause, type, depth + 1);
18813
19829
  };
19830
+ const wrapsErrorOfType = (error, type) => findWrappedErrorOfType(error, type) !== undefined;
18814
19831
  /**
18815
19832
  * How many *unclassified* per-item failures without an intervening successful
18816
19833
  * dispatch end the pass. Named per-item conditions (a lifecycle claim refusal,
@@ -18826,6 +19843,10 @@ const UNCLASSIFIED_DISPATCH_FAILURE_LIMIT = 5;
18826
19843
  */
18827
19844
  const isClassifiedPerItemDispatchFailure = (error) => error instanceof LiveDispatchStateChangedError ||
18828
19845
  error instanceof DispatchLifecycleClaimRefusedError ||
19846
+ // An external stop/held-agent deadline rejected the post-spawn boundary
19847
+ // while a provider claim was already in flight. That lifecycle's cleanup is
19848
+ // owned by the rejecting path, so the late claim unwind is an expected skip.
19849
+ error instanceof PostSpawnDispatchWaitRejectedError ||
18829
19850
  // #303: the never-placed deadline released this dispatch while its spawn was
18830
19851
  // still in flight. Named, expected and self-healing — the issue goes back to
18831
19852
  // the queue — and it recurs under exactly the slow-spawn conditions the
@@ -18850,7 +19871,8 @@ const isClassifiedPerItemDispatchFailure = (error) => error instanceof LiveDispa
18850
19871
  * of a missed one is leaked agents and duplicate workers on the next retry.
18851
19872
  */
18852
19873
  const mayHaveSpawnedBeforeFailing = (error) => !(error instanceof LiveDispatchStateChangedError) &&
18853
- !(error instanceof DispatchLifecycleClaimRefusedError);
19874
+ !(error instanceof DispatchLifecycleClaimRefusedError) &&
19875
+ !(error instanceof PostSpawnDispatchWaitRejectedError);
18854
19876
  /**
18855
19877
  * The run-report reason recorded for a work unit the pass could not dispatch.
18856
19878
  *
@@ -18868,6 +19890,8 @@ const perItemDispatchSkipReason = (error) => {
18868
19890
  return 'live state changed during dispatch';
18869
19891
  if (error instanceof LatePlacementReleasedError)
18870
19892
  return 'dispatch released while its agent was still spawning';
19893
+ if (error instanceof PostSpawnDispatchWaitRejectedError)
19894
+ return 'dispatch terminated during post-spawn claim';
18871
19895
  if (error instanceof DispatchLifecycleClaimRefusedError) {
18872
19896
  return error.refusal === 'terminal'
18873
19897
  ? 'dispatch lifecycle already terminal'
@@ -18875,6 +19899,54 @@ const perItemDispatchSkipReason = (error) => {
18875
19899
  }
18876
19900
  return `dispatch failed (${telemetryErrorClass(error)})`;
18877
19901
  };
19902
+ const UNCLASSIFIED_PHASE_CODES = {
19903
+ gate: 'unclassified-gate',
19904
+ triage: 'unclassified-triage',
19905
+ dispatch: 'unclassified-dispatch',
19906
+ };
19907
+ /**
19908
+ * Why a dispatch attempt failed, as a code from the published vocabulary (#355).
19909
+ *
19910
+ * The sibling of `perItemDispatchSkipReason`: that returns the operator's
19911
+ * sentence, this returns the one token that may cross onto the unauthenticated
19912
+ * health surface, and both are recorded at the skip site from the same thrown
19913
+ * value. The token is never parsed back out of the sentence — a reworded
19914
+ * message would silently empty a bucket, and this vocabulary is what an
19915
+ * operator reads when the daemon's stdout does not reach them.
19916
+ *
19917
+ * Ordered most specific first. Every branch follows the bounded cause chain,
19918
+ * because `contextualError` and the control-plane guard both rethrow wrapped.
19919
+ * `relayfileOverload` is also the loop's shedding predicate, so widening it at
19920
+ * the source keeps the health code, skip counter, fuse, and durable overload
19921
+ * ratchet on one verdict instead of merely relabelling the published bucket.
19922
+ */
19923
+ const perItemDispatchFailureCode = (error, phase) => {
19924
+ if (relayfileOverload(error) !== undefined)
19925
+ return 'relayfile-overloaded';
19926
+ if (wrapsErrorOfType(error, LiveDispatchStateChangedError))
19927
+ return 'live-state-changed';
19928
+ if (wrapsErrorOfType(error, LatePlacementReleasedError))
19929
+ return 'late-placement-released';
19930
+ const refused = findWrappedErrorOfType(error, DispatchLifecycleClaimRefusedError);
19931
+ if (refused)
19932
+ return refused.refusal === 'terminal' ? 'lifecycle-terminal' : 'lifecycle-owned-elsewhere';
19933
+ if (wrapsErrorOfType(error, FleetControlPlaneCircuitOpenError))
19934
+ return 'control-plane-open';
19935
+ // The class-name tail of the vocabulary; see its own doc comment for why
19936
+ // these five are not `instanceof`. Walked down the cause chain like the
19937
+ // branches above, because `contextualError` wraps in a plain `Error` and
19938
+ // reading only the outermost name would miss every wrapped spawn failure.
19939
+ // `telemetryErrorClass` is the same allowlist that guards every other
19940
+ // identifier leaving this process, so a hostile `name` cannot invent a key
19941
+ // here either — it collapses to `Error`, which the map does not hold.
19942
+ for (let cursor = error, depth = 0; cursor instanceof Error && depth <= PASS_FATAL_CAUSE_DEPTH; depth += 1) {
19943
+ const named = factoryDispatchFailureReasonCodeForErrorClass(telemetryErrorClass(cursor));
19944
+ if (named)
19945
+ return named;
19946
+ cursor = cursor.cause;
19947
+ }
19948
+ return UNCLASSIFIED_PHASE_CODES[phase];
19949
+ };
18878
19950
  const triageEscalationQuestion = (decision, issue) => {
18879
19951
  const routedRepos = decision.routes.map((route) => route.repo).filter(Boolean);
18880
19952
  const subject = issue?.title?.trim() || decision.issue.key;