@agent-relay/factory 0.1.73 → 0.1.74

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/README.md +9 -0
  2. package/dist/cli/diagnose.d.ts +15 -0
  3. package/dist/cli/diagnose.d.ts.map +1 -1
  4. package/dist/cli/diagnose.js +34 -0
  5. package/dist/cli/diagnose.js.map +1 -1
  6. package/dist/cli/fleet.d.ts.map +1 -1
  7. package/dist/cli/fleet.js +5 -0
  8. package/dist/cli/fleet.js.map +1 -1
  9. package/dist/cli/teammate-mcp.d.ts +19 -0
  10. package/dist/cli/teammate-mcp.d.ts.map +1 -0
  11. package/dist/cli/teammate-mcp.js +102 -0
  12. package/dist/cli/teammate-mcp.js.map +1 -0
  13. package/dist/environments/load-profile.d.ts +4 -4
  14. package/dist/environments/stack-descriptor.d.ts +17 -17
  15. package/dist/environments/verification-stack-descriptor.d.ts +32 -32
  16. package/dist/fleet/create-fleet.d.ts +2 -0
  17. package/dist/fleet/create-fleet.d.ts.map +1 -1
  18. package/dist/fleet/create-fleet.js +1 -0
  19. package/dist/fleet/create-fleet.js.map +1 -1
  20. package/dist/fleet/internal-fleet-client.d.ts +29 -1
  21. package/dist/fleet/internal-fleet-client.d.ts.map +1 -1
  22. package/dist/fleet/internal-fleet-client.js +161 -11
  23. package/dist/fleet/internal-fleet-client.js.map +1 -1
  24. package/dist/fleet/relay-fleet-client.d.ts +23 -1
  25. package/dist/fleet/relay-fleet-client.d.ts.map +1 -1
  26. package/dist/fleet/relay-fleet-client.js +167 -2
  27. package/dist/fleet/relay-fleet-client.js.map +1 -1
  28. package/dist/fleet/teammates.d.ts +46 -0
  29. package/dist/fleet/teammates.d.ts.map +1 -0
  30. package/dist/fleet/teammates.js +424 -0
  31. package/dist/fleet/teammates.js.map +1 -0
  32. package/dist/index.d.ts +9 -2
  33. package/dist/index.d.ts.map +1 -1
  34. package/dist/index.js +4 -0
  35. package/dist/index.js.map +1 -1
  36. package/dist/mount/relayfile-cloud-mount-client.d.ts +4 -1
  37. package/dist/mount/relayfile-cloud-mount-client.d.ts.map +1 -1
  38. package/dist/mount/relayfile-cloud-mount-client.js +18 -6
  39. package/dist/mount/relayfile-cloud-mount-client.js.map +1 -1
  40. package/dist/mount/relayfile-github-connection-write.d.ts +3 -2
  41. package/dist/mount/relayfile-github-connection-write.d.ts.map +1 -1
  42. package/dist/mount/relayfile-github-connection-write.js +33 -0
  43. package/dist/mount/relayfile-github-connection-write.js.map +1 -1
  44. package/dist/node/factory-node-runtime.d.ts +17 -0
  45. package/dist/node/factory-node-runtime.d.ts.map +1 -0
  46. package/dist/node/factory-node-runtime.js +100 -0
  47. package/dist/node/factory-node-runtime.js.map +1 -0
  48. package/dist/node/factory-node.d.ts +10 -1
  49. package/dist/node/factory-node.d.ts.map +1 -1
  50. package/dist/node/factory-node.js +13 -2
  51. package/dist/node/factory-node.js.map +1 -1
  52. package/dist/node/factory-persona-card.d.ts +38 -0
  53. package/dist/node/factory-persona-card.d.ts.map +1 -0
  54. package/dist/node/factory-persona-card.js +163 -0
  55. package/dist/node/factory-persona-card.js.map +1 -0
  56. package/dist/node/factory.node.d.ts +1 -1
  57. package/dist/observability/events.d.ts +24 -24
  58. package/dist/orchestrator/factory.d.ts.map +1 -1
  59. package/dist/orchestrator/factory.js +1016 -75
  60. package/dist/orchestrator/factory.js.map +1 -1
  61. package/dist/orchestrator/public-health.d.ts.map +1 -1
  62. package/dist/orchestrator/public-health.js +36 -0
  63. package/dist/orchestrator/public-health.js.map +1 -1
  64. package/dist/orchestrator/release-error.d.ts +26 -0
  65. package/dist/orchestrator/release-error.d.ts.map +1 -0
  66. package/dist/orchestrator/release-error.js +33 -0
  67. package/dist/orchestrator/release-error.js.map +1 -0
  68. package/dist/ports/fleet.d.ts +66 -0
  69. package/dist/ports/fleet.d.ts.map +1 -1
  70. package/dist/ports/fleet.js +10 -1
  71. package/dist/ports/fleet.js.map +1 -1
  72. package/dist/ports/index.d.ts +4 -3
  73. package/dist/ports/index.d.ts.map +1 -1
  74. package/dist/ports/index.js +1 -1
  75. package/dist/ports/index.js.map +1 -1
  76. package/dist/ports/mount.d.ts +10 -2
  77. package/dist/ports/mount.d.ts.map +1 -1
  78. package/dist/ports/writeback.d.ts +45 -4
  79. package/dist/ports/writeback.d.ts.map +1 -1
  80. package/dist/state/watch-state-document.js +4 -1
  81. package/dist/state/watch-state-document.js.map +1 -1
  82. package/dist/testing/fakes.d.ts +8 -3
  83. package/dist/testing/fakes.d.ts.map +1 -1
  84. package/dist/testing/fakes.js +25 -1
  85. package/dist/testing/fakes.js.map +1 -1
  86. package/dist/triage/schema.d.ts +14 -14
  87. package/dist/types.d.ts +52 -0
  88. package/dist/types.d.ts.map +1 -1
  89. package/dist/writeback/github.d.ts +17 -9
  90. package/dist/writeback/github.d.ts.map +1 -1
  91. package/dist/writeback/github.js +331 -9
  92. package/dist/writeback/github.js.map +1 -1
  93. package/dist/writeback/linear.d.ts +5 -1
  94. package/dist/writeback/linear.d.ts.map +1 -1
  95. package/dist/writeback/linear.js +89 -1
  96. package/dist/writeback/linear.js.map +1 -1
  97. package/package.json +4 -2
@@ -1,3 +1,4 @@
1
+ import { AsyncLocalStorage } from 'node:async_hooks';
1
2
  import { randomUUID } from 'node:crypto';
2
3
  import { readFile } from 'node:fs/promises';
3
4
  import { dirname, isAbsolute, resolve } from 'node:path';
@@ -37,6 +38,7 @@ import { telemetryErrorClass } from '../observability/error-class.js';
37
38
  import { derivedReadinessReconcileState, publicHealthFromHeartbeat, readinessReconcileInFlightMs, } from './public-health.js';
38
39
  import { factorySweepSkipReasonCounts } from './sweep-skip-reason.js';
39
40
  import { factoryDispatchFailureReasonCodeForErrorClass, factoryDispatchFailureReasonCounts, } from './dispatch-failure-reason.js';
41
+ import { isAgentAlreadyGoneOnRelease } from './release-error.js';
40
42
  import { boundedRunCostTotal, CostLedger } from '../cost/ledger.js';
41
43
  import { createTicketDispatchDelivery } from '../delivery/ticket-dispatch.js';
42
44
  import { canonicalTrajectorySessionRef, renderTrajectoryPointer, stripTrajectoryPointers, } from '../trajectory.js';
@@ -144,6 +146,12 @@ const CLARIFICATION_ESCALATION_LEASE_MS = 2 * 60_000;
144
146
  const CLARIFICATION_ESCALATION_RETRY_MS = 5_000;
145
147
  const CLARIFICATION_STALE_WARN_MS = 7 * 24 * 60 * 60_000;
146
148
  const STOP_TEARDOWN_TIMEOUT_MS = 2_500;
149
+ // A rejected post-spawn dispatch normally gets a final opportunity to undo an
150
+ // external claim before shutdown relinquishes its lifecycle lease. Provider
151
+ // writes are not guaranteed to settle, so that opportunity must stay inside a
152
+ // bounded shutdown budget; a successor can recover from the retained durable
153
+ // lifecycle after this process releases its local agents.
154
+ const STOP_REJECTED_DISPATCH_DRAIN_TIMEOUT_MS = 2_500;
147
155
  const DISPATCH_LIFECYCLE_LEASE_MS = 5 * 60_000;
148
156
  const DISPATCH_LIFECYCLE_RENEW_MS = 60_000;
149
157
  const DISPATCH_LIFECYCLE_RETRY_MS = 1_000;
@@ -348,6 +356,16 @@ class ReadinessReconcileTimeoutError extends Error {
348
356
  this.name = 'ReadinessReconcileTimeoutError';
349
357
  }
350
358
  }
359
+ class PostSpawnDispatchWaitRejectedError extends Error {
360
+ issueKey;
361
+ compensationError;
362
+ constructor(issueKey, compensationError) {
363
+ super(`Post-spawn dispatch waits were rejected for ${issueKey}`);
364
+ this.issueKey = issueKey;
365
+ this.compensationError = compensationError;
366
+ this.name = 'PostSpawnDispatchWaitRejectedError';
367
+ }
368
+ }
351
369
  const realClock = {
352
370
  now: () => Date.now(),
353
371
  sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
@@ -355,6 +373,26 @@ const realClock = {
355
373
  export function createFactory(config, ports) {
356
374
  return new FactoryLoop(FactoryConfigSchema.parse(config), ports);
357
375
  }
376
+ /**
377
+ * The discovery pass a tree read was issued by, carried per async call (#363
378
+ * review, CodeRabbit).
379
+ *
380
+ * `#discoverySweepEpoch` says a sweep is in flight; it cannot say the read in
381
+ * hand belongs to it. In live mode an event drain reaches the very same
382
+ * enumeration helpers — `#handlePrChange` -> `#advanceMergedPrToDone` ->
383
+ * `#findMergeAdvanceIssueForPr` -> `#githubIssuePaths()` is the concrete path —
384
+ * so a drain's full-root walk would land in a concurrent sweep's ratio while
385
+ * measuring a different instant. That is enough to hide a mount that went
386
+ * silent mid-sweep.
387
+ *
388
+ * A drain does not inherit this store: its continuation begins at the
389
+ * subscription callback, outside the `run()` below. Threading an epoch argument
390
+ * through `#ingestGithubIssues` / `#handleGithubIssueChange` /
391
+ * `#findGithubIssueMirror` / `#loadLinearMirrorCandidates` would express the
392
+ * same fact and put a parameter on every hot path that must never be passed
393
+ * wrong.
394
+ */
395
+ const discoveryEnumerationPass = new AsyncLocalStorage();
358
396
  export class FactoryLoop {
359
397
  #config;
360
398
  #mount;
@@ -590,6 +628,29 @@ export class FactoryLoop {
590
628
  #previewSweepTimer;
591
629
  #previewSweepInFlight;
592
630
  #completionInFlight = new Set();
631
+ #issueWritebackInFlight = new Map();
632
+ // A fast completion can make terminal issue state visible while dispatch is
633
+ // still performing its post-spawn readiness read. Completion publishes its
634
+ // provider receipt, then waits for that read to classify the state as owned
635
+ // or foreign before it releases agents under a terminal success reason.
636
+ #postSpawnIssueObservations = new Map();
637
+ // The post-spawn ready read and following dispatch claim form one local
638
+ // write boundary. A completion arriving first makes dispatch wait and
639
+ // re-read; one arriving after claim entry waits for the claim to finish.
640
+ #postSpawnDispatchClaimFences = new Map();
641
+ // Shutdown must not lose a local placement merely because a rejected
642
+ // provider claim unwinds before #releaseInFlightAgents snapshots the batch.
643
+ // The capture set makes both interleavings explicit: the dispatch catch may
644
+ // remove a record only after shutdown has copied its agents, otherwise the
645
+ // release sweep removes it after the copy is safe.
646
+ #shutdownReleaseCapturedDispatches = new Set();
647
+ #postSpawnDispatchesRejectedDuringStop = new Set();
648
+ // A rejected provider claim is unsafe to release until its conditional
649
+ // rollback succeeds or a newer provider status supersedes it. This runtime
650
+ // set closes the race before the durable dispatchClaim flag is persisted.
651
+ #uncompensatedDispatchClaims = new Set();
652
+ #pendingDispatchClaims = new Set();
653
+ #dispatchClaimSettlementsInFlight = new Set();
593
654
  #agentExitsInFlight = new Map();
594
655
  #reconciledAgentExitsActive = 0;
595
656
  #reconciledAgentExitWaiters = [];
@@ -678,6 +739,10 @@ export class FactoryLoop {
678
739
  #discoveryOverloadError;
679
740
  /** Relayfile operations this sweep has been shed on. */
680
741
  #discoverySweepOverloads = 0;
742
+ /** Tree reads this sweep issued that the backend served (#351 follow-up). */
743
+ #discoverySweepTreeReads = 0;
744
+ /** How many of those were served with zero entries. */
745
+ #discoverySweepEmptyTreeReads = 0;
681
746
  /**
682
747
  * The longest `Retry-After` any operation in this sweep advertised.
683
748
  *
@@ -984,6 +1049,15 @@ export class FactoryLoop {
984
1049
  }
985
1050
  }
986
1051
  async #start(opts) {
1052
+ // These sets coordinate one stop snapshot with dispatches unwinding during
1053
+ // that same stop. They are not durable lifecycle state: retaining their
1054
+ // keys across a later start would make the next stop mistake a fresh
1055
+ // dispatch for one already captured by the previous shutdown.
1056
+ this.#shutdownReleaseCapturedDispatches.clear();
1057
+ this.#postSpawnDispatchesRejectedDuringStop.clear();
1058
+ this.#uncompensatedDispatchClaims.clear();
1059
+ this.#pendingDispatchClaims.clear();
1060
+ this.#dispatchClaimSettlementsInFlight.clear();
987
1061
  this.#stopping = false;
988
1062
  this.#startMode = opts.mode ?? 'live';
989
1063
  const issueSource = await this.#issueSource();
@@ -1098,15 +1172,32 @@ export class FactoryLoop {
1098
1172
  async stop() {
1099
1173
  this.#started = false;
1100
1174
  this.#stopping = true;
1175
+ // These waits are armed before every planned spawn returns. Resolve them
1176
+ // before the first shutdown await: a fast first agent may be completing
1177
+ // while a later spawn is hung, and #drainAgentExitsInFlight would otherwise
1178
+ // wait forever on a resolver stranded inside that dispatch stack.
1179
+ const postSpawnWaitKeys = new Set([
1180
+ ...this.#postSpawnIssueObservations.keys(),
1181
+ ...this.#postSpawnDispatchClaimFences.keys(),
1182
+ ]);
1183
+ const rejectedClaimDispatches = new Set();
1184
+ for (const [key, fence] of this.#postSpawnDispatchClaimFences) {
1185
+ if (!fence.claimStarted)
1186
+ continue;
1187
+ const dispatchKeyPrefix = `${key}:live:`;
1188
+ for (const [dispatchKey, dispatch] of this.#dispatchInFlight) {
1189
+ if (dispatchKey.startsWith(dispatchKeyPrefix))
1190
+ rejectedClaimDispatches.add(dispatch);
1191
+ }
1192
+ }
1193
+ for (const key of postSpawnWaitKeys)
1194
+ this.#settlePostSpawnDispatchWaits(key, false, 'stop');
1101
1195
  if (this.#babysitterResourceDeliveryRetryTimer)
1102
1196
  clearTimeout(this.#babysitterResourceDeliveryRetryTimer);
1103
1197
  this.#babysitterResourceDeliveryRetryTimer = undefined;
1104
1198
  if (this.#babysitterResourceSubscriptionRenewTimer)
1105
1199
  clearTimeout(this.#babysitterResourceSubscriptionRenewTimer);
1106
1200
  this.#babysitterResourceSubscriptionRenewTimer = undefined;
1107
- if (this.#dispatchLifecycleRenewTimer)
1108
- clearInterval(this.#dispatchLifecycleRenewTimer);
1109
- this.#dispatchLifecycleRenewTimer = undefined;
1110
1201
  if (this.#heldAgentDeadlineTimer)
1111
1202
  clearTimeout(this.#heldAgentDeadlineTimer);
1112
1203
  this.#heldAgentDeadlineTimer = undefined;
@@ -1139,12 +1230,25 @@ export class FactoryLoop {
1139
1230
  await this.#previewSweepInFlight;
1140
1231
  this.#stoppingHeartbeatRefreshActive = await this.#stopLiveHeartbeat('stopping');
1141
1232
  try {
1233
+ // Only dispatches that had entered the provider-claim boundary can leave
1234
+ // an external claim behind. Drain those rejected promises before
1235
+ // relinquishing their lifecycle leases or tearing down the fleet/mount.
1236
+ // A pre-claim spawn hang is deliberately absent from this set, so it
1237
+ // cannot hold shutdown open.
1238
+ await this.#drainRejectedClaimDispatchesForStop(rejectedClaimDispatches);
1239
+ await this.#persistOutstandingClaimCancellationBlocks();
1240
+ // Keep renewing ownership through the potentially slow provider drain
1241
+ // and cancellation-fence handoff. Stopping renewal earlier can let the
1242
+ // lease expire before the fence is saved, allowing a successor to reap
1243
+ // agents while the old provider write can still land.
1142
1244
  // Relinquish durable ownership before waiting on mount-backed lifecycle
1143
1245
  // drives. A slow Relayfile scan must not consume the shutdown deadline
1144
1246
  // while every issue remains fenced to a publisher that is already
1145
- // stopping. The owner/epoch fence makes any late completion from those
1146
- // drives harmless; a second sweep below catches claims racing this one.
1147
- await this.#releaseOwnedDispatchLifecycleLeases();
1247
+ // stopping. Preserve leases for provider claims whose bounded drain
1248
+ // expired: the original caller still needs its epoch to persist a late
1249
+ // conclusive settlement. A second sweep below catches claims racing this
1250
+ // one.
1251
+ await this.#releaseOwnedDispatchLifecycleLeases((key) => this.#pendingDispatchClaims.has(key) || this.#dispatchClaimSettlementsInFlight.has(key));
1148
1252
  await Promise.allSettled([...this.#dispatchLifecycleDrives]);
1149
1253
  // Fence every source of new clarification work before touching the fleet.
1150
1254
  // A wake already past the fence is allowed to unwind, and is awaited
@@ -1167,13 +1271,18 @@ export class FactoryLoop {
1167
1271
  // non-durable (local/internal) records; terminal completion performs the
1168
1272
  // normal remote release before clearing the lifecycle.
1169
1273
  await this.#releaseInFlightAgents('factory-stopped', { preserveDurable: true });
1170
- await this.#releaseOwnedDispatchLifecycleLeases();
1274
+ if (this.#dispatchLifecycleRenewTimer)
1275
+ clearInterval(this.#dispatchLifecycleRenewTimer);
1276
+ this.#dispatchLifecycleRenewTimer = undefined;
1277
+ await this.#releaseOwnedDispatchLifecycleLeases((key) => this.#pendingDispatchClaims.has(key) || this.#dispatchClaimSettlementsInFlight.has(key));
1171
1278
  if (this.#livePollTimer)
1172
1279
  clearTimeout(this.#livePollTimer);
1173
1280
  this.#livePollTimer = undefined;
1174
1281
  this.#livePollInFlight = false;
1175
1282
  this.#liveEventQueue.length = 0;
1176
1283
  this.#completionInFlight.clear();
1284
+ this.#postSpawnIssueObservations.clear();
1285
+ this.#postSpawnDispatchClaimFences.clear();
1177
1286
  this.#babysitterSpawned.clear();
1178
1287
  this.#babysitterPr.clear();
1179
1288
  this.#babysitterIssueRefs.clear();
@@ -1220,10 +1329,13 @@ export class FactoryLoop {
1220
1329
  await this.#fleet.dispose();
1221
1330
  }
1222
1331
  finally {
1332
+ if (this.#dispatchLifecycleRenewTimer)
1333
+ clearInterval(this.#dispatchLifecycleRenewTimer);
1334
+ this.#dispatchLifecycleRenewTimer = undefined;
1223
1335
  this.#stoppingHeartbeatRefreshActive = false;
1224
1336
  }
1225
1337
  }
1226
- async #releaseOwnedDispatchLifecycleLeases() {
1338
+ async #releaseOwnedDispatchLifecycleLeases(preserve = () => false) {
1227
1339
  // The epoch cache is an execution optimization, not the durable ownership
1228
1340
  // authority. Error/fence paths may evict a cached epoch while its persisted
1229
1341
  // lease is still ours, so enumerate state before shutdown relinquishment.
@@ -1234,6 +1346,8 @@ export class FactoryLoop {
1234
1346
  }
1235
1347
  }
1236
1348
  for (const [key, epoch] of owned) {
1349
+ if (preserve(key))
1350
+ continue;
1237
1351
  await this.#state.releaseDispatchLifecycleLease(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch);
1238
1352
  if (this.#dispatchLifecycleEpochs.get(key) === epoch) {
1239
1353
  this.#dispatchLifecycleEpochs.delete(key);
@@ -1257,6 +1371,215 @@ export class FactoryLoop {
1257
1371
  await Promise.allSettled([...this.#clarificationQuestionDeliveryInFlight.values()]);
1258
1372
  }
1259
1373
  }
1374
+ async #drainRejectedClaimDispatchesForStop(dispatches) {
1375
+ if (dispatches.size === 0)
1376
+ return;
1377
+ let timer;
1378
+ const drained = Promise.allSettled([...dispatches]).then(() => true);
1379
+ // A durable fleet can recover ownership after this process exits. Local
1380
+ // and other non-durable fleets cannot: releasing their placements before
1381
+ // the provider claim has compensated would leave no successor able to
1382
+ // finish the lifecycle, and a late write could strand the issue in its
1383
+ // claimed state. Keep draining those fleets without the shutdown escape
1384
+ // hatch.
1385
+ if (this.#fleet.durableOwnership !== true) {
1386
+ await drained;
1387
+ return;
1388
+ }
1389
+ const timedOut = new Promise((resolve) => {
1390
+ timer = setTimeout(() => resolve(false), STOP_REJECTED_DISPATCH_DRAIN_TIMEOUT_MS);
1391
+ timer.unref?.();
1392
+ });
1393
+ const completed = await Promise.race([drained, timedOut]);
1394
+ if (timer)
1395
+ clearTimeout(timer);
1396
+ if (completed)
1397
+ return;
1398
+ this.#increment('postSpawnDispatchClaimDrainTimeouts');
1399
+ this.#logger.warn?.('[factory] rejected post-spawn dispatch compensation timed out; continuing shutdown', {
1400
+ dispatches: dispatches.size,
1401
+ timeoutMs: STOP_REJECTED_DISPATCH_DRAIN_TIMEOUT_MS,
1402
+ });
1403
+ }
1404
+ async #persistOutstandingClaimCancellationBlocks() {
1405
+ const batch = await this.#batch();
1406
+ for (const record of batch.inFlight) {
1407
+ const key = dispatchLifecycleKey(record.issue);
1408
+ if (!this.#pendingDispatchClaims.has(key))
1409
+ continue;
1410
+ // The bounded durable drain expired before the provider operation could
1411
+ // report whether rollback was safe. Persist the provisional release
1412
+ // fence while this publisher still owns the lifecycle lease, so a
1413
+ // successor cannot mistake the unresolved claim for abandonable work.
1414
+ const persisted = await this.#retainUncompensatedDispatchClaim(record, new Error('Rejected dispatch claim compensation did not settle before shutdown'), { preferExistingError: true, pending: true });
1415
+ if (!persisted) {
1416
+ const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
1417
+ if (durable?.dispatchClaim?.cancellationPending !== true) {
1418
+ throw new Error(`Unable to persist unresolved dispatch-claim fence for ${record.issue.key}; refusing shutdown handoff`);
1419
+ }
1420
+ }
1421
+ }
1422
+ }
1423
+ async #awaitRejectedClaimCompensation(settled) {
1424
+ let timer;
1425
+ const timedOut = new Promise((resolve) => {
1426
+ timer = setTimeout(() => resolve(false), STOP_REJECTED_DISPATCH_DRAIN_TIMEOUT_MS);
1427
+ timer.unref?.();
1428
+ });
1429
+ const compensated = await Promise.race([settled, timedOut]);
1430
+ if (timer)
1431
+ clearTimeout(timer);
1432
+ return compensated;
1433
+ }
1434
+ /**
1435
+ * Re-adopt this publisher's own lease epoch after a refused lifecycle write.
1436
+ *
1437
+ * `#saveDispatchLifecycle` cannot tell "the store rejected our fence" from
1438
+ * "the store declined this row's contents", so it evicts the cached epoch for
1439
+ * both. Only the first is a loss of ownership. Re-read the durable row and
1440
+ * take the epoch back when the lease is still ours and still live; a row that
1441
+ * is gone, terminal, expired, or owned by someone else keeps the eviction,
1442
+ * because for those the eviction was correct.
1443
+ */
1444
+ async #recoverDispatchLifecycleLeaseEpoch(key) {
1445
+ if (this.#dispatchLifecycleEpochs.has(key))
1446
+ return true;
1447
+ const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
1448
+ const lease = lifecycle?.lease;
1449
+ if (!lifecycle || !lease || isTerminalDispatchLifecycle(lifecycle))
1450
+ return false;
1451
+ if (lease.owner !== this.#dispatchLifecycleOwner)
1452
+ return false;
1453
+ if (lease.leaseUntilMs <= this.#clock.now())
1454
+ return false;
1455
+ this.#dispatchLifecycleEpochs.set(key, lease.epoch);
1456
+ this.#increment('dispatchLifecycleLeaseEpochsRecovered');
1457
+ return true;
1458
+ }
1459
+ /**
1460
+ * Persist a dispatch-claim fence without letting a refused write disarm the
1461
+ * lease the next one needs.
1462
+ *
1463
+ * The shutdown handoff deliberately probes with a provisional
1464
+ * `cancellationPending` fence, and a store may refuse that row on its
1465
+ * contents while this process still holds the lease. Losing the epoch there
1466
+ * would make every later write for this lifecycle short-circuit on
1467
+ * `epoch === undefined` — including the conclusive compensated settlement
1468
+ * that clears the block and frees the agents and the batch slot. That turns
1469
+ * one refused probe into a lifecycle retained until restart, so bracket the
1470
+ * write with epoch recovery instead.
1471
+ */
1472
+ async #persistDispatchClaimFence(record) {
1473
+ const key = dispatchLifecycleKey(record.issue);
1474
+ await this.#recoverDispatchLifecycleLeaseEpoch(key);
1475
+ const saved = await this.#saveDispatchLifecycle(record, 'running');
1476
+ if (!saved)
1477
+ await this.#recoverDispatchLifecycleLeaseEpoch(key);
1478
+ return saved;
1479
+ }
1480
+ async #retainUncompensatedDispatchClaim(record, error, opts = {}) {
1481
+ const key = dispatchLifecycleKey(record.issue);
1482
+ const alreadyBlocked = record.dispatchClaim?.cancellationBlocked === true;
1483
+ const errorMessage = describeError(error).errorMessage;
1484
+ this.#uncompensatedDispatchClaims.add(key);
1485
+ const { cancellationPending: _pending, ...existingClaim } = record.dispatchClaim ?? { state: 'degraded' };
1486
+ record.dispatchClaim = {
1487
+ ...existingClaim,
1488
+ state: 'degraded',
1489
+ cancellationBlocked: true,
1490
+ ...(opts.pending ? { cancellationPending: true } : {}),
1491
+ write: 'rejected dispatch claim rollback',
1492
+ error: opts.preferExistingError ? record.dispatchClaim?.error ?? errorMessage : errorMessage,
1493
+ deadLettered: true,
1494
+ updatedAtMs: this.#clock.now(),
1495
+ };
1496
+ this.#dispatchClaimStatuses.set(key, record.dispatchClaim);
1497
+ if (!alreadyBlocked) {
1498
+ this.#increment('postSpawnDispatchClaimRecoveryRetentions');
1499
+ this.#logger.warn?.('[factory] retaining dispatch lifecycle because rejected claim compensation is unproven', {
1500
+ issue: record.issue.key,
1501
+ });
1502
+ }
1503
+ if (this.#usesDurableDispatchLifecycle()) {
1504
+ return await this.#persistDispatchClaimFence(record);
1505
+ }
1506
+ else {
1507
+ await this.#writeDispatchClaimRegistry(record.issue);
1508
+ return true;
1509
+ }
1510
+ }
1511
+ async #clearDispatchCancellationBlock(record) {
1512
+ const key = dispatchLifecycleKey(record.issue);
1513
+ if (record.dispatchClaim?.cancellationBlocked !== true) {
1514
+ this.#uncompensatedDispatchClaims.delete(key);
1515
+ return true;
1516
+ }
1517
+ const previousClaim = record.dispatchClaim;
1518
+ const { cancellationBlocked: _blocked, cancellationPending: _pending, ...claim } = previousClaim;
1519
+ record.dispatchClaim = claim;
1520
+ this.#dispatchClaimStatuses.set(key, claim);
1521
+ try {
1522
+ const persisted = this.#usesDurableDispatchLifecycle()
1523
+ ? await this.#persistDispatchClaimFence(record)
1524
+ : await this.#writeDispatchClaimRegistry(record.issue).then(() => true);
1525
+ if (!persisted) {
1526
+ record.dispatchClaim = previousClaim;
1527
+ this.#dispatchClaimStatuses.set(key, previousClaim);
1528
+ return false;
1529
+ }
1530
+ this.#uncompensatedDispatchClaims.delete(key);
1531
+ return true;
1532
+ }
1533
+ catch (error) {
1534
+ record.dispatchClaim = previousClaim;
1535
+ this.#dispatchClaimStatuses.set(key, previousClaim);
1536
+ throw error;
1537
+ }
1538
+ }
1539
+ async #dispatchClaimBlocksAbandonment(record) {
1540
+ const key = dispatchLifecycleKey(record.issue);
1541
+ if (!this.#uncompensatedDispatchClaims.has(key) && record.dispatchClaim?.cancellationBlocked !== true) {
1542
+ return false;
1543
+ }
1544
+ // A provider request that was still unresolved at handoff can land after
1545
+ // any current-state read. No observation is proof of supersession until
1546
+ // the original caller records that the operation conclusively settled.
1547
+ if (record.dispatchClaim?.cancellationPending === true)
1548
+ return true;
1549
+ try {
1550
+ const issue = await this.#readIssue(record.issue.path);
1551
+ if (!issue)
1552
+ return true;
1553
+ if (isGithubIssue(issue)) {
1554
+ if (!this.#githubWriteback.getIssueStatus)
1555
+ return true;
1556
+ const status = await this.#githubWriteback.getIssueStatus(issue, {
1557
+ requireFresh: true,
1558
+ freshAfterMs: record.dispatchClaim?.claimStartedAtMs,
1559
+ });
1560
+ if (!status || status === 'in-progress')
1561
+ return true;
1562
+ }
1563
+ else {
1564
+ if (!this.#linear.getIssueStateId)
1565
+ return true;
1566
+ const implementingStateId = this.#states.idFor(issue.team, 'agentImplementing');
1567
+ const currentStateId = await this.#linear.getIssueStateId(issue);
1568
+ if (!implementingStateId || !currentStateId || currentStateId === implementingStateId)
1569
+ return true;
1570
+ }
1571
+ if (!await this.#clearDispatchCancellationBlock(record))
1572
+ return true;
1573
+ return false;
1574
+ }
1575
+ catch (error) {
1576
+ this.#logger.warn?.('[factory] unable to verify whether a blocked dispatch claim was superseded', {
1577
+ issue: record.issue.key,
1578
+ error: describeError(error).errorMessage,
1579
+ });
1580
+ return true;
1581
+ }
1582
+ }
1260
1583
  async #boundedStopTeardown(label, teardown) {
1261
1584
  let timer;
1262
1585
  const action = Promise.resolve()
@@ -2368,6 +2691,8 @@ export class FactoryLoop {
2368
2691
  this.#discoverySweepLeaseLost = false;
2369
2692
  this.#discoveryOverloadError = undefined;
2370
2693
  this.#discoverySweepOverloads = 0;
2694
+ this.#discoverySweepTreeReads = 0;
2695
+ this.#discoverySweepEmptyTreeReads = 0;
2371
2696
  this.#discoverySweepRetryAfterSeconds = undefined;
2372
2697
  this.#discoverySweepProgress = false;
2373
2698
  this.#startDiscoverySweepRenewal(claim.lease.epoch);
@@ -2448,6 +2773,8 @@ export class FactoryLoop {
2448
2773
  this.#discoverySweepStartedAtMs = undefined;
2449
2774
  this.#discoveryOverloadError = undefined;
2450
2775
  this.#discoverySweepOverloads = 0;
2776
+ this.#discoverySweepTreeReads = 0;
2777
+ this.#discoverySweepEmptyTreeReads = 0;
2451
2778
  this.#discoverySweepRetryAfterSeconds = undefined;
2452
2779
  this.#discoverySweepProgress = false;
2453
2780
  // This sweep is over either way (committed, deferred, or lease lost) —
@@ -2566,13 +2893,19 @@ export class FactoryLoop {
2566
2893
  this.#dependencyGithubPathsByIdentity = undefined;
2567
2894
  this.#dependencyLinearTreeLoaded = false;
2568
2895
  const issueSource = await this.#issueSource();
2896
+ // The sweep's own discovery pass, and the only region whose full-root
2897
+ // walks may enter the tree-read ratio (#363 review). Everything reached
2898
+ // from here inherits the marker; a concurrently running drain does not.
2899
+ const enumerate = (fn) => this.#discoverySweepEpoch === undefined
2900
+ ? fn()
2901
+ : discoveryEnumerationPass.run({ epoch: this.#discoverySweepEpoch }, fn);
2569
2902
  if (issueSource === 'linear') {
2570
- await this.#ingestGithubIssues({ dryRun });
2903
+ await enumerate(() => this.#ingestGithubIssues({ dryRun }));
2571
2904
  }
2572
2905
  else {
2573
2906
  await this.#ensureGithubIngestionReady();
2574
2907
  }
2575
- const paths = await this.#readyIssuePaths();
2908
+ const paths = await enumerate(() => this.#readyIssuePaths());
2576
2909
  const orphanRecovery = issueSource === 'github'
2577
2910
  ? await this.#githubOrphanRecoveryContext(dryRun)
2578
2911
  : undefined;
@@ -2877,6 +3210,10 @@ export class FactoryLoop {
2877
3210
  dispatched,
2878
3211
  skipped,
2879
3212
  dryRun,
3213
+ // Read before `#runOnceWithDiscoveryFence`'s finally resets them: this
3214
+ // is still inside that try, so the counts are this sweep's own.
3215
+ treeReads: this.#discoverySweepTreeReads,
3216
+ emptyTreeReads: this.#discoverySweepEmptyTreeReads,
2880
3217
  slackDegraded: this.#slackDegraded,
2881
3218
  ...(orphanRecoveryDegraded ? { orphanRecoveryDegraded } : {}),
2882
3219
  };
@@ -3782,6 +4119,10 @@ export class FactoryLoop {
3782
4119
  // that expects a specific PR to appear, or an escalation/comment-replay
3783
4120
  // scan that must not miss a marker or reply that landed after the cache
3784
4121
  // was populated. Those callers must omit `cache` and pay for a fresh list.
4122
+ //
4123
+ // `enumeration` marks the call sites that ARE the readiness sweep's
4124
+ // discovery pass — a full-root walk looking for candidate work. Only those
4125
+ // feed the tree-read pair below, and only while a sweep holds the lease.
3785
4126
  async #listRelayfileTree(prefix, phase, opts = {}) {
3786
4127
  if (this.#discoverySweepLeaseLost) {
3787
4128
  throw new Error('discovery sweep lease was lost; refusing another tree request');
@@ -3800,6 +4141,46 @@ export class FactoryLoop {
3800
4141
  logStart: true,
3801
4142
  logComplete: true,
3802
4143
  });
4144
+ // #351 follow-up: tree reads the backend SERVED, and how many of them it
4145
+ // answered with nothing.
4146
+ //
4147
+ // The per-call deadline made a hung dependency loud. This is its
4148
+ // companion: a mount that starts returning empty trees instead of hanging
4149
+ // raises no timeout, no failure and no `lastError`, and produces a sweep
4150
+ // that completes `healthy` having dispatched nothing.
4151
+ //
4152
+ // BOTH numbers, because one is not a signal. A healthy sweep lists two
4153
+ // path forms per repo and only one of them exists, so an empty read is
4154
+ // ordinary and a bare count of them fires constantly. What separates the
4155
+ // fault is the RATIO: `emptyTreeReads === treeReads` means the mount
4156
+ // served nothing at all, which `candidates: 0` cannot distinguish from a
4157
+ // workspace that simply has no ready work.
4158
+ //
4159
+ // SCOPED TWICE, and both guards are load-bearing (#363 review, codex P2).
4160
+ // The ratio is only readable if every read in it came from the same
4161
+ // question. In live mode a Slack identity lookup, a PR-confirmation poll
4162
+ // or a comment-replay scan can run concurrently with `runOnce()` — each is
4163
+ // a `listTree`, none is discovery, and one populated result among them
4164
+ // makes `emptyTreeReads < treeReads` on an all-empty sweep, silently
4165
+ // erasing exactly the signal this pair exists to raise. `enumeration`
4166
+ // keeps the numerator and denominator to the discovery pass; the epoch
4167
+ // check keeps a discovery walk issued outside any sweep — startup
4168
+ // backfill, most obviously — out of a sweep's totals.
4169
+ // `enumeration` says this is a full-root discovery walk rather than a point
4170
+ // lookup; the context says THIS sweep's discovery pass is what issued it.
4171
+ // Both, because either alone admits a read the ratio cannot use. Compared
4172
+ // by value rather than presence: an absent store and an absent epoch are
4173
+ // both `undefined` and must not read as a match.
4174
+ const issuingPass = discoveryEnumerationPass.getStore();
4175
+ if (opts.enumeration &&
4176
+ issuingPass !== undefined &&
4177
+ issuingPass.epoch === this.#discoverySweepEpoch) {
4178
+ this.#discoverySweepTreeReads += 1;
4179
+ if (paths.length === 0) {
4180
+ this.#increment('relayfileEmptyTreeReads');
4181
+ this.#discoverySweepEmptyTreeReads += 1;
4182
+ }
4183
+ }
3803
4184
  if (opts.cache)
3804
4185
  await this.#rememberDiscoveryTree(prefix, paths);
3805
4186
  return paths;
@@ -4315,6 +4696,66 @@ export class FactoryLoop {
4315
4696
  if (!dryRun)
4316
4697
  await this.#ensureGithubAgentQuestionWatch(record, liveIssue);
4317
4698
  const spawnedForReaperHandoff = [];
4699
+ // These waits belong to the durable work unit, not the ingestion surface.
4700
+ // Completion and lifecycle ownership use the same identity (#211); using
4701
+ // issueKey here would make every GitHub-native dispatch miss its own fence.
4702
+ const postSpawnKey = dispatchLifecycleKey(record.issue);
4703
+ let resolvePostSpawnIssueObservation;
4704
+ let postSpawnIssueObservationDidSettle = false;
4705
+ const postSpawnIssueObservationSettled = new Promise((resolve) => {
4706
+ resolvePostSpawnIssueObservation = resolve;
4707
+ });
4708
+ let postSpawnIssueObservation;
4709
+ postSpawnIssueObservation = {
4710
+ settled: postSpawnIssueObservationSettled,
4711
+ settle: (accepted) => {
4712
+ if (postSpawnIssueObservationDidSettle)
4713
+ return;
4714
+ postSpawnIssueObservationDidSettle = true;
4715
+ resolvePostSpawnIssueObservation(accepted);
4716
+ if (this.#postSpawnIssueObservations.get(postSpawnKey) === postSpawnIssueObservation) {
4717
+ this.#postSpawnIssueObservations.delete(postSpawnKey);
4718
+ }
4719
+ },
4720
+ };
4721
+ this.#postSpawnIssueObservations.set(postSpawnKey, postSpawnIssueObservation);
4722
+ let resolvePostSpawnDispatchClaim;
4723
+ let postSpawnDispatchClaimSettled = false;
4724
+ const postSpawnDispatchClaim = new Promise((resolve) => {
4725
+ resolvePostSpawnDispatchClaim = resolve;
4726
+ });
4727
+ let resolveRejectedClaimCompensation;
4728
+ let rejectedClaimCompensationSettled = false;
4729
+ const rejectedClaimCompensation = new Promise((resolve) => {
4730
+ resolveRejectedClaimCompensation = resolve;
4731
+ });
4732
+ let postSpawnDispatchClaimFence;
4733
+ postSpawnDispatchClaimFence = {
4734
+ completionAtWriteBoundary: false,
4735
+ claimStarted: false,
4736
+ settled: postSpawnDispatchClaim,
4737
+ rejectionSettled: rejectedClaimCompensation,
4738
+ settle: (accepted) => {
4739
+ if (postSpawnDispatchClaimSettled)
4740
+ return;
4741
+ postSpawnDispatchClaimSettled = true;
4742
+ postSpawnDispatchClaimFence.accepted = accepted;
4743
+ resolvePostSpawnDispatchClaim(accepted);
4744
+ if (this.#postSpawnDispatchClaimFences.get(postSpawnKey) === postSpawnDispatchClaimFence) {
4745
+ this.#postSpawnDispatchClaimFences.delete(postSpawnKey);
4746
+ }
4747
+ },
4748
+ settleRejection: (compensated) => {
4749
+ if (rejectedClaimCompensationSettled)
4750
+ return;
4751
+ rejectedClaimCompensationSettled = true;
4752
+ resolveRejectedClaimCompensation(compensated);
4753
+ },
4754
+ };
4755
+ this.#postSpawnDispatchClaimFences.set(postSpawnKey, postSpawnDispatchClaimFence);
4756
+ const settlePostSpawnIssueObservation = postSpawnIssueObservation.settle;
4757
+ const settlePostSpawnDispatchClaim = postSpawnDispatchClaimFence.settle;
4758
+ let rejectDispatchClaim;
4318
4759
  try {
4319
4760
  if (!dryRun) {
4320
4761
  const issue = await this.#readIssue(dispatchDecision.issue.path);
@@ -4339,47 +4780,92 @@ export class FactoryLoop {
4339
4780
  agents.push({ name: spawned.name, role: spec.role });
4340
4781
  }
4341
4782
  if (!dryRun) {
4783
+ const claimStartedAtMs = this.#clock.now();
4342
4784
  record.dispatchClaim = {
4343
4785
  state: 'pending',
4344
- updatedAtMs: this.#clock.now(),
4786
+ claimStartedAtMs,
4787
+ updatedAtMs: claimStartedAtMs,
4345
4788
  };
4346
4789
  this.#dispatchClaimStatuses.set(dispatchLifecycleKey(record.issue), record.dispatchClaim);
4347
4790
  }
4348
4791
  await this.#writeInFlightRegistry();
4349
4792
  const comment = dispatchComment(dispatchDecision, agents);
4350
4793
  let implementingStateId;
4794
+ const rejectDispatchClaimIfCancelled = async () => {
4795
+ if (postSpawnDispatchClaimFence.accepted !== false)
4796
+ return;
4797
+ if (rejectDispatchClaim)
4798
+ await rejectDispatchClaim();
4799
+ throw new PostSpawnDispatchWaitRejectedError(dispatchDecision.issue.key);
4800
+ };
4351
4801
  if (!dryRun) {
4352
- const issue = await this.#readIssue(dispatchDecision.issue.path);
4353
- if (!issue || !this.#isIssueReady(issue)) {
4354
- // The agents spawned a few lines above can reach terminal before we
4355
- // get here. Their completion writeback parks the issue — Linear
4356
- // `humanReview`, or the GitHub human-review label and stamps
4357
- // *this same record's* lifecycle on the way past. Re-reading the
4358
- // issue then shows "not ready", but the writer was us.
4359
- //
4360
- // Treating that as a foreign change is not a cosmetic misreport: the
4361
- // catch below classifies LiveDispatchStateChangedError as terminal
4362
- // and calls #releaseAndTerminateAgents, so a dispatch whose agents
4363
- // finished quickly tore down its own completed work and reported
4364
- // RETRYABLE to its supervisor (factory#319).
4802
+ let issue = await this.#readIssue(dispatchDecision.issue.path);
4803
+ if (postSpawnDispatchClaimFence.completionAtWriteBoundary) {
4804
+ // Completion reached its terminal provider-write boundary before
4805
+ // dispatch entered the claim boundary. Always re-read after its
4806
+ // receipt, even when the first snapshot was ready: the terminal
4807
+ // write may have started immediately after that snapshot.
4808
+ const issueWriteback = this.#issueWritebackInFlight.get(postSpawnKey);
4809
+ if (issueWriteback)
4810
+ await issueWriteback;
4811
+ issue = await this.#readIssue(dispatchDecision.issue.path);
4812
+ }
4813
+ else if ((!issue || !this.#isIssueReady(issue)) && record.issueWritebackConfirmedAtMs === undefined) {
4814
+ // A very fast agent can finish while its own dispatch is still
4815
+ // performing this post-spawn read. Do not guess authorship from the
4816
+ // first visible state: an idempotent provider mutation cannot tell us
4817
+ // whether Factory or another actor won the race. Instead, wait for
4818
+ // this record's concurrent completion writeback to settle, then use
4819
+ // only its provider-confirmed outcome (factory#319).
4365
4820
  //
4366
- // Only THIS dispatch's own confirmed issue writeback may excuse the
4367
- // change. Lifecycle phases are not enough: `publishing` is entered
4368
- // before the PR is published and `parking` before anything touches
4369
- // the issue, so a phase records local progress, not authorship — and
4370
- // a foreign park landing during those awaits would be misread as
4371
- // ours (codex review on #321). Losing the row to another owner is a
4372
- // different condition and is still caught where it always was, by
4373
- // #saveDispatchLifecycle returning false.
4821
+ // The completion promise settles as soon as terminal issue writeback
4822
+ // succeeds or fails, before Slack/release cleanup. If there is no
4823
+ // completion in flight, this is an ordinary foreign state change and
4824
+ // the existing abort remains immediate.
4825
+ const issueWriteback = this.#issueWritebackInFlight.get(postSpawnKey);
4826
+ if (issueWriteback) {
4827
+ await issueWriteback;
4828
+ issue = await this.#readIssue(dispatchDecision.issue.path);
4829
+ }
4830
+ }
4831
+ if (!issue || !this.#isIssueReady(issue)) {
4832
+ // Only a terminal writeback applied and confirmed by the completion
4833
+ // already in flight for this exact record may excuse the change. The
4834
+ // marker is stamped after provider acknowledgement/readback, never at
4835
+ // the earlier locally-visible write boundary. A confirmed no-op does
4836
+ // not establish authorship: another actor may have parked the issue
4837
+ // before our adapter's first provider read.
4374
4838
  if (record.issueWritebackConfirmedAtMs === undefined) {
4375
4839
  throw new LiveDispatchStateChangedError(dispatchDecision.issue.key);
4376
4840
  }
4377
4841
  // The claim is moot and would be wrong to write: it would drag an
4378
4842
  // issue our own lifecycle has already parked back to `implementing`.
4379
4843
  }
4844
+ else if (this.#stopping) {
4845
+ // Shutdown snapshots the post-spawn fence maps once and settles only
4846
+ // what it finds there. A dispatch that armed its fence after that
4847
+ // snapshot — it can still be inside the preview render, the
4848
+ // `dispatching` save or the question-watch setup when `stop()`
4849
+ // begins — would otherwise enter the claim boundary below against a
4850
+ // fleet already being disposed and leases already relinquished,
4851
+ // leaving a claimed issue with no lifecycle to recover it. Reject
4852
+ // here instead. No claim has started, so there is nothing to
4853
+ // compensate, and this stays synchronous so the completion-race
4854
+ // invariant documented below is untouched (#346 review, cubic).
4855
+ this.#increment('postSpawnDispatchClaimsRefusedDuringStop');
4856
+ postSpawnDispatchClaimFence.settle(false);
4857
+ throw new PostSpawnDispatchWaitRejectedError(dispatchDecision.issue.key);
4858
+ }
4380
4859
  else {
4381
- implementingStateId = await this.#applyDispatchClaim(record, issue, comment);
4382
- record.issueWritebackConfirmedAtMs = this.#clock.now();
4860
+ // This assignment is synchronous with the preceding completion flag
4861
+ // check. Completion either arrived first (the branch above) or will
4862
+ // now observe claimStarted and wait; no await-sized gap remains.
4863
+ postSpawnDispatchClaimFence.claimStarted = true;
4864
+ const appliedClaim = await this.#applyDispatchClaim(record, issue, comment, postSpawnDispatchClaimFence);
4865
+ implementingStateId = appliedClaim.implementingStateId;
4866
+ rejectDispatchClaim = appliedClaim.reject;
4867
+ await rejectDispatchClaimIfCancelled();
4868
+ record.issueWritebackConfirmedAtMs ??= this.#clock.now();
4383
4869
  this.#emit('writeback-verified', { issue: dispatchDecision.issue, path: issue.path });
4384
4870
  }
4385
4871
  }
@@ -4394,19 +4880,110 @@ export class FactoryLoop {
4394
4880
  dryRun,
4395
4881
  };
4396
4882
  record.result = result;
4397
- if (!await this.#saveDispatchLifecycle(record, 'running'))
4883
+ const runningSaved = await this.#saveDispatchLifecycle(record, 'running');
4884
+ await rejectDispatchClaimIfCancelled();
4885
+ if (!runningSaved) {
4886
+ settlePostSpawnDispatchClaim(true);
4887
+ settlePostSpawnIssueObservation(true);
4398
4888
  return result;
4399
- this.#increment('dispatched');
4400
- this.#emit('dispatched', { issue: dispatchDecision.issue, result });
4889
+ }
4401
4890
  if (!dryRun && this.#config.hooks?.onTicketDispatch) {
4402
4891
  await this.#notifyTicketDispatch(dispatchDecision, liveIssue, record, result);
4892
+ await rejectDispatchClaimIfCancelled();
4403
4893
  }
4404
4894
  if (!dryRun) {
4405
4895
  await this.#ensureSlackDispatchThread(record, result, liveIssue);
4896
+ await rejectDispatchClaimIfCancelled();
4406
4897
  }
4898
+ // Accept without an intervening await only after every durable
4899
+ // publication and post-dispatch side effect has completed. Stop or held
4900
+ // abandonment can therefore reject any still-publishing dispatch.
4901
+ settlePostSpawnDispatchClaim(true);
4902
+ settlePostSpawnIssueObservation(true);
4903
+ this.#increment('dispatched');
4904
+ this.#emit('dispatched', { issue: dispatchDecision.issue, result });
4407
4905
  return result;
4408
4906
  }
4409
- catch (error) {
4907
+ catch (caughtError) {
4908
+ // Stop/deadline may reject the fence while any awaited provider or
4909
+ // lifecycle operation is still in flight. If that operation then fails,
4910
+ // its raw error must not fall into ordinary dispatch cleanup: shutdown
4911
+ // still owns the already-spawned placements and needs them retained
4912
+ // until its release snapshot. If the claim receipt is already available,
4913
+ // conditionally reject it before releasing those placements. Without a
4914
+ // receipt the provider outcome is ambiguous, so preserve the original
4915
+ // failure as the compensation detail after the claim boundary began.
4916
+ let error = caughtError;
4917
+ if (postSpawnDispatchClaimFence.accepted === false
4918
+ && !(caughtError instanceof PostSpawnDispatchWaitRejectedError)) {
4919
+ if (rejectDispatchClaim) {
4920
+ try {
4921
+ await this.#rejectDispatchClaimAfterOperationError(dispatchDecision.issue.key, caughtError, rejectDispatchClaim);
4922
+ }
4923
+ catch (rejectionError) {
4924
+ error = rejectionError;
4925
+ }
4926
+ }
4927
+ else {
4928
+ error = new PostSpawnDispatchWaitRejectedError(dispatchDecision.issue.key, postSpawnDispatchClaimFence.claimStarted ? caughtError : undefined);
4929
+ }
4930
+ }
4931
+ settlePostSpawnDispatchClaim(false);
4932
+ if (error instanceof PostSpawnDispatchWaitRejectedError) {
4933
+ settlePostSpawnIssueObservation(false);
4934
+ const compensated = error.compensationError === undefined;
4935
+ // The original provider call has conclusively returned. Stop the
4936
+ // timeout handoff from writing a provisional pending fence, but keep a
4937
+ // separate lease fence until the definitive result is durable. Without
4938
+ // that second set, shutdown can relinquish this epoch in the await
4939
+ // below and make the late save fail after the provider compensated.
4940
+ this.#dispatchClaimSettlementsInFlight.add(postSpawnKey);
4941
+ this.#pendingDispatchClaims.delete(postSpawnKey);
4942
+ let settlementPersisted = false;
4943
+ try {
4944
+ settlementPersisted = compensated
4945
+ ? await this.#clearDispatchCancellationBlock(record)
4946
+ : await this.#retainUncompensatedDispatchClaim(record, error.compensationError);
4947
+ if (!settlementPersisted) {
4948
+ throw new Error(`Unable to persist settled dispatch-claim compensation for ${decision.issue.key}`);
4949
+ }
4950
+ postSpawnDispatchClaimFence.settleRejection(compensated);
4951
+ if (error.compensationError !== undefined) {
4952
+ this.#increment('postSpawnDispatchClaimCompensationFailures');
4953
+ this.#logger.error?.('[factory] rejected dispatch claim could not be safely compensated', {
4954
+ issue: decision.issue.key,
4955
+ error: describeError(error.compensationError).errorMessage,
4956
+ });
4957
+ }
4958
+ // External stop/abandonment owns agent and durable-lifecycle cleanup.
4959
+ // Remove only the process-local slot here; running the ordinary
4960
+ // dispatch-failure path would race or duplicate that cleanup. During
4961
+ // stop, however, retain local placements until the shutdown release
4962
+ // sweep has captured them. If it already has, removal is safe now.
4963
+ if (!compensated) {
4964
+ // The provider still exposes a claim this lifecycle authored. Keep
4965
+ // the process-local slot and durable row recoverable; releasing its
4966
+ // agents would leave the claimed issue with no successor lifecycle.
4967
+ }
4968
+ else if (this.#stopping) {
4969
+ this.#postSpawnDispatchesRejectedDuringStop.add(postSpawnKey);
4970
+ if (this.#shutdownReleaseCapturedDispatches.has(postSpawnKey)) {
4971
+ batch.abandon(decision.issue);
4972
+ this.#postSpawnDispatchesRejectedDuringStop.delete(postSpawnKey);
4973
+ }
4974
+ }
4975
+ else {
4976
+ batch.abandon(decision.issue);
4977
+ }
4978
+ }
4979
+ finally {
4980
+ if (!settlementPersisted)
4981
+ this.#pendingDispatchClaims.add(postSpawnKey);
4982
+ this.#dispatchClaimSettlementsInFlight.delete(postSpawnKey);
4983
+ }
4984
+ throw error;
4985
+ }
4986
+ settlePostSpawnIssueObservation(!(error instanceof LiveDispatchStateChangedError));
4410
4987
  // A spawn can fail after the broker accepted it but before its ack
4411
4988
  // reached Factory. Include every planned worktree agent, not only the
4412
4989
  // acknowledged spawns, so cleanup never races a name-only survivor.
@@ -4579,6 +5156,8 @@ export class FactoryLoop {
4579
5156
  this.#readinessReconcileLastSweepDeferred = undefined;
4580
5157
  this.#readinessReconcileLastSweep = {
4581
5158
  candidates: report.pulled.length,
5159
+ treeReads: report.treeReads ?? 0,
5160
+ emptyTreeReads: report.emptyTreeReads ?? 0,
4582
5161
  dispatched: report.dispatched.length,
4583
5162
  skipped: report.skipped.length,
4584
5163
  skipReasons: factorySweepSkipReasonCounts(report.skipped),
@@ -4673,6 +5252,13 @@ export class FactoryLoop {
4673
5252
  ...(this.#readinessReconcileLastSweep
4674
5253
  ? {
4675
5254
  candidates: this.#readinessReconcileLastSweep.candidates,
5255
+ // Unconditional for the same reason `dispatchFailures` is: these
5256
+ // are only meaningful as a pair, and only a published zero lets a
5257
+ // reader see that `emptyTreeReads < treeReads` — i.e. that the
5258
+ // mount served real content and a zero `candidates` beside it means
5259
+ // an empty workspace, not a silent mount (#351 follow-up).
5260
+ treeReads: this.#readinessReconcileLastSweep.treeReads,
5261
+ emptyTreeReads: this.#readinessReconcileLastSweep.emptyTreeReads,
4676
5262
  dispatched: this.#readinessReconcileLastSweep.dispatched,
4677
5263
  skipped: this.#readinessReconcileLastSweep.skipped,
4678
5264
  ...(Object.keys(this.#readinessReconcileLastSweep.skipReasons).length > 0
@@ -6259,9 +6845,11 @@ export class FactoryLoop {
6259
6845
  const comment = dispatchComment(record.decision, agents);
6260
6846
  let implementingStateId;
6261
6847
  if (!record.dryRun) {
6848
+ const claimStartedAtMs = this.#clock.now();
6262
6849
  record.dispatchClaim = {
6263
6850
  state: 'pending',
6264
- updatedAtMs: this.#clock.now(),
6851
+ claimStartedAtMs,
6852
+ updatedAtMs: claimStartedAtMs,
6265
6853
  };
6266
6854
  this.#dispatchClaimStatuses.set(dispatchLifecycleKey(record.issue), record.dispatchClaim);
6267
6855
  }
@@ -6271,7 +6859,7 @@ export class FactoryLoop {
6271
6859
  if (!issue)
6272
6860
  throw new Error(`Unable to recover durable dispatch ${record.issue.key}: issue is no longer readable`);
6273
6861
  await this.#ensureGithubAgentQuestionWatch(record, issue);
6274
- implementingStateId = await this.#applyDispatchClaim(record, issue, comment);
6862
+ implementingStateId = (await this.#applyDispatchClaim(record, issue, comment)).implementingStateId;
6275
6863
  }
6276
6864
  const recoveredPreviews = uniquePreviewReferences([
6277
6865
  ...dispatchSpecs(record.decision).map((spec) => spec.preview),
@@ -6580,33 +7168,161 @@ export class FactoryLoop {
6580
7168
  }
6581
7169
  await this.#linear.postComment(issue, body);
6582
7170
  }
6583
- async #applyDispatchClaim(record, issue, comment) {
7171
+ async #applyDispatchClaim(record, issue, comment, claimFence) {
7172
+ const claimWasRejected = () => claimFence?.accepted === false;
7173
+ // stop() can reject this fence after the post-spawn readiness read but
7174
+ // before this method starts. Never begin a provider claim in that state;
7175
+ // shutdown intentionally does not drain dispatches that never crossed the
7176
+ // claim boundary.
7177
+ if (claimWasRejected()) {
7178
+ throw new PostSpawnDispatchWaitRejectedError(record.issue.key);
7179
+ }
6584
7180
  let implementingStateId;
7181
+ let rejectClaim;
6585
7182
  if (isGithubIssue(issue)) {
7183
+ const statusClaim = {
7184
+ value: { result: undefined },
7185
+ };
6586
7186
  await this.#retryDispatchWriteback(record, issue, 'GitHub label factory:in-progress', async () => {
6587
- await this.#githubWriteback.setStatus(issue, 'in-progress');
7187
+ statusClaim.value = this.#githubWriteback.claimStatus
7188
+ ? await this.#githubWriteback.claimStatus(issue, 'in-progress')
7189
+ : { result: await this.#githubWriteback.setStatus(issue, 'in-progress') };
6588
7190
  });
7191
+ rejectClaim = async () => await this.#rejectGithubDispatchClaim(record, issue, statusClaim.value);
7192
+ if (claimWasRejected()) {
7193
+ await rejectClaim();
7194
+ }
6589
7195
  const commentApplied = this.#githubWriteback.hasCommentMarker
6590
7196
  ? async () => this.#githubWriteback.hasCommentMarker(issue, comment)
6591
7197
  : undefined;
6592
- await this.#retryDispatchWriteback(record, issue, 'GitHub dispatch comment', async () => this.#githubWriteback.postComment(issue, comment), commentApplied);
7198
+ try {
7199
+ await this.#retryDispatchWriteback(record, issue, 'GitHub dispatch comment', async () => {
7200
+ if (claimWasRejected())
7201
+ await rejectClaim();
7202
+ await this.#githubWriteback.postComment(issue, comment);
7203
+ }, commentApplied);
7204
+ }
7205
+ catch (error) {
7206
+ if (claimWasRejected() && !(error instanceof PostSpawnDispatchWaitRejectedError)) {
7207
+ await this.#rejectDispatchClaimAfterOperationError(record.issue.key, error, rejectClaim);
7208
+ }
7209
+ throw error;
7210
+ }
7211
+ if (claimWasRejected())
7212
+ await rejectClaim();
6593
7213
  }
6594
7214
  else {
6595
7215
  implementingStateId = this.#states.idFor(issue.team, 'agentImplementing');
7216
+ const readyStateId = issue.stateId;
7217
+ let claimToken;
6596
7218
  await this.#retryDispatchWriteback(record, issue, `Linear state ${implementingStateId}`, async () => {
6597
- await this.#linear.setState(issue, implementingStateId);
6598
- });
6599
- await this.#retryDispatchWriteback(record, issue, 'Linear dispatch comment', async () => {
6600
- await this.#linear.postComment(issue, comment);
7219
+ claimToken = (await this.#linear.setState(issue, implementingStateId))?.claimToken;
6601
7220
  });
7221
+ rejectClaim = async () => await this.#rejectLinearDispatchClaim(record, issue, implementingStateId, readyStateId, claimToken);
7222
+ if (claimWasRejected()) {
7223
+ await rejectClaim();
7224
+ }
7225
+ try {
7226
+ await this.#retryDispatchWriteback(record, issue, 'Linear dispatch comment', async () => {
7227
+ if (claimWasRejected())
7228
+ await rejectClaim();
7229
+ await this.#linear.postComment(issue, comment);
7230
+ });
7231
+ }
7232
+ catch (error) {
7233
+ if (claimWasRejected() && !(error instanceof PostSpawnDispatchWaitRejectedError)) {
7234
+ await this.#rejectDispatchClaimAfterOperationError(record.issue.key, error, rejectClaim);
7235
+ }
7236
+ throw error;
7237
+ }
7238
+ if (claimWasRejected())
7239
+ await rejectClaim();
6602
7240
  }
6603
7241
  record.dispatchClaim = {
6604
7242
  state: 'verified',
7243
+ // Cancellation can still arrive after this returns — while the running
7244
+ // lifecycle save, the dispatch hook or the Slack publication is pending.
7245
+ // The rejection fence rebuilds the claim from this record, so a verified
7246
+ // state that drops the claim-start stamp leaves `cancellationBlocked`
7247
+ // with no watermark for `getIssueStatus(..., { requireFresh: true })`.
7248
+ // A private repository has no unauthenticated fallback to fall back on,
7249
+ // so that lifecycle, its agents and its batch slot would then be retained
7250
+ // for good, even once a human superseded the claim (#346 review, codex).
7251
+ ...(record.dispatchClaim?.claimStartedAtMs === undefined
7252
+ ? {}
7253
+ : { claimStartedAtMs: record.dispatchClaim.claimStartedAtMs }),
6605
7254
  updatedAtMs: this.#clock.now(),
6606
7255
  };
6607
7256
  this.#dispatchClaimStatuses.set(dispatchLifecycleKey(record.issue), record.dispatchClaim);
6608
7257
  await this.#writeDispatchClaimRegistry(record.issue);
6609
- return implementingStateId;
7258
+ if (claimWasRejected())
7259
+ await rejectClaim?.();
7260
+ return {
7261
+ ...(implementingStateId ? { implementingStateId } : {}),
7262
+ reject: rejectClaim,
7263
+ };
7264
+ }
7265
+ async #rejectGithubDispatchClaim(record, issue, claim) {
7266
+ let compensationError;
7267
+ if (claim.result !== 'already-matched') {
7268
+ try {
7269
+ const claimToken = claim.claimToken;
7270
+ if (!claimToken) {
7271
+ throw new Error('GitHub rejected dispatch claim has no immutable ownership token');
7272
+ }
7273
+ const rollbackStatusClaim = this.#githubWriteback.rollbackStatusClaim?.bind(this.#githubWriteback);
7274
+ if (!rollbackStatusClaim) {
7275
+ throw new Error('GitHub writeback cannot conditionally roll back a rejected dispatch claim');
7276
+ }
7277
+ let outcome = 'unproven';
7278
+ await this.#retryDispatchWriteback(record, issue, 'GitHub rejected dispatch claim rollback', async () => {
7279
+ outcome = await rollbackStatusClaim(issue, 'in-progress', claimToken);
7280
+ if (outcome === 'unproven') {
7281
+ throw new Error('GitHub rejected dispatch claim rollback could not prove ownership');
7282
+ }
7283
+ });
7284
+ }
7285
+ catch (error) {
7286
+ compensationError = error;
7287
+ }
7288
+ }
7289
+ throw new PostSpawnDispatchWaitRejectedError(record.issue.key, compensationError);
7290
+ }
7291
+ async #rejectDispatchClaimAfterOperationError(issueKey, operationError, rejectClaim) {
7292
+ try {
7293
+ await rejectClaim();
7294
+ }
7295
+ catch (rejectionError) {
7296
+ if (rejectionError instanceof PostSpawnDispatchWaitRejectedError) {
7297
+ if (rejectionError.compensationError === undefined)
7298
+ throw rejectionError;
7299
+ throw new PostSpawnDispatchWaitRejectedError(issueKey, new AggregateError([operationError, rejectionError.compensationError], 'Cancelled dispatch operation failed and its claim could not be safely compensated'));
7300
+ }
7301
+ throw new PostSpawnDispatchWaitRejectedError(issueKey, new AggregateError([operationError, rejectionError], 'Cancelled dispatch operation failed and its claim rejection threw unexpectedly'));
7302
+ }
7303
+ throw new PostSpawnDispatchWaitRejectedError(issueKey);
7304
+ }
7305
+ async #rejectLinearDispatchClaim(record, issue, implementingStateId, readyStateId, claimToken) {
7306
+ let compensationError;
7307
+ try {
7308
+ const compareAndSetState = this.#linear.compareAndSetState?.bind(this.#linear);
7309
+ if (!compareAndSetState) {
7310
+ throw new Error('Linear writeback cannot conditionally roll back a rejected dispatch claim');
7311
+ }
7312
+ if (!claimToken) {
7313
+ throw new Error('Linear rejected dispatch claim has no immutable ownership token');
7314
+ }
7315
+ await this.#retryDispatchWriteback(record, issue, `Linear rejected dispatch claim rollback to ${readyStateId}`, async () => {
7316
+ const outcome = await compareAndSetState(issue, implementingStateId, claimToken, readyStateId);
7317
+ if (outcome === 'unproven') {
7318
+ throw new Error('Linear rejected dispatch claim rollback could not prove an atomic provider transition');
7319
+ }
7320
+ });
7321
+ }
7322
+ catch (error) {
7323
+ compensationError = error;
7324
+ }
7325
+ throw new PostSpawnDispatchWaitRejectedError(record.issue.key, compensationError);
6610
7326
  }
6611
7327
  async #retryDispatchWriteback(record, issue, write, apply, isApplied) {
6612
7328
  let lastError;
@@ -6621,11 +7337,23 @@ export class FactoryLoop {
6621
7337
  return;
6622
7338
  }
6623
7339
  catch (error) {
7340
+ // Claim rejection is terminal for this dispatch. In particular, do not
7341
+ // turn a cancellation discovered between comment attempts into another
7342
+ // provider retry that can publish after shutdown began.
7343
+ if (error instanceof PostSpawnDispatchWaitRejectedError) {
7344
+ if (lastError !== undefined && error.compensationError !== undefined) {
7345
+ throw new PostSpawnDispatchWaitRejectedError(record.issue.key, new AggregateError([lastError, error.compensationError], 'Dispatch writeback failed before cancellation and its claim could not be safely compensated'));
7346
+ }
7347
+ throw error;
7348
+ }
6624
7349
  lastError = error;
6625
7350
  const deadLettered = attempt === DISPATCH_WRITEBACK_MAX_ATTEMPTS;
6626
7351
  this.#increment('dispatchWritebackFailures');
6627
7352
  record.dispatchClaim = {
6628
7353
  state: 'degraded',
7354
+ ...(record.dispatchClaim?.claimStartedAtMs === undefined
7355
+ ? {}
7356
+ : { claimStartedAtMs: record.dispatchClaim.claimStartedAtMs }),
6629
7357
  write,
6630
7358
  attempts: attempt,
6631
7359
  maxAttempts: DISPATCH_WRITEBACK_MAX_ATTEMPTS,
@@ -6728,7 +7456,7 @@ export class FactoryLoop {
6728
7456
  const candidates = [];
6729
7457
  let scanned = 0;
6730
7458
  let lastProgressAtMs = startedAtMs;
6731
- for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'GitHub mirror candidate loading', { cache: true })) {
7459
+ for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'GitHub mirror candidate loading', { cache: true, enumeration: true })) {
6732
7460
  await this.#refreshLiveHeartbeatIfDue();
6733
7461
  if (!isLinearIssueMirrorCandidatePath(path)) {
6734
7462
  continue;
@@ -6778,7 +7506,7 @@ export class FactoryLoop {
6778
7506
  else {
6779
7507
  pathBatches = [];
6780
7508
  for (const root of roots) {
6781
- pathBatches.push(await this.#listRelayfileTree(root, 'GitHub issue ingestion', { cache: true }));
7509
+ pathBatches.push(await this.#listRelayfileTree(root, 'GitHub issue ingestion', { cache: true, enumeration: true }));
6782
7510
  }
6783
7511
  this.#increment('githubIssueIndexFallbacks');
6784
7512
  }
@@ -7256,6 +7984,10 @@ export class FactoryLoop {
7256
7984
  // The pass-scoped flag also lets every later blocked issue reuse the
7257
7985
  // resulting dependency index instead of rescanning the full tree.
7258
7986
  this.#dependencyLinearTreeLoaded = true;
7987
+ // Cached like the enumeration walks, but deliberately NOT `enumeration`:
7988
+ // this is a lookup for named blocker identities, not the sweep asking
7989
+ // what work exists, and mixing a lookup's result into the ratio is the
7990
+ // masking the pair's scoping exists to prevent (#363 review).
7259
7991
  for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'dependency blocker discovery', { cache: true })) {
7260
7992
  if (isIssueFilePath(path))
7261
7993
  await this.#readIssue(path);
@@ -7706,14 +8438,14 @@ export class FactoryLoop {
7706
8438
  }
7707
8439
  const pathsByKey = new Map();
7708
8440
  const canonicalPathsByKey = new Map();
7709
- for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'Linear ready issue canonical discovery', { cache: true })) {
8441
+ for (const path of await this.#listRelayfileTree(ISSUE_ROOT, 'Linear ready issue canonical discovery', { cache: true, enumeration: true })) {
7710
8442
  if (isIssueFilePath(path)) {
7711
8443
  const key = keyFromPath(path);
7712
8444
  canonicalPathsByKey.set(key, path);
7713
8445
  pathsByKey.set(key, path);
7714
8446
  }
7715
8447
  }
7716
- for (const path of await this.#listRelayfileTree(linearByStatePath('ready-for-agent'), 'Linear ready issue alias discovery', { cache: true })) {
8448
+ for (const path of await this.#listRelayfileTree(linearByStatePath('ready-for-agent'), 'Linear ready issue alias discovery', { cache: true, enumeration: true })) {
7717
8449
  if (isIssueAliasFilePath(path)) {
7718
8450
  const canonicalPath = canonicalPathsByKey.get(keyFromPath(path));
7719
8451
  if (canonicalPath) {
@@ -7767,10 +8499,20 @@ export class FactoryLoop {
7767
8499
  }
7768
8500
  async #releaseInFlightAgents(reason, opts = {}) {
7769
8501
  const agents = new Map();
7770
- for (const record of (await this.#batch()).inFlight) {
8502
+ const blockedDispatches = new Set();
8503
+ const batch = await this.#batch();
8504
+ const records = [...batch.inFlight];
8505
+ for (const record of records) {
8506
+ const key = dispatchLifecycleKey(record.issue);
7771
8507
  if (record.dryRun) {
8508
+ this.#shutdownReleaseCapturedDispatches.add(key);
7772
8509
  continue;
7773
8510
  }
8511
+ if (this.#uncompensatedDispatchClaims.has(key) || record.dispatchClaim?.cancellationBlocked === true) {
8512
+ blockedDispatches.add(key);
8513
+ continue;
8514
+ }
8515
+ this.#shutdownReleaseCapturedDispatches.add(key);
7774
8516
  if (opts.preserveDurable && [...record.agents.values()].some((tracked) => tracked.result?.locality === 'remote')) {
7775
8517
  continue;
7776
8518
  }
@@ -7779,7 +8521,15 @@ export class FactoryLoop {
7779
8521
  }
7780
8522
  }
7781
8523
  await this.#releaseAndTerminateAgents([...agents], reason, 'stop');
7782
- await this.#writeInFlightRegistry(undefined, undefined, true);
8524
+ for (const record of records) {
8525
+ const key = dispatchLifecycleKey(record.issue);
8526
+ if (blockedDispatches.has(key))
8527
+ continue;
8528
+ if (!this.#postSpawnDispatchesRejectedDuringStop.delete(key))
8529
+ continue;
8530
+ batch.abandon(record.issue);
8531
+ }
8532
+ await this.#writeInFlightRegistry(undefined, undefined, true, blockedDispatches);
7783
8533
  }
7784
8534
  async #releaseAndTerminateAgents(agents, reason, context) {
7785
8535
  const failed = [];
@@ -7838,17 +8588,40 @@ export class FactoryLoop {
7838
8588
  await this.#reportAgent(record, tracked, 'agent.released', { releaseReason: reason });
7839
8589
  }
7840
8590
  catch (error) {
7841
- failed.push(agentName);
7842
- this.#logger.warn?.(`[factory] failed to release ${agentName} during ${context}`, error);
7843
- if (record) {
7844
- const lifecycle = await this.#state
7845
- .getDispatchLifecycle(this.#workspaceId, dispatchLifecycleKey(record.issue))
7846
- .catch(() => undefined);
7847
- if (lifecycle) {
7848
- await this.#reportLifecycle(lifecycle, 'factory.failure', {
7849
- level: 'error',
7850
- errorCode: 'release_failed',
7851
- });
8591
+ if (isAgentAlreadyGoneOnRelease(error)) {
8592
+ // Do the same bookkeeping the success path does — the agent IS
8593
+ // gone. Skipping this would leave `batch.recordRelease` unset and
8594
+ // the next retry attempt would think the invocation is still
8595
+ // dispatchable.
8596
+ this.#increment('releaseAgentAlreadyGone');
8597
+ this.#logger.info?.(`[factory] release skipped: agent already gone during ${context}`, { agentName, reason });
8598
+ if (record && batch && context !== 'stop') {
8599
+ const releasedInvocationId = batch.recordRelease(record, agentName, this.#clock.now());
8600
+ if (releasedInvocationId) {
8601
+ this.#logger.debug?.('[factory] released agent invocation is no longer dispatchable', {
8602
+ issue: record.issue.key,
8603
+ agentName,
8604
+ reason,
8605
+ invocationId: releasedInvocationId,
8606
+ });
8607
+ }
8608
+ }
8609
+ if (record)
8610
+ await this.#reportAgent(record, tracked, 'agent.released', { releaseReason: reason });
8611
+ }
8612
+ else {
8613
+ failed.push(agentName);
8614
+ this.#logger.warn?.(`[factory] failed to release ${agentName} during ${context}`, error);
8615
+ if (record) {
8616
+ const lifecycle = await this.#state
8617
+ .getDispatchLifecycle(this.#workspaceId, dispatchLifecycleKey(record.issue))
8618
+ .catch(() => undefined);
8619
+ if (lifecycle) {
8620
+ await this.#reportLifecycle(lifecycle, 'factory.failure', {
8621
+ level: 'error',
8622
+ errorCode: 'release_failed',
8623
+ });
8624
+ }
7852
8625
  }
7853
8626
  }
7854
8627
  }
@@ -7998,7 +8771,7 @@ export class FactoryLoop {
7998
8771
  this.#increment('agentWorktreesCleaned');
7999
8772
  }
8000
8773
  }
8001
- async #writeInFlightRegistry(path = this.#config.loop.registryPath, heartbeatPath = this.#config.loop.heartbeatPath, empty = false) {
8774
+ async #writeInFlightRegistry(path = this.#config.loop.registryPath, heartbeatPath = this.#config.loop.heartbeatPath, empty = false, retainIssueKeys) {
8002
8775
  const updatedAtMs = this.#clock.now();
8003
8776
  const agents = [];
8004
8777
  const seenAgents = new Set();
@@ -8036,10 +8809,12 @@ export class FactoryLoop {
8036
8809
  } : {}),
8037
8810
  });
8038
8811
  };
8039
- if (!empty) {
8812
+ if (!empty || retainIssueKeys?.size) {
8040
8813
  for (const record of (await this.#batch()).inFlight) {
8041
8814
  if (record.dryRun)
8042
8815
  continue;
8816
+ if (empty && !retainIssueKeys?.has(dispatchLifecycleKey(record.issue)))
8817
+ continue;
8043
8818
  if (record.dispatchClaim) {
8044
8819
  this.#dispatchClaimStatuses.set(dispatchLifecycleKey(record.issue), record.dispatchClaim);
8045
8820
  }
@@ -9254,10 +10029,59 @@ export class FactoryLoop {
9254
10029
  // the release-driven exit event so it cannot re-trigger a resume before the
9255
10030
  // record leaves the batch.
9256
10031
  async #abandonStuckDispatch(record, reason) {
9257
- const key = issueKey(record.issue);
10032
+ const key = dispatchLifecycleKey(record.issue);
10033
+ // Fence the periodic held sweep before any await below. A blocked claim is
10034
+ // retried by the single keyed abandonment timer; letting the held sweep
10035
+ // enter concurrently can release the same placements twice when provider
10036
+ // supersession becomes visible.
10037
+ this.#abandonedDispatchReasons.set(key, reason);
10038
+ try {
10039
+ await this.#abandonStuckDispatchFenced(record, reason, key);
10040
+ }
10041
+ catch (error) {
10042
+ // The early fence above suppresses the periodic held sweep. Any
10043
+ // exceptional exit must therefore install the keyed retry before
10044
+ // propagating, or one transient persistence/read failure becomes a
10045
+ // permanent absorbing state.
10046
+ this.#increment('abandonedDispatchReleaseRetries');
10047
+ this.#logger.warn?.('[factory] abandoned dispatch cleanup failed before retry was armed', {
10048
+ issue: record.issue.key,
10049
+ error: describeError(error).errorMessage,
10050
+ });
10051
+ this.#scheduleAbandonedDispatchRetry(record, reason);
10052
+ throw error;
10053
+ }
10054
+ }
10055
+ async #abandonStuckDispatchFenced(record, reason, key) {
10056
+ const claimFence = this.#postSpawnDispatchClaimFences.get(key);
10057
+ const rejectedClaimCompensation = claimFence?.claimStarted
10058
+ ? claimFence.rejectionSettled
10059
+ : undefined;
10060
+ // A dispatch can be stuck inside a later spawn after an earlier agent has
10061
+ // already started completion. Release both post-spawn waits before the
10062
+ // first abandonment await so terminal processing cannot remain an
10063
+ // absorbing promise after this lifecycle is reaped.
10064
+ this.#settlePostSpawnDispatchWaits(key, false, 'abandonment');
10065
+ if (rejectedClaimCompensation) {
10066
+ const compensated = await this.#awaitRejectedClaimCompensation(rejectedClaimCompensation);
10067
+ if (!compensated) {
10068
+ if (!this.#pendingDispatchClaims.has(key)) {
10069
+ // The provider call settled while this bounded wait expired; its
10070
+ // caller is recording the definitive compensated/unproven result.
10071
+ this.#scheduleAbandonedDispatchRetry(record, reason);
10072
+ return;
10073
+ }
10074
+ await this.#retainUncompensatedDispatchClaim(record, new Error('Rejected dispatch claim compensation did not complete before abandonment'), { preferExistingError: true, pending: true });
10075
+ this.#scheduleAbandonedDispatchRetry(record, reason);
10076
+ return;
10077
+ }
10078
+ }
10079
+ if (await this.#dispatchClaimBlocksAbandonment(record)) {
10080
+ this.#scheduleAbandonedDispatchRetry(record, reason);
10081
+ return;
10082
+ }
9258
10083
  const heldPastDeadline = reason === HELD_PAST_DEADLINE_RELEASE_REASON;
9259
10084
  const agentReleaseReason = heldPastDeadline ? HELD_PAST_DEADLINE_RELEASE_REASON : 'issue-abandoned';
9260
- this.#abandonedDispatchReasons.set(key, reason);
9261
10085
  if (!await this.#saveDispatchLifecycle(record, 'abandoning', undefined, reason, new Set(), { cancellationReason: 'dispatch_failed' })) {
9262
10086
  this.#increment('abandonedDispatchReleaseRetries');
9263
10087
  // The generic durable retry can recover the in-memory reason in this
@@ -13285,6 +14109,22 @@ export class FactoryLoop {
13285
14109
  return;
13286
14110
  }
13287
14111
  this.#completionInFlight.add(completionKey);
14112
+ const postSpawnIssueObservation = this.#postSpawnIssueObservations.get(completionKey);
14113
+ const postSpawnDispatchClaimFence = this.#postSpawnDispatchClaimFences.get(completionKey);
14114
+ let settleIssueWriteback;
14115
+ const issueWritebackSettled = new Promise((resolve) => {
14116
+ settleIssueWriteback = resolve;
14117
+ });
14118
+ let issueWritebackDidSettle = false;
14119
+ const settleIssueWritebackOnce = () => {
14120
+ if (issueWritebackDidSettle)
14121
+ return;
14122
+ issueWritebackDidSettle = true;
14123
+ settleIssueWriteback();
14124
+ if (this.#issueWritebackInFlight.get(completionKey) === issueWritebackSettled) {
14125
+ this.#issueWritebackInFlight.delete(completionKey);
14126
+ }
14127
+ };
13288
14128
  let releaseReasonForRetry;
13289
14129
  try {
13290
14130
  if (!await this.#assertDispatchLifecycleOwner(record))
@@ -13334,13 +14174,59 @@ export class FactoryLoop {
13334
14174
  const humanReview = configuredHumanReview || (githubIssue && !githubMerged);
13335
14175
  const statusLabel = humanReview ? 'In Human Review' : 'Done';
13336
14176
  if (issue) {
14177
+ if (postSpawnDispatchClaimFence) {
14178
+ // Publish boundary arrival before any await. Dispatch performs the
14179
+ // paired synchronous claimStarted assignment after its ready read,
14180
+ // so exactly one side wins: an earlier completion is awaited and
14181
+ // re-read; a claim already entering its provider write finishes
14182
+ // before this terminal write begins.
14183
+ postSpawnDispatchClaimFence.completionAtWriteBoundary = true;
14184
+ if (postSpawnDispatchClaimFence.claimStarted || postSpawnDispatchClaimFence.accepted !== undefined) {
14185
+ let claimAccepted = postSpawnDispatchClaimFence.accepted;
14186
+ if (claimAccepted === undefined) {
14187
+ // This counter makes the otherwise internal promise boundary
14188
+ // observable in deterministic race regressions and in incident
14189
+ // telemetry. External abandonment/stop settlement must release
14190
+ // every completion counted here.
14191
+ this.#increment('postSpawnDispatchClaimFenceWaits');
14192
+ claimAccepted = await postSpawnDispatchClaimFence.settled;
14193
+ }
14194
+ if (!claimAccepted)
14195
+ return;
14196
+ }
14197
+ }
14198
+ // Register only at the provider-write boundary. Work before this point
14199
+ // (PR discovery/merge gating) has not changed the issue, so a foreign
14200
+ // park during it must still abort immediately rather than waiting on a
14201
+ // possibly long completion path.
14202
+ this.#issueWritebackInFlight.set(completionKey, issueWritebackSettled);
13337
14203
  if (githubIssue) {
13338
14204
  if (humanReview) {
13339
- await this.#githubWriteback.setStatus(issue, 'human-review');
14205
+ const statusWrite = await this.#githubWriteback.setStatus(issue, 'human-review');
14206
+ if (statusWrite === undefined)
14207
+ this.#recordMissingGithubWritebackReceipt('setStatus');
14208
+ // Only an explicit provider-proven transition establishes that
14209
+ // this dispatch owns the visible park. Legacy void adapters and
14210
+ // App acknowledgements remain deliberately untrusted.
14211
+ if (statusWrite === 'applied') {
14212
+ record.issueWritebackConfirmedAtMs ??= this.#clock.now();
14213
+ }
14214
+ // The lifecycle-state outcome is now known. Unblock the concurrent
14215
+ // post-spawn read before the separate completion comment write.
14216
+ settleIssueWritebackOnce();
13340
14217
  await this.#githubWriteback.postComment(issue, `Factory agents completed; this issue is awaiting human review. The pull request remains open.\n\nMerge policy: ${this.#config.mergePolicy}`);
13341
14218
  }
13342
14219
  else {
13343
- await this.#githubWriteback.closeIssue(issue, 'Factory observed the linked pull request merge and completed this issue.');
14220
+ const closeWrite = await this.#githubWriteback.closeIssue(issue, 'Factory observed the linked pull request merge and completed this issue.');
14221
+ if (closeWrite === undefined)
14222
+ this.#recordMissingGithubWritebackReceipt('closeIssue');
14223
+ // A provider-confirmed, actor-attributed close is the only safe
14224
+ // proof that this dispatch owns the visible terminal state. An
14225
+ // idempotent no-op, legacy void adapter, or App acknowledgement
14226
+ // may instead mean a third party won the close race.
14227
+ if (closeWrite === 'applied') {
14228
+ record.issueWritebackConfirmedAtMs ??= this.#clock.now();
14229
+ }
13344
14230
  }
13345
14231
  }
13346
14232
  else {
@@ -13348,13 +14234,28 @@ export class FactoryLoop {
13348
14234
  ? this.#states.idFor(issueTeam, 'humanReview')
13349
14235
  : this.#states.idFor(issueTeam, 'done');
13350
14236
  await this.#linear.setState(issue, targetState);
14237
+ record.issueWritebackConfirmedAtMs ??= this.#clock.now();
13351
14238
  await this.#recordCanonicalIssueState({ ...record.issue, stateId: targetState });
13352
14239
  }
13353
- record.issueWritebackConfirmedAtMs = this.#clock.now();
13354
- this.#emit('writeback-verified', { issue: record.issue, path: issue.path });
14240
+ if (record.issueWritebackConfirmedAtMs !== undefined) {
14241
+ this.#emit('writeback-verified', { issue: record.issue, path: issue.path });
14242
+ }
14243
+ // Unblock a concurrent post-spawn read as soon as the issue writeback
14244
+ // outcome is known. Completion still has dependency, Slack and release
14245
+ // work to do; none determines whether the observed issue state is safe.
14246
+ settleIssueWritebackOnce();
14247
+ // If completion raced dispatch's post-spawn read, let that read consume
14248
+ // the receipt before claiming terminal success. An unproven provider
14249
+ // transition is a foreign live-state change: dispatch owns the
14250
+ // abandonment and must release agents with that reason, not issue-done.
14251
+ if (postSpawnIssueObservation && !await postSpawnIssueObservation.settled)
14252
+ return;
13355
14253
  if (!humanReview)
13356
14254
  await this.#markDependencyTerminalAndReconcile(issue);
13357
14255
  }
14256
+ else {
14257
+ settleIssueWritebackOnce();
14258
+ }
13358
14259
  if (!await this.#saveDispatchLifecycle(record, 'writeback-applied'))
13359
14260
  return;
13360
14261
  if (issue && this.#slack && this.#config.slack && !await this.#shouldSkipSlackWriteback('completion-thread')) {
@@ -13419,6 +14320,10 @@ export class FactoryLoop {
13419
14320
  this.#scheduleDispatchLifecycleRetry(record);
13420
14321
  }
13421
14322
  finally {
14323
+ // Errors before or during provider confirmation leave the marker unset;
14324
+ // the waiting dispatch will re-read and preserve the foreign-change
14325
+ // abort. Always settle so a failed write cannot strand that dispatch.
14326
+ settleIssueWritebackOnce();
13422
14327
  this.#completionInFlight.delete(completionKey);
13423
14328
  const stateKey = issueStateKey(record.issue);
13424
14329
  this.#probePrGhBackoffUntilMs.delete(stateKey);
@@ -13435,6 +14340,35 @@ export class FactoryLoop {
13435
14340
  }
13436
14341
  }
13437
14342
  }
14343
+ #recordMissingGithubWritebackReceipt(method) {
14344
+ const implementation = this.#githubWriteback.constructor.name || 'anonymous GithubWriteback';
14345
+ this.#increment('githubWritebackReceiptMissing');
14346
+ this.#logger.warn?.('[factory] GitHub writeback returned no ownership receipt', {
14347
+ implementation,
14348
+ method,
14349
+ });
14350
+ }
14351
+ #settlePostSpawnDispatchWaits(key, accepted, source) {
14352
+ const observation = this.#postSpawnIssueObservations.get(key);
14353
+ const claimFence = this.#postSpawnDispatchClaimFences.get(key);
14354
+ if (!observation && !claimFence)
14355
+ return false;
14356
+ // Capture both before either settlement deletes its own map entry.
14357
+ if (!accepted && claimFence?.claimStarted) {
14358
+ // The rejection handler clears this provisional block only after a
14359
+ // conditional provider rollback succeeds. Until then, shutdown and held
14360
+ // abandonment must not release the lifecycle's agents.
14361
+ this.#uncompensatedDispatchClaims.add(key);
14362
+ this.#pendingDispatchClaims.add(key);
14363
+ }
14364
+ observation?.settle(accepted);
14365
+ claimFence?.settle(accepted);
14366
+ if (source === 'abandonment')
14367
+ this.#increment('postSpawnWaitsSettledByAbandonment');
14368
+ if (source === 'stop')
14369
+ this.#increment('postSpawnWaitsSettledByStop');
14370
+ return true;
14371
+ }
13438
14372
  #emit(event, payload) {
13439
14373
  for (const listener of this.#listeners.get(event) ?? []) {
13440
14374
  try {
@@ -16105,7 +17039,7 @@ const defaultGithubWriteback = (config, mount) => {
16105
17039
  if (!mount.githubWrite) {
16106
17040
  throw new Error('GitHub identity "app" requires a connected workspace GitHub App lifecycle write path; refusing to fall back to the local gh user');
16107
17041
  }
16108
- return new AppGithubWriteback(mount.githubWrite);
17042
+ return new AppGithubWriteback(mount.githubWrite, mount.githubRead);
16109
17043
  };
16110
17044
  export function parseLinearIssue(path, content) {
16111
17045
  const parsed = parseJsonContent(content);
@@ -18909,6 +19843,10 @@ const UNCLASSIFIED_DISPATCH_FAILURE_LIMIT = 5;
18909
19843
  */
18910
19844
  const isClassifiedPerItemDispatchFailure = (error) => error instanceof LiveDispatchStateChangedError ||
18911
19845
  error instanceof DispatchLifecycleClaimRefusedError ||
19846
+ // An external stop/held-agent deadline rejected the post-spawn boundary
19847
+ // while a provider claim was already in flight. That lifecycle's cleanup is
19848
+ // owned by the rejecting path, so the late claim unwind is an expected skip.
19849
+ error instanceof PostSpawnDispatchWaitRejectedError ||
18912
19850
  // #303: the never-placed deadline released this dispatch while its spawn was
18913
19851
  // still in flight. Named, expected and self-healing — the issue goes back to
18914
19852
  // the queue — and it recurs under exactly the slow-spawn conditions the
@@ -18933,7 +19871,8 @@ const isClassifiedPerItemDispatchFailure = (error) => error instanceof LiveDispa
18933
19871
  * of a missed one is leaked agents and duplicate workers on the next retry.
18934
19872
  */
18935
19873
  const mayHaveSpawnedBeforeFailing = (error) => !(error instanceof LiveDispatchStateChangedError) &&
18936
- !(error instanceof DispatchLifecycleClaimRefusedError);
19874
+ !(error instanceof DispatchLifecycleClaimRefusedError) &&
19875
+ !(error instanceof PostSpawnDispatchWaitRejectedError);
18937
19876
  /**
18938
19877
  * The run-report reason recorded for a work unit the pass could not dispatch.
18939
19878
  *
@@ -18951,6 +19890,8 @@ const perItemDispatchSkipReason = (error) => {
18951
19890
  return 'live state changed during dispatch';
18952
19891
  if (error instanceof LatePlacementReleasedError)
18953
19892
  return 'dispatch released while its agent was still spawning';
19893
+ if (error instanceof PostSpawnDispatchWaitRejectedError)
19894
+ return 'dispatch terminated during post-spawn claim';
18954
19895
  if (error instanceof DispatchLifecycleClaimRefusedError) {
18955
19896
  return error.refusal === 'terminal'
18956
19897
  ? 'dispatch lifecycle already terminal'