@agent-relay/factory 0.1.60 → 0.1.62

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/README.md +23 -1
  2. package/dist/cli/fleet.d.ts +1 -0
  3. package/dist/cli/fleet.d.ts.map +1 -1
  4. package/dist/cli/fleet.js +51 -11
  5. package/dist/cli/fleet.js.map +1 -1
  6. package/dist/config/schema.d.ts +19 -0
  7. package/dist/config/schema.d.ts.map +1 -1
  8. package/dist/config/schema.js +7 -0
  9. package/dist/config/schema.js.map +1 -1
  10. package/dist/index.d.ts +3 -3
  11. package/dist/index.d.ts.map +1 -1
  12. package/dist/index.js +2 -2
  13. package/dist/index.js.map +1 -1
  14. package/dist/orchestrator/batch-tracker.d.ts +5 -0
  15. package/dist/orchestrator/batch-tracker.d.ts.map +1 -1
  16. package/dist/orchestrator/batch-tracker.js +2 -0
  17. package/dist/orchestrator/batch-tracker.js.map +1 -1
  18. package/dist/orchestrator/factory.d.ts.map +1 -1
  19. package/dist/orchestrator/factory.js +480 -67
  20. package/dist/orchestrator/factory.js.map +1 -1
  21. package/dist/orchestrator/index.d.ts +1 -1
  22. package/dist/orchestrator/index.d.ts.map +1 -1
  23. package/dist/orchestrator/index.js +1 -1
  24. package/dist/orchestrator/index.js.map +1 -1
  25. package/dist/orchestrator/reaper.d.ts +16 -2
  26. package/dist/orchestrator/reaper.d.ts.map +1 -1
  27. package/dist/orchestrator/reaper.js +79 -5
  28. package/dist/orchestrator/reaper.js.map +1 -1
  29. package/dist/ports/state.d.ts +20 -5
  30. package/dist/ports/state.d.ts.map +1 -1
  31. package/dist/state/file-state-store.d.ts +4 -1
  32. package/dist/state/file-state-store.d.ts.map +1 -1
  33. package/dist/state/file-state-store.js +45 -4
  34. package/dist/state/file-state-store.js.map +1 -1
  35. package/dist/state/in-memory-state-store.d.ts +2 -1
  36. package/dist/state/in-memory-state-store.d.ts.map +1 -1
  37. package/dist/state/in-memory-state-store.js +12 -3
  38. package/dist/state/in-memory-state-store.js.map +1 -1
  39. package/dist/types.d.ts +33 -1
  40. package/dist/types.d.ts.map +1 -1
  41. package/package.json +1 -1
@@ -137,6 +137,8 @@ const DISPATCH_LIFECYCLE_RENEW_MS = 60_000;
137
137
  const DISPATCH_LIFECYCLE_RETRY_MS = 1_000;
138
138
  const DISPATCH_WRITEBACK_MAX_ATTEMPTS = 3;
139
139
  const DISPATCH_WRITEBACK_RETRY_MS = 250;
140
+ const HELD_PAST_DEADLINE_RELEASE_REASON = 'held-past-deadline';
141
+ const HELD_DEADLINE_OVERDUE_RETRY_MS = 1_000;
140
142
  const STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS = 30_000;
141
143
  const RECONCILED_AGENT_EXIT_CONCURRENCY = 4;
142
144
  const SLACK_EVENT_WATERMARK_CACHE_MS = 60_000;
@@ -152,6 +154,7 @@ const REMOTE_OPERATION_PROGRESS_INTERVAL_MS = 15_000;
152
154
  const REMOTE_OPERATION_SLOW_WARN_MS = 30_000;
153
155
  const DISCOVERY_SWEEP_LEASE_MS = 5 * 60_000;
154
156
  const DISCOVERY_SWEEP_RENEW_MS = 30_000;
157
+ const READINESS_RECONCILE_FAILURE_THRESHOLD = 3;
155
158
  const DISCOVERY_CHANGE_EVENT_LIMIT = 1_000;
156
159
  const DISCOVERY_OVERLOAD_BACKOFF_MAX_MS = 5 * 60_000;
157
160
  const GITHUB_FACTORY_LABEL = 'factory';
@@ -310,6 +313,9 @@ export class FactoryLoop {
310
313
  #dispatchClaimStatuses = new Map();
311
314
  #localReleaseCheckpoints = new Map();
312
315
  #dispatchLifecycleRenewTimer;
316
+ #heldAgentDeadlineTimer;
317
+ #heldAgentDeadlineDueAtMs;
318
+ #heldAgentDeadlineSweepInFlight;
313
319
  #clarificationSweepTimer;
314
320
  #clarificationSweepDueAtMs;
315
321
  #clarificationSweepInFlight;
@@ -341,6 +347,12 @@ export class FactoryLoop {
341
347
  #readinessReconcileTimer;
342
348
  #readinessReconcileInFlight;
343
349
  #readinessReconcileIntervalMs = 60_000;
350
+ #readinessReconcileConsecutiveFailures = 0;
351
+ #readinessReconcileLastDurationMs;
352
+ #readinessReconcileLastStartedAtMs;
353
+ #readinessReconcileLastCompletedAtMs;
354
+ #readinessReconcileLastFailureAtMs;
355
+ #readinessReconcileLastError;
344
356
  #liveEventQueue = [];
345
357
  #liveEventDrainScheduled = false;
346
358
  #liveEventDrainActive = false;
@@ -419,7 +431,9 @@ export class FactoryLoop {
419
431
  #runOnceInFlightDryRun;
420
432
  #discoverySession;
421
433
  #discoverySweepEpoch;
434
+ #discoverySweepStartedAtMs;
422
435
  #discoverySweepRenewTimer;
436
+ #discoverySweepRenewalInFlight;
423
437
  #discoverySweepLeaseLost = false;
424
438
  #discoveryOverloadError;
425
439
  #resolvedIssueSource;
@@ -785,6 +799,11 @@ export class FactoryLoop {
785
799
  if (this.#dispatchLifecycleRenewTimer)
786
800
  clearInterval(this.#dispatchLifecycleRenewTimer);
787
801
  this.#dispatchLifecycleRenewTimer = undefined;
802
+ if (this.#heldAgentDeadlineTimer)
803
+ clearTimeout(this.#heldAgentDeadlineTimer);
804
+ this.#heldAgentDeadlineTimer = undefined;
805
+ this.#heldAgentDeadlineDueAtMs = undefined;
806
+ await this.#heldAgentDeadlineSweepInFlight;
788
807
  for (const timer of this.#dispatchLifecycleRetryTimers.values())
789
808
  clearTimeout(timer);
790
809
  this.#dispatchLifecycleRetryTimers.clear();
@@ -1122,24 +1141,40 @@ export class FactoryLoop {
1122
1141
  this.#readinessReconcileTimer.unref?.();
1123
1142
  }
1124
1143
  async #reconcileReadyIssues() {
1144
+ const startedAtMs = this.#clock.now();
1145
+ this.#readinessReconcileLastStartedAtMs = startedAtMs;
1125
1146
  this.#increment('readinessReconcileSweeps');
1126
1147
  this.#logger.info?.('[factory] periodic readiness reconciliation started', {
1127
1148
  intervalMs: this.#readinessReconcileIntervalMs,
1128
1149
  });
1129
1150
  try {
1130
1151
  const report = await this.runOnce();
1152
+ this.#readinessReconcileConsecutiveFailures = 0;
1153
+ this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
1154
+ this.#readinessReconcileLastCompletedAtMs = this.#clock.now();
1155
+ this.#readinessReconcileLastError = undefined;
1131
1156
  this.#logger.info?.('[factory] periodic readiness reconciliation completed', {
1157
+ durationMs: this.#readinessReconcileLastDurationMs,
1132
1158
  candidates: report.pulled.length,
1133
1159
  dispatched: report.dispatched.length,
1134
1160
  skipped: report.skipped.length,
1135
1161
  });
1136
1162
  }
1137
1163
  catch (error) {
1164
+ const errorMessage = describeError(error).errorMessage;
1165
+ this.#readinessReconcileConsecutiveFailures += 1;
1166
+ this.#readinessReconcileLastDurationMs = this.#elapsedSince(startedAtMs);
1167
+ this.#readinessReconcileLastFailureAtMs = this.#clock.now();
1168
+ this.#readinessReconcileLastError = errorMessage;
1138
1169
  this.#increment('readinessReconcileErrors');
1139
1170
  this.#logger.warn?.('[factory] periodic readiness reconciliation failed; retry remains scheduled', {
1140
- error: describeError(error).errorMessage,
1171
+ error: errorMessage,
1172
+ durationMs: this.#readinessReconcileLastDurationMs,
1173
+ consecutiveFailures: this.#readinessReconcileConsecutiveFailures,
1174
+ degraded: this.#readinessReconcileConsecutiveFailures >= READINESS_RECONCILE_FAILURE_THRESHOLD,
1141
1175
  });
1142
1176
  }
1177
+ await this.#refreshLiveHeartbeat();
1143
1178
  }
1144
1179
  #scheduleLivePoll(delayMs, options) {
1145
1180
  if (this.#livePollTimer || !this.#started)
@@ -1797,6 +1832,7 @@ export class FactoryLoop {
1797
1832
  }
1798
1833
  }
1799
1834
  async #runOnceWithDiscoveryFence(opts) {
1835
+ const sweepStartedAtMs = this.#clock.now();
1800
1836
  let claim = await this.#state.claimDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, this.#clock.now(), DISCOVERY_SWEEP_LEASE_MS);
1801
1837
  if (!claim.acquired && claim.reason === 'backoff') {
1802
1838
  const delayMs = Math.max(0, claim.state.backoffUntilMs - this.#clock.now());
@@ -1812,6 +1848,8 @@ export class FactoryLoop {
1812
1848
  if (!claim.acquired || !claim.lease) {
1813
1849
  this.#increment('discoverySweepsSkippedInFlight');
1814
1850
  this.#logger.info?.('[factory] skipped discovery because another process owns the sweep lease', {
1851
+ owner: claim.state.lease?.owner,
1852
+ epoch: claim.state.lease?.epoch,
1815
1853
  leaseUntilMs: claim.state.lease?.leaseUntilMs,
1816
1854
  });
1817
1855
  return {
@@ -1823,7 +1861,23 @@ export class FactoryLoop {
1823
1861
  discoveryDeferred: 'sweep-in-flight',
1824
1862
  };
1825
1863
  }
1864
+ if (claim.reclaimedLease) {
1865
+ this.#increment('discoverySweepOrphanTakeovers');
1866
+ this.#logger.warn?.('[factory] reclaimed discovery sweep lease from a stopped process', {
1867
+ owner: claim.lease.owner,
1868
+ epoch: claim.lease.epoch,
1869
+ previousOwner: claim.reclaimedLease.owner,
1870
+ previousEpoch: claim.reclaimedLease.epoch,
1871
+ previousLeaseUntilMs: claim.reclaimedLease.leaseUntilMs,
1872
+ });
1873
+ }
1874
+ this.#logger.info?.('[factory] discovery sweep lease claimed', {
1875
+ owner: claim.lease.owner,
1876
+ epoch: claim.lease.epoch,
1877
+ leaseUntilMs: claim.lease.leaseUntilMs,
1878
+ });
1826
1879
  this.#discoverySweepEpoch = claim.lease.epoch;
1880
+ this.#discoverySweepStartedAtMs = sweepStartedAtMs;
1827
1881
  this.#discoverySweepLeaseLost = false;
1828
1882
  this.#discoveryOverloadError = undefined;
1829
1883
  this.#startDiscoverySweepRenewal(claim.lease.epoch);
@@ -1834,6 +1888,11 @@ export class FactoryLoop {
1834
1888
  if (this.#discoveryOverloadError)
1835
1889
  throw this.#discoveryOverloadError;
1836
1890
  const checkpoint = await this.#finalizeDiscoveryCheckpoint();
1891
+ // Do not clear the durable lease while a renewal can still be waiting on
1892
+ // the same state-file lock. A late renewal that observes the completed
1893
+ // (lease-less) checkpoint is a false lease-loss signal and can poison an
1894
+ // otherwise successful reconcile cycle.
1895
+ await this.#stopDiscoverySweepRenewal();
1837
1896
  if (this.#discoverySweepLeaseLost) {
1838
1897
  throw new Error('discovery sweep lease was lost before checkpoint commit');
1839
1898
  }
@@ -1841,9 +1900,16 @@ export class FactoryLoop {
1841
1900
  leaseReleased = completed;
1842
1901
  if (!completed)
1843
1902
  throw new Error('discovery sweep lease was lost before completion');
1903
+ this.#logger.info?.('[factory] discovery sweep checkpoint committed', {
1904
+ owner: claim.lease.owner,
1905
+ epoch: claim.lease.epoch,
1906
+ durationMs: this.#elapsedSince(sweepStartedAtMs),
1907
+ checkpointUpdatedAtMs: checkpoint?.updatedAtMs,
1908
+ });
1844
1909
  return report;
1845
1910
  }
1846
1911
  catch (error) {
1912
+ await this.#stopDiscoverySweepRenewal();
1847
1913
  const overload = relayfileOverload(error);
1848
1914
  if (overload) {
1849
1915
  const consecutiveOverloads = claim.state.consecutiveOverloads + 1;
@@ -1868,11 +1934,10 @@ export class FactoryLoop {
1868
1934
  throw error;
1869
1935
  }
1870
1936
  finally {
1871
- if (this.#discoverySweepRenewTimer)
1872
- clearInterval(this.#discoverySweepRenewTimer);
1873
- this.#discoverySweepRenewTimer = undefined;
1937
+ await this.#stopDiscoverySweepRenewal();
1874
1938
  this.#discoverySession = undefined;
1875
1939
  this.#discoverySweepEpoch = undefined;
1940
+ this.#discoverySweepStartedAtMs = undefined;
1876
1941
  this.#discoveryOverloadError = undefined;
1877
1942
  // This sweep is over either way (committed, deferred, or lease lost) —
1878
1943
  // a stale `true` here would otherwise make every #listRelayfileTree
@@ -2080,27 +2145,69 @@ export class FactoryLoop {
2080
2145
  }
2081
2146
  }
2082
2147
  #startDiscoverySweepRenewal(epoch) {
2083
- let renewalInFlight = false;
2084
2148
  this.#discoverySweepRenewTimer = setInterval(() => {
2085
- if (renewalInFlight || this.#discoverySweepLeaseLost)
2149
+ if (this.#discoverySweepRenewalInFlight || this.#discoverySweepLeaseLost)
2086
2150
  return;
2087
- renewalInFlight = true;
2088
- void this.#state.renewDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, epoch, this.#clock.now(), DISCOVERY_SWEEP_LEASE_MS).then((renewed) => {
2089
- if (!renewed && this.#discoverySweepEpoch === epoch) {
2090
- this.#discoverySweepLeaseLost = true;
2091
- this.#increment('discoverySweepLeaseLosses');
2092
- this.#logger.warn?.('[factory] discovery sweep lease renewal was rejected; fencing further tree requests');
2151
+ const renewal = this.#renewDiscoverySweepLease(epoch);
2152
+ this.#discoverySweepRenewalInFlight = renewal;
2153
+ void renewal.finally(() => {
2154
+ if (this.#discoverySweepRenewalInFlight === renewal) {
2155
+ this.#discoverySweepRenewalInFlight = undefined;
2093
2156
  }
2094
- }).catch((error) => {
2095
- this.#logger.warn?.('[factory] discovery sweep lease renewal failed; retaining the current lease window', {
2096
- error: describeError(error).errorMessage,
2097
- });
2098
- }).finally(() => {
2099
- renewalInFlight = false;
2100
2157
  });
2101
2158
  }, DISCOVERY_SWEEP_RENEW_MS);
2102
2159
  this.#discoverySweepRenewTimer.unref?.();
2103
2160
  }
2161
+ async #renewDiscoverySweepLease(epoch) {
2162
+ const requestedAtMs = this.#clock.now();
2163
+ try {
2164
+ let renewal;
2165
+ if (this.#state.renewDiscoverySweepWithDetails) {
2166
+ renewal = await this.#state.renewDiscoverySweepWithDetails(this.#workspaceId, this.#discoverySweepOwner, epoch, requestedAtMs, DISCOVERY_SWEEP_LEASE_MS);
2167
+ }
2168
+ else {
2169
+ const renewed = await this.#state.renewDiscoverySweep(this.#workspaceId, this.#discoverySweepOwner, epoch, requestedAtMs, DISCOVERY_SWEEP_LEASE_MS);
2170
+ renewal = renewed
2171
+ ? {
2172
+ renewed: true,
2173
+ lease: {
2174
+ owner: this.#discoverySweepOwner,
2175
+ epoch,
2176
+ leaseUntilMs: requestedAtMs + DISCOVERY_SWEEP_LEASE_MS,
2177
+ },
2178
+ }
2179
+ : { renewed: false, reason: 'unknown' };
2180
+ }
2181
+ if (!renewal.renewed && this.#discoverySweepEpoch === epoch) {
2182
+ this.#discoverySweepLeaseLost = true;
2183
+ this.#increment('discoverySweepLeaseLosses');
2184
+ this.#logger.warn?.('[factory] discovery sweep lease renewal was rejected; fencing further tree requests', {
2185
+ reason: renewal.reason,
2186
+ requestedOwner: this.#discoverySweepOwner,
2187
+ requestedEpoch: epoch,
2188
+ requestedAtMs,
2189
+ elapsedMs: this.#elapsedSince(this.#discoverySweepStartedAtMs ?? requestedAtMs),
2190
+ observedOwner: renewal.observedLease?.owner,
2191
+ observedEpoch: renewal.observedLease?.epoch,
2192
+ observedLeaseUntilMs: renewal.observedLease?.leaseUntilMs,
2193
+ });
2194
+ }
2195
+ }
2196
+ catch (error) {
2197
+ this.#logger.warn?.('[factory] discovery sweep lease renewal failed; retaining the current lease window', {
2198
+ owner: this.#discoverySweepOwner,
2199
+ epoch,
2200
+ requestedAtMs,
2201
+ error: describeError(error).errorMessage,
2202
+ });
2203
+ }
2204
+ }
2205
+ async #stopDiscoverySweepRenewal() {
2206
+ if (this.#discoverySweepRenewTimer)
2207
+ clearInterval(this.#discoverySweepRenewTimer);
2208
+ this.#discoverySweepRenewTimer = undefined;
2209
+ await this.#discoverySweepRenewalInFlight;
2210
+ }
2104
2211
  async #prepareDiscoverySession(claim) {
2105
2212
  const checkpoint = structuredClone(claim.state.checkpoint ?? {
2106
2213
  trees: {},
@@ -3282,7 +3389,7 @@ export class FactoryLoop {
3282
3389
  await this.#notifyTicketDispatch(dispatchDecision, liveIssue, record, result);
3283
3390
  }
3284
3391
  if (!dryRun) {
3285
- await this.#ensureSlackDispatchThread(record, result);
3392
+ await this.#ensureSlackDispatchThread(record, result, liveIssue);
3286
3393
  }
3287
3394
  return result;
3288
3395
  }
@@ -3375,6 +3482,7 @@ export class FactoryLoop {
3375
3482
  }
3376
3483
  status() {
3377
3484
  const batch = this.#batchView;
3485
+ const nowMs = this.#clock.now();
3378
3486
  const inFlightDispatches = batch?.inFlight
3379
3487
  .filter((record) => !record.dryRun)
3380
3488
  .map((record) => ({
@@ -3389,7 +3497,7 @@ export class FactoryLoop {
3389
3497
  claim: {
3390
3498
  ...(record.dispatchClaim ?? this.#dispatchClaimStatuses.get(issueKey(record.issue)) ?? {
3391
3499
  state: 'pending',
3392
- updatedAtMs: this.#clock.now(),
3500
+ updatedAtMs: nowMs,
3393
3501
  }),
3394
3502
  },
3395
3503
  })) ?? [];
@@ -3407,6 +3515,8 @@ export class FactoryLoop {
3407
3515
  slackDegraded: this.#slackDegraded,
3408
3516
  slackDegradedReason: this.#slackDegradedReason,
3409
3517
  eventListener: this.#eventListenerStatus(),
3518
+ readinessReconcile: this.#readinessReconcileStatus(),
3519
+ heldAgents: batch?.inFlight.flatMap((record) => heldAgentsForRecord(record, nowMs, this.#config.dispatch.agentHoldTimeoutMs, this.#config.terminalState)) ?? [],
3410
3520
  };
3411
3521
  }
3412
3522
  #eventListenerStatus() {
@@ -3427,6 +3537,34 @@ export class FactoryLoop {
3427
3537
  }
3428
3538
  return { state: 'starting' };
3429
3539
  }
3540
+ #readinessReconcileStatus() {
3541
+ const consecutiveFailures = this.#readinessReconcileConsecutiveFailures;
3542
+ const state = this.#startMode !== 'live'
3543
+ ? 'not-running'
3544
+ : consecutiveFailures >= READINESS_RECONCILE_FAILURE_THRESHOLD
3545
+ ? 'degraded'
3546
+ : consecutiveFailures > 0
3547
+ ? 'retrying'
3548
+ : 'healthy';
3549
+ return {
3550
+ state,
3551
+ consecutiveFailures,
3552
+ failureThreshold: READINESS_RECONCILE_FAILURE_THRESHOLD,
3553
+ ...(this.#readinessReconcileLastDurationMs !== undefined
3554
+ ? { lastDurationMs: this.#readinessReconcileLastDurationMs }
3555
+ : {}),
3556
+ ...(this.#readinessReconcileLastStartedAtMs !== undefined
3557
+ ? { lastStartedAtMs: this.#readinessReconcileLastStartedAtMs }
3558
+ : {}),
3559
+ ...(this.#readinessReconcileLastCompletedAtMs !== undefined
3560
+ ? { lastCompletedAtMs: this.#readinessReconcileLastCompletedAtMs }
3561
+ : {}),
3562
+ ...(this.#readinessReconcileLastFailureAtMs !== undefined
3563
+ ? { lastFailureAtMs: this.#readinessReconcileLastFailureAtMs }
3564
+ : {}),
3565
+ ...(this.#readinessReconcileLastError ? { lastError: this.#readinessReconcileLastError } : {}),
3566
+ };
3567
+ }
3430
3568
  on(event, listener) {
3431
3569
  let listeners = this.#listeners.get(event);
3432
3570
  if (!listeners) {
@@ -3687,6 +3825,7 @@ export class FactoryLoop {
3687
3825
  });
3688
3826
  }
3689
3827
  }
3828
+ this.#rescheduleHeldAgentDeadlineSweep();
3690
3829
  }
3691
3830
  catch (error) {
3692
3831
  this.#logger.warn?.('[factory] failed to re-adopt durable in-flight agents', { error });
@@ -3799,6 +3938,93 @@ export class FactoryLoop {
3799
3938
  }, DISPATCH_LIFECYCLE_RENEW_MS);
3800
3939
  this.#dispatchLifecycleRenewTimer.unref?.();
3801
3940
  }
3941
+ #scheduleHeldAgentDeadline(record) {
3942
+ if (this.#stopping || record.dryRun || record.heldSinceAtMs === undefined || record.agents.size === 0)
3943
+ return;
3944
+ const dueAtMs = record.heldSinceAtMs + this.#config.dispatch.agentHoldTimeoutMs;
3945
+ if (this.#heldAgentDeadlineTimer &&
3946
+ this.#heldAgentDeadlineDueAtMs !== undefined &&
3947
+ this.#heldAgentDeadlineDueAtMs <= dueAtMs)
3948
+ return;
3949
+ if (this.#heldAgentDeadlineTimer)
3950
+ clearTimeout(this.#heldAgentDeadlineTimer);
3951
+ this.#heldAgentDeadlineDueAtMs = dueAtMs;
3952
+ const remainingMs = dueAtMs - this.#clock.now();
3953
+ // An overdue lifecycle can temporarily be fenced by another owner. Avoid
3954
+ // a zero-delay reschedule loop while its lease is being reclaimed.
3955
+ const delayMs = remainingMs <= 0
3956
+ ? HELD_DEADLINE_OVERDUE_RETRY_MS
3957
+ : Math.min(remainingMs, 2_147_483_647);
3958
+ this.#heldAgentDeadlineTimer = setTimeout(() => {
3959
+ this.#heldAgentDeadlineTimer = undefined;
3960
+ this.#heldAgentDeadlineDueAtMs = undefined;
3961
+ const sweep = this.#sweepHeldAgentDeadlines()
3962
+ .catch((error) => {
3963
+ this.#logger.warn?.('[factory] held-agent deadline sweep failed; retrying', {
3964
+ error: describeError(error).errorMessage,
3965
+ });
3966
+ })
3967
+ .finally(() => {
3968
+ if (this.#heldAgentDeadlineSweepInFlight === sweep)
3969
+ this.#heldAgentDeadlineSweepInFlight = undefined;
3970
+ this.#rescheduleHeldAgentDeadlineSweep();
3971
+ });
3972
+ this.#heldAgentDeadlineSweepInFlight = sweep;
3973
+ }, delayMs);
3974
+ this.#heldAgentDeadlineTimer.unref?.();
3975
+ }
3976
+ #rescheduleHeldAgentDeadlineSweep() {
3977
+ if (this.#stopping)
3978
+ return;
3979
+ for (const record of this.#batchView?.inFlight ?? [])
3980
+ this.#scheduleHeldAgentDeadline(record);
3981
+ }
3982
+ async #sweepHeldAgentDeadlines() {
3983
+ const nowMs = this.#clock.now();
3984
+ const timeoutMs = this.#config.dispatch.agentHoldTimeoutMs;
3985
+ for (const record of [...(await this.#batch()).inFlight]) {
3986
+ const heldSinceAtMs = record.heldSinceAtMs;
3987
+ if (record.dryRun ||
3988
+ heldSinceAtMs === undefined ||
3989
+ record.agents.size === 0 ||
3990
+ nowMs - heldSinceAtMs < timeoutMs)
3991
+ continue;
3992
+ const key = issueKey(record.issue);
3993
+ if (this.#abandonedDispatchReasons.has(key))
3994
+ continue;
3995
+ if (this.#usesDurableDispatchLifecycle()) {
3996
+ const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
3997
+ if (!lifecycle || isTerminalDispatchLifecycle(lifecycle))
3998
+ continue;
3999
+ // Terminal writeback already won the race. Finish its normal release
4000
+ // reason instead of relabeling an acknowledged completion as a timeout.
4001
+ if (lifecycle.phase === 'releasing') {
4002
+ await this.#finishDurableRelease(record, lifecycle.releaseReason);
4003
+ continue;
4004
+ }
4005
+ if (!await this.#assertDispatchLifecycleOwner(record))
4006
+ continue;
4007
+ }
4008
+ const heldForMs = Math.max(0, this.#clock.now() - heldSinceAtMs);
4009
+ const details = {
4010
+ issue: record.issue.key,
4011
+ heldForMs,
4012
+ holdTimeoutMs: timeoutMs,
4013
+ waitingForTerminalState: this.#config.terminalState,
4014
+ reason: HELD_PAST_DEADLINE_RELEASE_REASON,
4015
+ agents: [...record.agents.keys()].sort(),
4016
+ };
4017
+ this.#logger.warn?.('[factory] releasing agents held past deadline', details);
4018
+ await this.#abandonStuckDispatch(record, HELD_PAST_DEADLINE_RELEASE_REASON);
4019
+ const lifecycle = this.#usesDurableDispatchLifecycle()
4020
+ ? await this.#state.getDispatchLifecycle(this.#workspaceId, key)
4021
+ : undefined;
4022
+ if (!lifecycle || isTerminalDispatchLifecycle(lifecycle)) {
4023
+ this.#increment('heldPastDeadlineReleases');
4024
+ this.#logger.warn?.('[factory] released agents held past deadline', details);
4025
+ }
4026
+ }
4027
+ }
3802
4028
  async #renewDispatchLifecycles() {
3803
4029
  for (const [key, epoch] of [...this.#dispatchLifecycleEpochs]) {
3804
4030
  const renewed = await this.#state.renewDispatchLifecycle(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS);
@@ -4164,6 +4390,7 @@ export class FactoryLoop {
4164
4390
  });
4165
4391
  }
4166
4392
  async #saveDispatchLifecycle(record, phase, pullRequest, releaseReason, releasedAgentNames = new Set(), telemetry = {}) {
4393
+ record.lifecyclePhase = phase;
4167
4394
  if (record.dryRun || !this.#usesDurableDispatchLifecycle())
4168
4395
  return true;
4169
4396
  if (isTerminalDispatchPhase(phase))
@@ -4367,6 +4594,7 @@ export class FactoryLoop {
4367
4594
  const batch = await this.#batch();
4368
4595
  const durableRecord = inFlightRecordFromLifecycle(lifecycle);
4369
4596
  const record = lifecycle.phase === 'releasing' ? durableRecord : batch.restore(durableRecord);
4597
+ this.#scheduleHeldAgentDeadline(record);
4370
4598
  if (!await this.#assertDispatchLifecycleOwner(record))
4371
4599
  return;
4372
4600
  if (acquiredNow && this.#config.babysitter.enabled)
@@ -5790,6 +6018,7 @@ export class FactoryLoop {
5790
6018
  updatedAtMs,
5791
6019
  registryPath,
5792
6020
  eventListener: this.#eventListenerStatus(),
6021
+ readinessReconcile: this.#readinessReconcileStatus(),
5793
6022
  };
5794
6023
  await mkdir(dirname(path), { recursive: true });
5795
6024
  await writeFile(path, `${JSON.stringify(heartbeat, null, 2)}\n`, 'utf8');
@@ -6214,7 +6443,7 @@ export class FactoryLoop {
6214
6443
  const updatedAtMs = this.#clock.now();
6215
6444
  const agents = [];
6216
6445
  const seenAgents = new Set();
6217
- const appendAgent = async (issue, agentName, tracked) => {
6446
+ const appendAgent = async (issue, agentName, tracked, hold) => {
6218
6447
  const key = registryHandoffKey(issue, agentName);
6219
6448
  if (seenAgents.has(key)) {
6220
6449
  return;
@@ -6240,6 +6469,12 @@ export class FactoryLoop {
6240
6469
  ...(fleetTracked?.invocationId ? { invocationId: fleetTracked.invocationId } : {}),
6241
6470
  ...(fleetTracked?.node ? { node: fleetTracked.node } : {}),
6242
6471
  ...(dispatchClaim ? { dispatchClaim: { ...dispatchClaim } } : {}),
6472
+ ...(hold?.heldSinceAtMs !== undefined ? {
6473
+ heldSinceAtMs: hold.heldSinceAtMs,
6474
+ holdDeadlineAtMs: hold.heldSinceAtMs + this.#config.dispatch.agentHoldTimeoutMs,
6475
+ waitingForTerminalState: this.#config.terminalState,
6476
+ ...(hold.lifecyclePhase ? { lifecyclePhase: hold.lifecyclePhase } : {}),
6477
+ } : {}),
6243
6478
  });
6244
6479
  };
6245
6480
  if (!empty) {
@@ -6250,7 +6485,7 @@ export class FactoryLoop {
6250
6485
  this.#dispatchClaimStatuses.set(issueKey(record.issue), record.dispatchClaim);
6251
6486
  }
6252
6487
  for (const [agentName, tracked] of record.agents) {
6253
- await appendAgent(record.issue, agentName, tracked);
6488
+ await appendAgent(record.issue, agentName, tracked, record);
6254
6489
  }
6255
6490
  }
6256
6491
  }
@@ -6272,6 +6507,7 @@ export class FactoryLoop {
6272
6507
  const invocationId = batch.invocationIdFor(record.issue, spec);
6273
6508
  const existing = record.agents.get(spec.name);
6274
6509
  if (existing?.result) {
6510
+ this.#scheduleHeldAgentDeadline(record);
6275
6511
  return { name: existing.result?.name ?? spec.name };
6276
6512
  }
6277
6513
  if (!batch.shouldSpawn(record, invocationId)) {
@@ -6298,6 +6534,7 @@ export class FactoryLoop {
6298
6534
  const rosterAgent = roster.agents.find((agent) => agent.name === spec.name);
6299
6535
  if (rosterAgent) {
6300
6536
  const trackedPlacement = this.#fleet.trackedAgents?.().get(spec.name);
6537
+ record.heldSinceAtMs ??= this.#clock.now();
6301
6538
  batch.recordSpawn(record, spec, invocationId, {
6302
6539
  name: spec.name,
6303
6540
  sessionRef: existing?.sessionRef ?? spec.sessionRef,
@@ -6307,6 +6544,7 @@ export class FactoryLoop {
6307
6544
  if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
6308
6545
  throw new Error(`Dispatch lifecycle ownership lost after adopting ${spec.name}`);
6309
6546
  }
6547
+ this.#scheduleHeldAgentDeadline(record);
6310
6548
  const adopted = record.agents.get(spec.name);
6311
6549
  if (adopted)
6312
6550
  await this.#reportAgent(record, adopted, 'agent.adopted');
@@ -6339,10 +6577,12 @@ export class FactoryLoop {
6339
6577
  : 'agent_spawn_failed',
6340
6578
  });
6341
6579
  }
6580
+ record.heldSinceAtMs ??= this.#clock.now();
6342
6581
  batch.recordSpawn(record, spec, invocationId, result);
6343
6582
  if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
6344
6583
  throw new Error(`Dispatch lifecycle ownership lost after spawning ${spec.name}`);
6345
6584
  }
6585
+ this.#scheduleHeldAgentDeadline(record);
6346
6586
  const spawned = record.agents.get(result.name);
6347
6587
  if (spawned)
6348
6588
  await this.#reportAgent(record, spawned, 'agent.spawned');
@@ -7301,7 +7541,10 @@ export class FactoryLoop {
7301
7541
  try {
7302
7542
  const thread = await this.#slackDispatchThreadFor(record);
7303
7543
  if (thread && this.#slack) {
7304
- await this.#slack.reply(thread.threadId, `:warning: ${record.issue.key}: the implementer exited without opening a PR (after a retry). It needs a human look.`);
7544
+ const issue = await this.#readIssue(record.issue.path);
7545
+ if (!issue)
7546
+ return;
7547
+ await this.#slack.reply(thread.threadId, `:warning: ${slackIssueSubject(issue, slackNotificationRepos(record.decision))}\nThe implementer exited without opening a PR after a retry; this needs a human look.`);
7305
7548
  }
7306
7549
  }
7307
7550
  catch (error) {
@@ -7362,6 +7605,8 @@ export class FactoryLoop {
7362
7605
  // record leaves the batch.
7363
7606
  async #abandonStuckDispatch(record, reason) {
7364
7607
  const key = issueKey(record.issue);
7608
+ const heldPastDeadline = reason === HELD_PAST_DEADLINE_RELEASE_REASON;
7609
+ const agentReleaseReason = heldPastDeadline ? HELD_PAST_DEADLINE_RELEASE_REASON : 'issue-abandoned';
7365
7610
  this.#abandonedDispatchReasons.set(key, reason);
7366
7611
  if (!await this.#saveDispatchLifecycle(record, 'abandoning', undefined, reason, new Set(), { cancellationReason: 'dispatch_failed' })) {
7367
7612
  this.#increment('abandonedDispatchReleaseRetries');
@@ -7387,9 +7632,9 @@ export class FactoryLoop {
7387
7632
  }
7388
7633
  const agents = [...record.agents];
7389
7634
  for (const [agentName, tracked] of agents) {
7390
- if (tracked.spec.role === 'implementer')
7635
+ if (!heldPastDeadline && tracked.spec.role === 'implementer')
7391
7636
  continue;
7392
- this.#fleet.markAgentTerminal?.(agentName, `implementer-terminal:${reason}`);
7637
+ this.#fleet.markAgentTerminal?.(agentName, heldPastDeadline ? HELD_PAST_DEADLINE_RELEASE_REASON : `implementer-terminal:${reason}`);
7393
7638
  }
7394
7639
  const worktreeHandoffs = this.#dispatchFailureHandoffs(record, []);
7395
7640
  let cleanupComplete = true;
@@ -7402,13 +7647,13 @@ export class FactoryLoop {
7402
7647
  const worktreeAgentNames = new Set(worktreeHandoffs.map((handoff) => handoff.name));
7403
7648
  const nonWorktreeAgents = agents.filter(([name]) => !worktreeAgentNames.has(name));
7404
7649
  if (nonWorktreeAgents.length > 0) {
7405
- const failed = await this.#releaseAndTerminateAgents(nonWorktreeAgents, 'issue-abandoned', 'completion');
7650
+ const failed = await this.#releaseAndTerminateAgents(nonWorktreeAgents, agentReleaseReason, 'completion');
7406
7651
  cleanupComplete = failed.length === 0;
7407
7652
  }
7408
- cleanupComplete = await this.#teardownFailedDispatchWorktrees(worktreeHandoffs) && cleanupComplete;
7653
+ cleanupComplete = await this.#teardownFailedDispatchWorktrees(worktreeHandoffs, agentReleaseReason) && cleanupComplete;
7409
7654
  }
7410
7655
  else if (agents.length > 0) {
7411
- const failed = await this.#releaseAndTerminateAgents(agents, 'issue-abandoned', 'completion');
7656
+ const failed = await this.#releaseAndTerminateAgents(agents, agentReleaseReason, 'completion');
7412
7657
  cleanupComplete = failed.length === 0;
7413
7658
  }
7414
7659
  if (!cleanupComplete) {
@@ -7997,7 +8242,11 @@ export class FactoryLoop {
7997
8242
  return await this.#postAgentQuestionToGithub(record, question, 'no Slack dispatch thread exists');
7998
8243
  }
7999
8244
  try {
8000
- await this.#slack.reply(threadId, agentQuestionSlackText(record.issue, question, this.#config.slack.stakeholderUserIds));
8245
+ const issue = await this.#readIssue(record.issue.path);
8246
+ if (!issue) {
8247
+ throw new Error(`Unable to describe Slack question notification for unreadable issue ${record.issue.key}`);
8248
+ }
8249
+ await this.#slack.reply(threadId, agentQuestionSlackText(issue, question, this.#config.slack.stakeholderUserIds, slackNotificationRepos(record.decision)));
8001
8250
  this.#increment('agentQuestionsPostedToSlack');
8002
8251
  this.#recordSlackWritebackSuccess('agent-question');
8003
8252
  return true;
@@ -8099,10 +8348,13 @@ export class FactoryLoop {
8099
8348
  return await this.#deliverClarificationQuestionToGithub(key, claimed, `Slack writeback is degraded${this.#slackDegradedReason ? `: ${this.#slackDegradedReason}` : ''}`);
8100
8349
  }
8101
8350
  try {
8102
- await this.#slack.reply(claimed.threadId, agentQuestionSlackText(claimed.issue, {
8351
+ if (!claimedIssue) {
8352
+ throw new Error(`Unable to describe Slack question notification for unreadable issue ${claimed.issue.key}`);
8353
+ }
8354
+ await this.#slack.reply(claimed.threadId, agentQuestionSlackText(claimedIssue, {
8103
8355
  agentName: claimed.askerName,
8104
8356
  question: claimed.question,
8105
- }, this.#config.slack.stakeholderUserIds));
8357
+ }, this.#config.slack.stakeholderUserIds, slackNotificationRepos(claimed.decision)));
8106
8358
  const completed = await this.#state.completeClarificationQuestionDelivery(this.#workspaceId, key, this.#clarificationWakeOwner, this.#clock.now());
8107
8359
  if (!completed) {
8108
8360
  this.#increment('clarificationQuestionDeliveryOwnershipLost');
@@ -8300,7 +8552,11 @@ export class FactoryLoop {
8300
8552
  return;
8301
8553
  }
8302
8554
  try {
8303
- await this.#slack.reply(threadId, agentQuestionSlackText(record.issue, question, this.#config.slack.stakeholderUserIds));
8555
+ const issue = await this.#readIssue(record.issue.path);
8556
+ if (!issue) {
8557
+ throw new Error(`Unable to describe Slack question mirror for unreadable issue ${record.issue.key}`);
8558
+ }
8559
+ await this.#slack.reply(threadId, agentQuestionSlackText(issue, question, this.#config.slack.stakeholderUserIds, slackNotificationRepos(record.decision)));
8304
8560
  this.#increment('agentQuestionsMirroredToSlack');
8305
8561
  this.#recordSlackWritebackSuccess('agent-question-mirror');
8306
8562
  }
@@ -9395,7 +9651,7 @@ export class FactoryLoop {
9395
9651
  prMetaShowsMerged(snapshot) &&
9396
9652
  prSnapshotIssueMatchScore(snapshot, session.issue.key) >= 30) {
9397
9653
  await this.#state.clearBabysitterSession(this.#workspaceId, persistedKey);
9398
- await this.#advanceMergedPrToDone(snapshot, record);
9654
+ await this.#advanceMergedPrToDone(snapshot, session.repo, record);
9399
9655
  this.#increment('babysitterOwnershipRestoreMerged');
9400
9656
  this.#logger.info?.('[factory] completed restored lifecycle whose pull request was already merged', {
9401
9657
  issue: session.issue.key,
@@ -10516,7 +10772,7 @@ export class FactoryLoop {
10516
10772
  if (owned) {
10517
10773
  if (prMetaShowsMerged(snapshot)) {
10518
10774
  if (owned.record)
10519
- await this.#advanceMergedPrToDone(snapshot, owned.record);
10775
+ await this.#advanceMergedPrToDone(snapshot, repo, owned.record);
10520
10776
  else
10521
10777
  await this.#cancelBabysitterWake(owned.key);
10522
10778
  return;
@@ -10572,7 +10828,7 @@ export class FactoryLoop {
10572
10828
  return;
10573
10829
  }
10574
10830
  if (prMetaShowsMerged(snapshot)) {
10575
- await this.#advanceMergedPrToDone(snapshot, record);
10831
+ await this.#advanceMergedPrToDone(snapshot, repo, record);
10576
10832
  return;
10577
10833
  }
10578
10834
  if (!this.#config.babysitter.enabled) {
@@ -10630,9 +10886,15 @@ export class FactoryLoop {
10630
10886
  }
10631
10887
  return best?.record;
10632
10888
  }
10633
- async #advanceMergedPrToDone(snapshot, record) {
10889
+ async #advanceMergedPrToDone(snapshot, repo, record) {
10890
+ const mergedPullRequest = { repo, number: snapshot.number, url: snapshot.url };
10634
10891
  if (record) {
10635
- await this.#completeIssue(record, { targetState: 'done', runMergeGate: false, completionReason: 'pr-merged' });
10892
+ await this.#completeIssue(record, {
10893
+ targetState: 'done',
10894
+ runMergeGate: false,
10895
+ completionReason: 'pr-merged',
10896
+ mergedPullRequest,
10897
+ });
10636
10898
  return;
10637
10899
  }
10638
10900
  const issue = await this.#findMergeAdvanceIssueForPr(snapshot);
@@ -10670,11 +10932,13 @@ export class FactoryLoop {
10670
10932
  const channel = await this.#slackChannelDir();
10671
10933
  if (channel) {
10672
10934
  const systemOfRecord = githubIssue ? 'GitHub issue closed' : 'Linear state set to Done';
10935
+ const subject = slackIssueSubject(issue, [repo]);
10936
+ const pullRequest = slackPullRequestLink(mergedPullRequest);
10673
10937
  const root = await this.#slack.postThread({
10674
10938
  channel,
10675
- text: `${issue.key}: PR merged; ${systemOfRecord}.`,
10939
+ text: `${subject}\nPR merged · ${pullRequest} · ${systemOfRecord}`,
10676
10940
  });
10677
- await this.#slack.reply(root.threadId, `${issue.key}: ${systemOfRecord}.`);
10941
+ await this.#slack.reply(root.threadId, `${subject}\n${systemOfRecord} · ${pullRequest}`);
10678
10942
  this.#recordSlackWritebackSuccess('merge-done-thread');
10679
10943
  }
10680
10944
  }
@@ -11280,20 +11544,26 @@ export class FactoryLoop {
11280
11544
  }
11281
11545
  if (!await this.#saveDispatchLifecycle(record, 'writeback-applied'))
11282
11546
  return;
11283
- if (this.#slack && this.#config.slack && !await this.#shouldSkipSlackWriteback('completion-thread')) {
11547
+ if (issue && this.#slack && this.#config.slack && !await this.#shouldSkipSlackWriteback('completion-thread')) {
11284
11548
  try {
11285
11549
  const channel = await this.#slackChannelDir();
11286
11550
  if (channel) {
11287
11551
  const merged = opts.completionReason === 'pr-merged';
11288
11552
  const systemOfRecord = githubIssue ? 'GitHub status' : 'Linear state';
11289
- const completionText = merged
11290
- ? `${record.issue.key}: PR merged; ${systemOfRecord} set to ${statusLabel}.`
11291
- : `${record.issue.key}: factory agents completed${humanReview ? '; awaiting human review' : ''}.\nStatus: ${statusLabel}\nMerge policy: ${this.#config.mergePolicy}`;
11292
- const stateText = merged
11293
- ? `${record.issue.key}: ${systemOfRecord} set to ${statusLabel}.`
11294
- : humanReview
11295
- ? `${record.issue.key}: awaiting human review; ${systemOfRecord} set to ${statusLabel}.`
11296
- : `${record.issue.key}: ${systemOfRecord} set to ${statusLabel}.`;
11553
+ const pullRequests = await this.#slackPullRequestRefs(record, opts.mergedPullRequest ? [opts.mergedPullRequest] : []);
11554
+ const pullRequestLinks = pullRequests.map(slackPullRequestLink).join(' · ');
11555
+ const subject = slackIssueSubject(issue, slackNotificationRepos(record.decision));
11556
+ const stateResult = githubIssue && merged && !humanReview
11557
+ ? 'GitHub issue closed'
11558
+ : `${systemOfRecord} set to ${statusLabel}`;
11559
+ const completionText = [
11560
+ subject,
11561
+ merged
11562
+ ? `PR merged${pullRequestLinks ? ` · ${pullRequestLinks}` : ''} · ${stateResult}`
11563
+ : `Completed${humanReview ? ' · awaiting human review' : ''}${pullRequestLinks ? ` · ${pullRequestLinks}` : ''}`,
11564
+ ...(!merged ? [`Status: ${statusLabel} · Merge policy: ${this.#config.mergePolicy}`] : []),
11565
+ ].join('\n');
11566
+ const stateText = `${subject}\n${stateResult}${pullRequestLinks ? ` · ${pullRequestLinks}` : ''}`;
11297
11567
  const root = await this.#slack.postThread({
11298
11568
  channel,
11299
11569
  text: completionText,
@@ -11769,7 +12039,30 @@ export class FactoryLoop {
11769
12039
  this.#logger.warn?.('[factory] cleared invalid persisted Slack thread id', { issue: key });
11770
12040
  return undefined;
11771
12041
  }
11772
- async #ensureSlackDispatchThread(record, result) {
12042
+ async #slackPullRequestRefs(record, additional = []) {
12043
+ const lifecycle = await this.#state
12044
+ .getDispatchLifecycle(this.#workspaceId, issueKey(record.issue))
12045
+ .catch(() => undefined);
12046
+ const lifecycleRefs = publishedPullRequests(lifecycle).map((receipt) => ({
12047
+ repo: receipt.repo,
12048
+ number: receipt.number,
12049
+ url: receipt.url,
12050
+ }));
12051
+ const trackedRefs = [...record.agents.values()]
12052
+ .map((tracked) => tracked.spec.ownedPullRequest)
12053
+ .filter((ref) => Boolean(ref))
12054
+ .map((ref) => ({ repo: ref.repo, number: ref.number }));
12055
+ const babysitterRefs = [...this.#babysitterPr.entries()]
12056
+ .filter(([key]) => issueKey(this.#babysitterIssueRefs.get(key) ?? record.issue) === issueKey(record.issue))
12057
+ .map(([, ref]) => ({ repo: ref.repo, number: ref.prNumber }));
12058
+ return uniqueSlackPullRequestRefs([
12059
+ ...additional,
12060
+ ...lifecycleRefs,
12061
+ ...babysitterRefs,
12062
+ ...trackedRefs,
12063
+ ]);
12064
+ }
12065
+ async #ensureSlackDispatchThread(record, result, sourceIssue) {
11773
12066
  if (!this.#slack || !this.#config.slack || result.dryRun) {
11774
12067
  return;
11775
12068
  }
@@ -11805,7 +12098,7 @@ export class FactoryLoop {
11805
12098
  }
11806
12099
  return;
11807
12100
  }
11808
- const start = this.#postAndWatchSlackDispatchThread(record, result);
12101
+ const start = this.#postAndWatchSlackDispatchThread(record, result, sourceIssue);
11809
12102
  this.#slackWatcherStarts.set(key, start);
11810
12103
  try {
11811
12104
  await start;
@@ -11818,19 +12111,24 @@ export class FactoryLoop {
11818
12111
  this.#slackWatcherStarts.delete(key);
11819
12112
  }
11820
12113
  }
11821
- async #postAndWatchSlackDispatchThread(record, result) {
12114
+ async #postAndWatchSlackDispatchThread(record, result, sourceIssue) {
11822
12115
  if (!this.#slack || !this.#config.slack) {
11823
12116
  return;
11824
12117
  }
11825
- const previews = uniquePreviewReferences(result.previews ?? []);
12118
+ const issue = sourceIssue ?? await this.#readIssue(record.issue.path);
12119
+ if (!issue) {
12120
+ throw new Error(`Unable to describe Slack dispatch notification for unreadable issue ${record.issue.key}`);
12121
+ }
12122
+ const previews = uniquePreviewReferences(result.previews ?? this.#previewReferences.get(issueKey(record.issue)) ?? []);
12123
+ const repos = slackNotificationRepos(record.decision);
11826
12124
  const root = await this.#slack.postThread({
11827
12125
  channel: await this.#slackChannelDir() ?? this.#config.slack.channel,
11828
12126
  text: [
11829
- `${record.issue.key}: factory agents dispatched.`,
11830
- `State: ${result.stateId ?? 'dispatching'} · Agents: ${result.agents.map((agent) => agent.name).join(', ') || 'none'}`,
12127
+ slackIssueSubject(issue, repos),
12128
+ `Dispatched · ${result.agents.map((agent) => agent.name).join(', ') || 'no agents'} · Repos: ${slackRepoList(repos)}`,
11831
12129
  ...(previews.length > 0
11832
12130
  ? [previews.map((preview) => `Live preview (${preview.repo}, tailnet access required): ${preview.url}`).join(' · ')]
11833
- : []),
12131
+ : [`State: ${result.stateId ?? 'dispatching'}`]),
11834
12132
  ].join('\n'),
11835
12133
  });
11836
12134
  await this.#state.setSlackThread(this.#workspaceId, issueKey(record.issue), root.threadId);
@@ -12193,13 +12491,13 @@ export class FactoryLoop {
12193
12491
  .filter((part) => Boolean(part))
12194
12492
  .join(' ');
12195
12493
  const replyInstruction = source?.url
12196
- ? `Reply on the GitHub issue so Factory can resume: ${source.url}`
12494
+ ? 'Reply on the linked GitHub issue so Factory can resume.'
12197
12495
  : 'Reply on the source GitHub issue so Factory can resume.';
12198
12496
  const root = await this.#slack.postThread({
12199
12497
  channel: await this.#slackChannelDir() ?? this.#config.slack.channel,
12200
12498
  text: [
12201
- `${audience ? `${audience} ` : ''}${decision.issue.key}: factory triage escalation for ${issue.title}`,
12202
- `Reason: ${reason}`,
12499
+ `${audience ? `${audience} ` : ''}${slackIssueSubject(issue, slackNotificationRepos(decision))}`,
12500
+ `Triage blocked · Reason: ${reason}`,
12203
12501
  `Question: ${triageEscalationQuestion(decision, issue)} ${replyInstruction}`,
12204
12502
  ].join('\n'),
12205
12503
  });
@@ -12323,12 +12621,15 @@ export class FactoryLoop {
12323
12621
  return;
12324
12622
  }
12325
12623
  const issue = await this.#readIssue(decision.issue.path);
12624
+ if (!issue) {
12625
+ throw new Error(`Unable to describe Slack triage escalation for unreadable issue ${decision.issue.key}`);
12626
+ }
12326
12627
  const stakeholderMentions = slackMentions(this.#config.slack.stakeholderUserIds);
12327
12628
  const root = await this.#slack.postThread({
12328
12629
  channel: await this.#slackChannelDir() ?? this.#config.slack.channel,
12329
12630
  text: [
12330
- `${stakeholderMentions ? `${stakeholderMentions} ` : ''}${decision.issue.key}: factory triage escalation for ${issue?.title ?? decision.issue.key}`,
12331
- `Reason: ${reason}`,
12631
+ `${stakeholderMentions ? `${stakeholderMentions} ` : ''}${slackIssueSubject(issue, slackNotificationRepos(decision))}`,
12632
+ `Triage blocked · Reason: ${reason}`,
12332
12633
  `Question: ${triageEscalationQuestion(decision, issue)}`,
12333
12634
  ].join('\n'),
12334
12635
  });
@@ -12671,7 +12972,11 @@ export class FactoryLoop {
12671
12972
  waitingAgeMs,
12672
12973
  });
12673
12974
  try {
12674
- await this.#slack.reply(waiting.threadId, clarificationStaleSlackText(escalated, this.#config.slack.stakeholderUserIds));
12975
+ const issue = await this.#readIssue(escalated.issue.path);
12976
+ if (!issue) {
12977
+ throw new Error(`Unable to describe stale Slack clarification for unreadable issue ${escalated.issue.key}`);
12978
+ }
12979
+ await this.#slack.reply(waiting.threadId, clarificationStaleSlackText(escalated, issue, this.#config.slack.stakeholderUserIds));
12675
12980
  const completed = await this.#state.completeClarificationEscalation(this.#workspaceId, key, this.#clarificationWakeOwner, this.#clock.now());
12676
12981
  if (!completed) {
12677
12982
  this.#increment('clarificationEscalationOwnershipLost');
@@ -12935,6 +13240,9 @@ export class FactoryLoop {
12935
13240
  this.#increment('clarificationWakesQueuedForCapacity');
12936
13241
  return;
12937
13242
  }
13243
+ // A human-answer wake starts a new agent-hold generation. Time spent
13244
+ // parked with the previous team released must not consume its deadline.
13245
+ record.heldSinceAtMs = undefined;
12938
13246
  if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
12939
13247
  batch.complete(waiting.issue);
12940
13248
  await this.#state.releaseClarificationWake(this.#workspaceId, key, this.#clarificationWakeOwner);
@@ -12964,8 +13272,10 @@ export class FactoryLoop {
12964
13272
  }
12965
13273
  : await this.#resumeOrColdStartClarificationAgent(parked.name, tracked, waiting);
12966
13274
  const invocationId = batch.invocationIdFor(record.issue, tracked.spec);
13275
+ record.heldSinceAtMs ??= this.#clock.now();
12967
13276
  batch.recordSpawn(record, tracked.spec, invocationId, result);
12968
13277
  await this.#saveDispatchLifecycle(record, 'dispatching');
13278
+ this.#scheduleHeldAgentDeadline(record);
12969
13279
  const live = record.agents.get(result.name);
12970
13280
  if (live) {
12971
13281
  resumed.push([result.name, live]);
@@ -13784,6 +14094,82 @@ function dispatchIssueUrl(issue) {
13784
14094
  ?? stringValue(source?.url)
13785
14095
  ?? issue.path;
13786
14096
  }
14097
+ const SLACK_ISSUE_TITLE_MAX_LENGTH = 120;
14098
+ function slackEscapeText(value) {
14099
+ return value
14100
+ .replace(/&/gu, '&amp;')
14101
+ .replace(/</gu, '&lt;')
14102
+ .replace(/>/gu, '&gt;');
14103
+ }
14104
+ function slackLinkUrl(value) {
14105
+ try {
14106
+ const url = new URL(value);
14107
+ if (url.protocol !== 'https:' && url.protocol !== 'http:')
14108
+ return undefined;
14109
+ return url.toString()
14110
+ .replace(/\|/gu, '%7C')
14111
+ .replace(/</gu, '%3C')
14112
+ .replace(/>/gu, '%3E');
14113
+ }
14114
+ catch {
14115
+ return undefined;
14116
+ }
14117
+ }
14118
+ function truncateSlackIssueTitle(title) {
14119
+ const normalized = title.replace(/\s+/gu, ' ').trim() || 'Untitled issue';
14120
+ const characters = Array.from(normalized);
14121
+ if (characters.length <= SLACK_ISSUE_TITLE_MAX_LENGTH)
14122
+ return normalized;
14123
+ return `${characters.slice(0, SLACK_ISSUE_TITLE_MAX_LENGTH - 1).join('').trimEnd()}…`;
14124
+ }
14125
+ function slackRepoName(repo) {
14126
+ const normalized = repo.trim().replace(/^\/+|\/+$/gu, '');
14127
+ return normalized.slice(normalized.lastIndexOf('/') + 1) || normalized || 'unknown repo';
14128
+ }
14129
+ function slackNotificationRepos(decision) {
14130
+ const repos = [
14131
+ ...decision.routes.map((route) => route.repo),
14132
+ ...decision.implementers.map((implementer) => implementer.repo),
14133
+ ];
14134
+ return [...new Map(repos
14135
+ .filter((repo) => Boolean(repo.trim()))
14136
+ .map((repo) => [repo.trim().toLowerCase(), repo.trim()])).values()];
14137
+ }
14138
+ function slackRepoList(repos) {
14139
+ const names = [...new Set(repos.map(slackRepoName))];
14140
+ return slackEscapeText(names.join(', ') || 'unknown repo');
14141
+ }
14142
+ function slackIssueSubject(issue, repos = []) {
14143
+ const githubSource = githubIssueSourceRef(issue);
14144
+ const githubPath = githubIssuePathParts(issue.path);
14145
+ const githubIdentity = githubSource ?? githubPath;
14146
+ const payload = wrappedPayload(issue.raw);
14147
+ const source = asRecord(payload.source);
14148
+ const fallbackIssueUrl = githubIdentity
14149
+ ? `https://github.com/${githubIdentity.owner}/${githubIdentity.repo}/issues/${githubIdentity.number}`
14150
+ : `https://linear.app/issue/${encodeURIComponent(issue.key)}`;
14151
+ const issueUrl = slackLinkUrl(githubSource?.url
14152
+ ?? stringValue(payload.url)
14153
+ ?? stringValue(payload.html_url)
14154
+ ?? stringValue(source?.url)
14155
+ ?? fallbackIssueUrl) ?? fallbackIssueUrl;
14156
+ const repoNames = [...new Set(repos.map(slackRepoName))];
14157
+ const label = githubIdentity
14158
+ ? `${githubIdentity.repo}#${githubIdentity.number}`
14159
+ : `${repoNames.join(', ') || 'unknown repo'} · ${issue.key}`;
14160
+ return `<${issueUrl}|${slackEscapeText(label)}> — ${slackEscapeText(truncateSlackIssueTitle(issue.title))}`;
14161
+ }
14162
+ function slackPullRequestLink(pullRequest) {
14163
+ const fallbackUrl = `https://github.com/${pullRequest.repo}/pull/${pullRequest.number}`;
14164
+ const url = slackLinkUrl(pullRequest.url ?? fallbackUrl) ?? fallbackUrl;
14165
+ const label = `${slackRepoName(pullRequest.repo)}#${pullRequest.number}`;
14166
+ return `<${url}|${slackEscapeText(label)}>`;
14167
+ }
14168
+ function uniqueSlackPullRequestRefs(refs) {
14169
+ return [...new Map(refs
14170
+ .filter((ref) => Boolean(ref.repo.trim()) && Number.isSafeInteger(ref.number) && ref.number > 0)
14171
+ .map((ref) => [`${ref.repo.trim().toLowerCase()}#${ref.number}`, { ...ref, repo: ref.repo.trim() }])).values()];
14172
+ }
13787
14173
  function ticketDispatchNotificationText(payload) {
13788
14174
  return `${payload.summary}\n${JSON.stringify(payload)}`;
13789
14175
  }
@@ -15722,6 +16108,28 @@ const durableBabysitterTrackedAgent = (session, capability = 'spawn:claude') =>
15722
16108
  });
15723
16109
  const isTerminalDispatchLifecycle = (lifecycle) => lifecycle.phase === 'complete' || lifecycle.phase === 'abandoned';
15724
16110
  const isTerminalDispatchPhase = (phase) => phase === 'complete' || phase === 'abandoned';
16111
+ const heldAgentsForRecord = (record, nowMs, holdTimeoutMs, terminalState) => {
16112
+ if (record.dryRun || record.heldSinceAtMs === undefined)
16113
+ return [];
16114
+ const heldSinceAtMs = record.heldSinceAtMs;
16115
+ const holdDeadlineAtMs = heldSinceAtMs + holdTimeoutMs;
16116
+ const heldForMs = Math.max(0, nowMs - heldSinceAtMs);
16117
+ return [...record.agents]
16118
+ .filter(([, tracked]) => Boolean(tracked.result))
16119
+ .map(([name, tracked]) => ({
16120
+ name,
16121
+ role: tracked.spec.role,
16122
+ issue: { ...record.issue },
16123
+ ...(record.lifecyclePhase ? { lifecyclePhase: record.lifecyclePhase } : {}),
16124
+ waitingForTerminalState: terminalState,
16125
+ heldSince: new Date(heldSinceAtMs).toISOString(),
16126
+ heldSinceAtMs,
16127
+ heldForMs,
16128
+ holdDeadline: new Date(holdDeadlineAtMs).toISOString(),
16129
+ holdDeadlineAtMs,
16130
+ pastDeadline: nowMs >= holdDeadlineAtMs,
16131
+ }));
16132
+ };
15725
16133
  const costUsageGroupId = (runId, tracked) => JSON.stringify([runId, tracked.spec.invocationId ?? tracked.spec.name]);
15726
16134
  const costEntryId = (groupId, model) => JSON.stringify([groupId, model]);
15727
16135
  const publishedPullRequests = (lifecycle) => {
@@ -15767,6 +16175,7 @@ const lifecycleFromInFlightRecord = (record, runId, phase, updatedAtMs, pullRequ
15767
16175
  ...(pullRequest ? { pullRequest: { ...pullRequest } } : {}),
15768
16176
  ...(releaseReason ? { releaseReason } : {}),
15769
16177
  ...(cost ? { cost: structuredClone(cost) } : {}),
16178
+ ...(record.heldSinceAtMs !== undefined ? { heldSinceAtMs: record.heldSinceAtMs } : {}),
15770
16179
  updatedAtMs,
15771
16180
  });
15772
16181
  const inFlightRecordFromLifecycle = (lifecycle) => ({
@@ -15777,6 +16186,10 @@ const inFlightRecordFromLifecycle = (lifecycle) => ({
15777
16186
  invocationIds: new Set(lifecycle.invocationIds),
15778
16187
  result: lifecycle.result ? structuredClone(lifecycle.result) : undefined,
15779
16188
  ...(lifecycle.dispatchClaim ? { dispatchClaim: { ...lifecycle.dispatchClaim } } : {}),
16189
+ heldSinceAtMs: lifecycle.heldSinceAtMs ?? (lifecycle.agents.some((agent) => agent.releasedAtMs === undefined)
16190
+ ? lifecycle.updatedAtMs
16191
+ : undefined),
16192
+ lifecyclePhase: lifecycle.phase,
15780
16193
  });
15781
16194
  const dispatchResultFromLifecycle = (lifecycle) => lifecycle.result ? structuredClone(lifecycle.result) : {
15782
16195
  issue: { ...lifecycle.issue },
@@ -15971,14 +16384,14 @@ const slackUserIdMatchingIdentity = (payload, identity) => {
15971
16384
  ? userId
15972
16385
  : undefined;
15973
16386
  };
15974
- const agentQuestionSlackText = (issue, question, stakeholderUserIds = []) => [
15975
- slackMentions(stakeholderUserIds),
15976
- `${issue.key}: ${question.agentName} needs input.`,
16387
+ const agentQuestionSlackText = (issue, question, stakeholderUserIds = [], repos = []) => [
16388
+ `${slackMentions(stakeholderUserIds) ?? ''} ${slackIssueSubject(issue, repos)}`.trim(),
16389
+ `${question.agentName} needs input.`,
15977
16390
  `Question: ${question.question}`,
15978
16391
  ].filter((line) => Boolean(line)).join('\n');
15979
- const clarificationStaleSlackText = (waiting, stakeholderUserIds = []) => [
15980
- slackMentions(stakeholderUserIds),
15981
- `${waiting.issue.key} has been parked for seven days without a reply.`,
16392
+ const clarificationStaleSlackText = (waiting, issue, stakeholderUserIds = []) => [
16393
+ `${slackMentions(stakeholderUserIds) ?? ''} ${slackIssueSubject(issue, slackNotificationRepos(waiting.decision))}`.trim(),
16394
+ 'This issue has been parked for seven days without a reply.',
15982
16395
  `Question from ${waiting.askerName}: ${waiting.question} Reply in this thread to wake the saved agent team, or move the issue out of Agent Implementing to cancel the wake.`,
15983
16396
  ].filter((line) => Boolean(line)).join('\n');
15984
16397
  const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');