owenloop 0.5.8 → 0.5.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/CHANGELOG.md +69 -0
  2. package/README.md +81 -39
  3. package/bin/owenloop.mjs +16 -1
  4. package/dist/packages/work/src/agent/loop.d.ts +23 -8
  5. package/dist/packages/work/src/agent/loop.js +55 -6
  6. package/dist/packages/work/src/bundle/pull.d.ts +21 -0
  7. package/dist/packages/work/src/bundle/pull.js +118 -0
  8. package/dist/packages/work/src/consumed-verifier.js +6 -17
  9. package/dist/packages/work/src/exec/instructions.d.ts +4 -1
  10. package/dist/packages/work/src/exec/instructions.js +3 -0
  11. package/dist/packages/work/src/exec/loop.d.ts +2 -2
  12. package/dist/packages/work/src/exec/loop.js +273 -46
  13. package/dist/packages/work/src/exec/receipt.d.ts +7 -7
  14. package/dist/packages/work/src/exec/receipt.js +6 -6
  15. package/dist/packages/work/src/harness/claude.js +116 -0
  16. package/dist/packages/work/src/harness/codex.d.ts +8 -0
  17. package/dist/packages/work/src/harness/codex.js +27 -1
  18. package/dist/packages/work/src/harness/contract.d.ts +9 -0
  19. package/dist/packages/work/src/hub/client.d.ts +19 -4
  20. package/dist/packages/work/src/hub/client.js +7 -3
  21. package/dist/packages/work/src/hub/types.d.ts +84 -0
  22. package/dist/packages/work/src/main.js +1 -0
  23. package/dist/packages/work/src/roles/agent-run.js +57 -32
  24. package/dist/packages/work/src/roles/exec.js +27 -8
  25. package/dist/packages/work/src/roles/settings.js +2 -0
  26. package/dist/packages/work/src/roles/shift.js +14 -2
  27. package/dist/packages/work/src/roles/util.d.ts +1 -0
  28. package/dist/packages/work/src/roles/util.js +10 -0
  29. package/dist/packages/work/src/settings/hub-roster-cache.d.ts +68 -0
  30. package/dist/packages/work/src/settings/hub-roster-cache.js +370 -0
  31. package/dist/packages/work/src/settings/roster.d.ts +84 -0
  32. package/dist/packages/work/src/settings/roster.js +364 -5
  33. package/dist/packages/work/src/settings/serving.d.ts +17 -0
  34. package/dist/packages/work/src/settings/serving.js +79 -0
  35. package/dist/packages/work/src/settings/settings.d.ts +14 -1
  36. package/dist/packages/work/src/settings/settings.js +14 -0
  37. package/dist/packages/work/src/shift/loop.d.ts +108 -5
  38. package/dist/packages/work/src/shift/loop.js +416 -54
  39. package/dist/packages/work/src/shift/protocol.d.ts +7 -5
  40. package/dist/packages/work/src/shift/runtime.d.ts +18 -0
  41. package/dist/packages/work/src/shift/runtime.js +104 -5
  42. package/dist/packages/work/src/shift/server.js +2 -0
  43. package/dist/packages/work/src/shift/spawn.d.ts +17 -2
  44. package/dist/packages/work/src/shift/spawn.js +16 -13
  45. package/dist/packages/work/src/shift/state.d.ts +29 -2
  46. package/dist/packages/work/src/shift/state.js +45 -15
  47. package/dist/packages/work/src/submit-proof.d.ts +11 -8
  48. package/dist/packages/work/src/submit-proof.js +11 -8
  49. package/dist/packages/work/src/usage.d.ts +1 -1
  50. package/dist/packages/work/src/usage.js +7 -1
  51. package/dist/packages/work/src/util/modifier-init.d.ts +7 -0
  52. package/dist/packages/work/src/util/modifier-init.js +73 -0
  53. package/dist/src/bundle/tar.d.ts +9 -4
  54. package/dist/src/bundle/tar.js +9 -4
  55. package/dist/src/capabilities.d.ts +76 -6
  56. package/dist/src/capabilities.js +49 -6
  57. package/dist/src/capability-mapping-client.d.ts +29 -0
  58. package/dist/src/capability-mapping-client.js +160 -0
  59. package/dist/src/cli.d.ts +16 -3
  60. package/dist/src/cli.js +1925 -281
  61. package/dist/src/config-dir.d.ts +1 -1
  62. package/dist/src/config-dir.js +1 -1
  63. package/dist/src/crypto/index.d.ts +1 -1
  64. package/dist/src/crypto/index.js +1 -1
  65. package/dist/src/crypto/org-root.d.ts +30 -12
  66. package/dist/src/crypto/org-root.js +162 -14
  67. package/dist/src/crypto/records.d.ts +2 -0
  68. package/dist/src/crypto/records.js +2 -0
  69. package/dist/src/defs.d.ts +38 -2
  70. package/dist/src/defs.js +593 -86
  71. package/dist/src/engine.d.ts +103 -25
  72. package/dist/src/engine.js +331 -79
  73. package/dist/src/hub.d.ts +77 -0
  74. package/dist/src/hub.js +162 -0
  75. package/dist/src/implements.d.ts +56 -0
  76. package/dist/src/implements.js +790 -0
  77. package/dist/src/index.d.ts +11 -9
  78. package/dist/src/index.js +6 -5
  79. package/dist/src/mcp/serve.d.ts +22 -0
  80. package/dist/src/mcp/serve.js +330 -41
  81. package/dist/src/mcp/server.d.ts +10 -7
  82. package/dist/src/mcp/server.js +53 -8
  83. package/dist/src/model.d.ts +48 -10
  84. package/dist/src/model.js +252 -99
  85. package/dist/src/order-resolver.d.ts +1 -0
  86. package/dist/src/order-resolver.js +8 -1
  87. package/dist/src/schemas/index.d.ts +11 -0
  88. package/dist/src/schemas/order.v1.schema.json +7 -1
  89. package/dist/src/store/def-source.d.ts +19 -0
  90. package/dist/src/store/def-source.js +87 -17
  91. package/dist/src/store/gc-recovery.d.ts +46 -0
  92. package/dist/src/store/gc-recovery.js +258 -0
  93. package/dist/src/store/gc.d.ts +94 -0
  94. package/dist/src/store/gc.js +562 -0
  95. package/dist/src/store/index.d.ts +3 -1
  96. package/dist/src/store/index.js +2 -0
  97. package/dist/src/store/install.d.ts +34 -5
  98. package/dist/src/store/install.js +73 -11
  99. package/dist/src/store/pre-commit-verifier.js +42 -2
  100. package/dist/src/store/snapshot-guard.d.ts +18 -0
  101. package/dist/src/store/snapshot-guard.js +94 -0
  102. package/dist/src/store.d.ts +69 -4
  103. package/dist/src/store.js +254 -16
  104. package/dist/src/types.d.ts +101 -20
  105. package/dist/src/util.js +4 -1
  106. package/docs/authoring.md +235 -10
  107. package/docs/bundles.md +45 -2
  108. package/docs/cli.md +437 -55
  109. package/docs/crypto.md +32 -10
  110. package/docs/design.md +177 -47
  111. package/docs/evals/mcp-charter-baseline.json +428 -0
  112. package/docs/mcp-charter-eval.md +124 -0
  113. package/docs/routing.md +189 -34
  114. package/docs/shift-logs.md +38 -9
  115. package/docs/wire-contracts.md +78 -14
  116. package/examples/workflows/compile-dev-playbook.yaml +1 -1
  117. package/examples/workflows/delivery.yaml +31 -14
  118. package/examples/workflows/ship.yaml +38 -4
  119. package/package.json +2 -1
  120. package/plugins/claude-code/plugin/.claude-plugin/plugin.json +1 -1
  121. package/plugins/claude-code/plugin/.mcp.json +1 -1
  122. package/plugins/claude-code/plugin/skills/author/SKILL.md +22 -0
  123. package/plugins/claude-code/plugin/skills/ephemeral/SKILL.md +95 -0
  124. package/plugins/claude-code/plugin/skills/graduate/SKILL.md +156 -0
  125. package/plugins/claude-code/plugin/skills/plan/SKILL.md +236 -0
  126. package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +1 -1
  127. package/plugins/codex/plugins/owenloop/.mcp.json +1 -1
  128. package/plugins/codex/plugins/owenloop/skills/author/SKILL.md +22 -0
  129. package/plugins/codex/plugins/owenloop/skills/ephemeral/SKILL.md +95 -0
  130. package/plugins/codex/plugins/owenloop/skills/graduate/SKILL.md +156 -0
  131. package/plugins/codex/plugins/owenloop/skills/plan/SKILL.md +236 -0
@@ -27,9 +27,11 @@
27
27
  * liveInFlight counts every live child record, each one pid-probed (see
28
28
  * state.ts). When `k <= 0` the loop skips `whats_next` but remembers that the
29
29
  * adopted wake still needs a sweep. Orders already returned by `whats_next`
30
- * are already claimed, so over-cap orders enter an in-memory queue and dispatch
31
- * as soon as local child capacity frees; waiting for another hub wake would
32
- * strand those claims until pickup expiry.
30
+ * are already claimed. Orders this shift cannot dispatch are released back to
31
+ * the hub immediately, so another daemon can take them; `localQueueHoldMs`
32
+ * optionally retains them locally for a bounded window instead. The agent lane
33
+ * is additionally bounded by `cap − execReserve`, preserving room for an exec
34
+ * order while agents are saturated.
33
35
  *
34
36
  * RESILIENCE: presence/wake/whats_next failures are logged and never kill the
35
37
  * loop. `stop()` flips a flag checked between awaits; the in-flight sweep
@@ -53,21 +55,73 @@
53
55
  */
54
56
  import { join } from 'node:path';
55
57
  import { performance } from 'node:perf_hooks';
56
- import { acquireFileLockSync, releaseFileLock } from "../../../../src/lock.js";
58
+ import { acquireFileLockSync, FileLockTimeoutError, releaseFileLock, } from "../../../../src/lock.js";
57
59
  import { readDispatchBundle } from "../bundle/cache.js";
58
60
  import { HubError } from "../hub/types.js";
59
61
  import { isCommandStep, resolveCommandRouting } from "./routing.js";
60
- import { cancelReservedChild, finalizeChildReservation, reconcileInFlight, reserveChild, startReservedChild, removeChildRecord, ensureStateDir, } from "./state.js";
62
+ import { cancelReservedChild, finalizeChildReservation, reconcileInFlight, reserveChild, startReservedChild, removeChildRecord, settleChildGate, ensureStateDir, } from "./state.js";
61
63
  import { DEFAULT_WORK_DIR_TTL_MS, sweepWorkDirs as sweepWorkDirsImpl } from "../agent/workdir.js";
64
+ import { withHubRosterSyncTimeout } from "../settings/hub-roster-cache.js";
62
65
  import { sessionsPath } from "../harness/session-store.js";
63
66
  import { stampShiftEvent, } from "./protocol.js";
67
+ /** Run a state mutation while holding the lock that serializes dispatch state. */
68
+ export function withDispatchLock(stateDir, options = {}, fn) {
69
+ const dispatchLock = acquireFileLockSync(join(stateDir, '.dispatch.lock'), {
70
+ ...options.dispatchLockOptions,
71
+ waitMs: options.waitMs ?? 30_000,
72
+ label: options.label ?? 'owenloop Shift dispatch-state removal',
73
+ });
74
+ try {
75
+ return fn();
76
+ }
77
+ finally {
78
+ releaseFileLock(dispatchLock);
79
+ }
80
+ }
81
+ /**
82
+ * Remove one child record while holding the same lock that serializes durable
83
+ * reservation creation. The record is deliberately re-read by
84
+ * `removeChildRecord` only after acquisition, so a stale observer cannot erase
85
+ * a newer dispatch for the same run.
86
+ */
87
+ export function removeChildRecordUnderDispatchLock(stateDir, run, recordOptions, options = {}) {
88
+ return withDispatchLock(stateDir, options, () => removeChildRecord(stateDir, run, recordOptions));
89
+ }
90
+ /**
91
+ * Reconciliation callbacks for every unlocked Shift daemon path. Keep these
92
+ * together with the lock-held removal primitive so startup and the poll loop
93
+ * cannot silently diverge.
94
+ */
95
+ export function createLockedRemovalCallbacks(stateDir, options = {}) {
96
+ return {
97
+ removeAbandonedReservation: (reservation) => withDispatchLock(stateDir, options, () => cancelReservedChild(stateDir, reservation)),
98
+ removeDeadChild: (record) => withDispatchLock(stateDir, options, () => removeChildRecord(stateDir, record.run, { pid: record.pid })),
99
+ settleLiveChild: (record) => withDispatchLock(stateDir, options, () => settleChildGate(stateDir, record)),
100
+ };
101
+ }
64
102
  /** The `maxConcurrentAgents` fallback when the option is absent. */
65
103
  const DEFAULT_MAX_AGENTS = 4;
104
+ /** The `execReserve` fallback when the option is absent. */
105
+ const DEFAULT_EXEC_RESERVE = 1;
106
+ /** The `localQueueHoldMs` fallback when the option is absent. */
107
+ const DEFAULT_LOCAL_QUEUE_HOLD_MS = 0;
108
+ /** The hub reaps a never-contacted claim after this long. Every client-side
109
+ * re-offer latency must stay strictly under it. */
110
+ export const HUB_PICKUP_WINDOW_MS = 120_000;
111
+ /** Headroom for a detached child to make first contact after dispatch. */
112
+ const CHILD_FIRST_CONTACT_MARGIN_MS = 30_000;
66
113
  /**
67
114
  * The hub reaps a never-contacted claim after 120 seconds. Stop local queueing
68
115
  * after 90 seconds so a detached child retains 30 seconds to make first contact.
69
116
  */
70
- export const MAX_PENDING_CANDIDATE_AGE_MS = 90_000;
117
+ export const MAX_PENDING_CANDIDATE_AGE_MS = HUB_PICKUP_WINDOW_MS - CHILD_FIRST_CONTACT_MARGIN_MS;
118
+ /**
119
+ * A capacity-refused claim is returned to the hub immediately, so sibling
120
+ * shifts can take it. This short, fixed monotonic window stops this shift from
121
+ * claiming that same stable step again on every busy wake while its relevant
122
+ * capacity is still unavailable.
123
+ */
124
+ const CAPACITY_RELEASE_COOLDOWN_MS = 30_000;
71
125
  /**
72
126
  * THE PER-STEP FAILURE BRAKE.
73
127
  *
@@ -101,11 +155,11 @@ export const MAX_PENDING_CANDIDATE_AGE_MS = 90_000;
101
155
  * It is a rate limit, never a ban. A step that has not failed is dispatched
102
156
  * with zero delay, however many orders arrive for it at once.
103
157
  */
104
- const STEP_BRAKE_DELAYS_MS = [2_000, 8_000, 30_000, 120_000, 300_000];
158
+ export const STEP_BRAKE_DELAYS_MS = [2_000, 8_000, 30_000, 60_000, 90_000];
105
159
  /**
106
160
  * Treat a failure this long after the previous one as the start of a NEW
107
161
  * streak, resetting the count to zero first. Comfortably longer than the
108
- * longest delay in the table (300s), so a step failing at maximum backoff never
162
+ * longest delay in the table (90s), so a step failing at maximum backoff never
109
163
  * decays out of it, while an isolated transient long after an old streak starts
110
164
  * again at the shortest delay instead of inheriting a stale penalty.
111
165
  */
@@ -121,7 +175,7 @@ const STEP_BRAKE_FORGET_MS = 1_800_000;
121
175
  * worker failures. `count` is at least 1 at every call site (this is reached
122
176
  * only from the failure path), so the FIRST failure already costs 2s; past the
123
177
  * end of the table the last entry repeats, so a permanently-failing step
124
- * settles at one dispatch per five minutes instead of one per poll.
178
+ * settles at one dispatch per 90 seconds instead of one per poll.
125
179
  */
126
180
  function stepBrakeDelayMs(count) {
127
181
  const index = Math.min(count, STEP_BRAKE_DELAYS_MS.length) - 1;
@@ -146,6 +200,23 @@ function isNonServableRace(e) {
146
200
  export function createShiftLoop(opts) {
147
201
  let stopped = false;
148
202
  let cap = opts.cap;
203
+ /**
204
+ * The agent lane's live budget. Recomputed per call because `setCap`
205
+ * rewrites `cap` at runtime.
206
+ */
207
+ function agentLane() {
208
+ const requested = opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS;
209
+ const reserve = Math.min(opts.execReserve ?? DEFAULT_EXEC_RESERVE, Math.max(0, cap - 1));
210
+ return {
211
+ ceiling: Math.max(0, Math.min(requested, cap - reserve)),
212
+ requested,
213
+ reserve,
214
+ };
215
+ }
216
+ /** Effective local hold, capped below the hub pickup window. */
217
+ function localQueueHoldMs() {
218
+ return Math.min(Math.max(0, opts.localQueueHoldMs ?? DEFAULT_LOCAL_QUEUE_HOLD_MS), MAX_PENDING_CANDIDATE_AGE_MS);
219
+ }
149
220
  const isAlive = opts.isAlive;
150
221
  const monotonicNow = opts.monotonicNow ?? performance.now.bind(performance);
151
222
  // Live shift identity (MCP `clock_in`, D3-D7 of the plan). Seeded from opts,
@@ -153,9 +224,32 @@ export function createShiftLoop(opts) {
153
224
  // nor a caller of getShift/setShift can mutate the other's state afterward.
154
225
  let shiftName = opts.name;
155
226
  let serveCrews = [...opts.serveCrews];
227
+ let serveCapabilities = [];
228
+ let initial = true;
229
+ function refreshServeCapabilities() {
230
+ if (opts.computeServeCapabilities === undefined)
231
+ return;
232
+ let next;
233
+ try {
234
+ next = [...opts.computeServeCapabilities(serveCrews)];
235
+ }
236
+ catch {
237
+ return; // Keep the last known-good advertisement on a transient failure.
238
+ }
239
+ const changed = next.length !== serveCapabilities.length ||
240
+ next.some((capability, index) => capability !== serveCapabilities[index]);
241
+ serveCapabilities = next;
242
+ if (changed && !initial)
243
+ opts.out(`serving ${next.length === 0 ? '(nothing)' : next.join(', ')}`);
244
+ }
245
+ refreshServeCapabilities();
246
+ initial = false;
156
247
  // Park state persists across `iterate()` calls (the MCP park reuses it).
157
248
  let cursor;
158
249
  let lastPresence = Number.NEGATIVE_INFINITY;
250
+ // Runtime performs the required shift-start refresh before creating this
251
+ // loop. Start this cadence now so it is not immediately fetched twice.
252
+ let lastRosterSync = monotonicNow();
159
253
  let attendedAt;
160
254
  /**
161
255
  * Orders returned by `whats_next` are already claimed. If the hub returns more
@@ -168,6 +262,18 @@ export function createShiftLoop(opts) {
168
262
  /** A changed wake whose sweep was skipped or failed must be retried after the
169
263
  * cursor is adopted; otherwise the next unchanged wake hides that work. */
170
264
  let sweepOwed = false;
265
+ /** Earliest monotonic time at which a braked step becomes dispatchable again.
266
+ * Drives a sweep without a hub cursor move, so a brake never waits forever. */
267
+ let brakeSweepDueAt;
268
+ /**
269
+ * Recently capacity-released stable orders. This is deliberately separate
270
+ * from `stepBrake`: a worker failure leaves its claim to lapse, while a
271
+ * capacity refusal hands the claim back to the hub and needs a reason-aware
272
+ * targeted-poll damper.
273
+ */
274
+ const capacityCooldown = new Map();
275
+ /** Earliest fixed cooldown deadline, used to re-admit an unchanged wake. */
276
+ let capacityCooldownDueAt;
171
277
  /**
172
278
  * Has the current at-capacity EPISODE already produced a `capacity` event?
173
279
  *
@@ -182,6 +288,73 @@ export function createShiftLoop(opts) {
182
288
  /** Bumped whenever a ping starts or is forced due, so a ping that completes
183
289
  * after a force does not stamp the cadence over that force. */
184
290
  let presenceGeneration = 0;
291
+ function refreshCapacityCooldownDueAt() {
292
+ let earliest;
293
+ for (const entry of capacityCooldown.values()) {
294
+ earliest = earliest === undefined ? entry.retryAt : Math.min(earliest, entry.retryAt);
295
+ }
296
+ capacityCooldownDueAt = earliest;
297
+ }
298
+ /** Remove all expired entries; a real post-expiry retry can arm a new window. */
299
+ function pruneExpiredCapacityCooldown(now = monotonicNow()) {
300
+ let changed = false;
301
+ for (const [key, entry] of capacityCooldown) {
302
+ if (entry.retryAt <= now) {
303
+ capacityCooldown.delete(key);
304
+ changed = true;
305
+ }
306
+ }
307
+ if (changed)
308
+ refreshCapacityCooldownDueAt();
309
+ }
310
+ function armCapacityCooldown(workflow, order, reason) {
311
+ const now = monotonicNow();
312
+ pruneExpiredCapacityCooldown(now);
313
+ const key = stepBrakeKey(workflow, order.step, order.key);
314
+ // Do not let cursor activity or duplicate observations slide an active
315
+ // deadline. Only a genuine retry after expiry creates a fresh window.
316
+ if (capacityCooldown.has(key))
317
+ return;
318
+ capacityCooldown.set(key, { workflow, reason, retryAt: now + CAPACITY_RELEASE_COOLDOWN_MS });
319
+ refreshCapacityCooldownDueAt();
320
+ }
321
+ /** Read-only admission signal from the freshly reconciled capacity snapshot. */
322
+ function capacityCooldownReady(remaining, agentRoom, now = monotonicNow()) {
323
+ for (const entry of capacityCooldown.values()) {
324
+ if (entry.retryAt <= now)
325
+ continue;
326
+ if (entry.reason === 'dispatch-cap-full' ? remaining > 0 : agentRoom > 0)
327
+ return true;
328
+ }
329
+ return false;
330
+ }
331
+ /**
332
+ * Remove entries for `workflow` whose own capacity class is now eligible and
333
+ * report whether any active entry still suppresses its targeted request.
334
+ * Expiry is global housekeeping so workflows which disappear from the inbox
335
+ * cannot pin this private map forever.
336
+ */
337
+ function workflowCapacityCooldownActive(workflow, remaining, agentRoom) {
338
+ const now = monotonicNow();
339
+ pruneExpiredCapacityCooldown(now);
340
+ let changed = false;
341
+ let active = false;
342
+ for (const [key, entry] of capacityCooldown) {
343
+ if (entry.workflow !== workflow)
344
+ continue;
345
+ const ready = entry.reason === 'dispatch-cap-full' ? remaining > 0 : agentRoom > 0;
346
+ if (ready) {
347
+ capacityCooldown.delete(key);
348
+ changed = true;
349
+ }
350
+ else {
351
+ active = true;
352
+ }
353
+ }
354
+ if (changed)
355
+ refreshCapacityCooldownDueAt();
356
+ return active;
357
+ }
185
358
  function noteServerBackoff(error) {
186
359
  if (!(error instanceof HubError) || error.status !== 429)
187
360
  return false;
@@ -213,10 +386,41 @@ export function createShiftLoop(opts) {
213
386
  opts.err(`shift event sink failed: ${errMsg(e)} (continuing)`);
214
387
  }
215
388
  }
389
+ /**
390
+ * Hand a claim back so another shift can be offered it immediately. This
391
+ * deliberately stays off the dispatch critical path; a failed release falls
392
+ * back to the existing pickup-window behavior and is observable.
393
+ */
394
+ function releaseClaim(workflow, run) {
395
+ void opts.hub.release({ workflow, run }).catch((e) => {
396
+ noteServerBackoff(e);
397
+ const message = errMsg(e);
398
+ opts.err(`[${workflow}/${run}] release failed: ${message} — leaving the hub pickup window to re-offer it`);
399
+ emit({ type: 'hub-error', op: 'release', workflow, message });
400
+ });
401
+ }
402
+ function acquireDispatchLock(waitMs, label) {
403
+ return acquireFileLockSync(join(opts.stateDir, '.dispatch.lock'), {
404
+ ...opts.dispatchLockOptions,
405
+ waitMs,
406
+ label,
407
+ });
408
+ }
409
+ const lockedRemovalCallbacks = createLockedRemovalCallbacks(opts.stateDir, {
410
+ dispatchLockOptions: opts.dispatchLockOptions,
411
+ });
412
+ function removeRecordUnderDispatchLock(run, recordOptions, waitMs = 30_000, label = 'owenloop Shift dispatch-state removal') {
413
+ return removeChildRecordUnderDispatchLock(opts.stateDir, run, recordOptions, {
414
+ dispatchLockOptions: opts.dispatchLockOptions,
415
+ waitMs,
416
+ label,
417
+ });
418
+ }
216
419
  function reconcile() {
217
420
  const result = reconcileInFlight(opts.stateDir, {
218
421
  ...(isAlive !== undefined ? { isAlive } : {}),
219
422
  now: opts.now(),
423
+ ...lockedRemovalCallbacks,
220
424
  });
221
425
  for (const rec of result.reaped) {
222
426
  emit({
@@ -263,10 +467,7 @@ export function createShiftLoop(opts) {
263
467
  */
264
468
  function reserveCandidate(c) {
265
469
  const childKind = c.kind === 'command' ? 'exec' : 'agent-run';
266
- const dispatchLock = acquireFileLockSync(join(opts.stateDir, '.dispatch.lock'), {
267
- waitMs: 30_000,
268
- label: 'owenloop Shift dispatch',
269
- });
470
+ const dispatchLock = acquireDispatchLock(30_000, 'owenloop Shift dispatch');
270
471
  try {
271
472
  const fresh = reconcileInFlight(opts.stateDir, {
272
473
  ...(isAlive === undefined ? {} : { isAlive }),
@@ -280,7 +481,7 @@ export function createShiftLoop(opts) {
280
481
  if (childKind === 'agent-run') {
281
482
  const agents = fresh.live.filter((record) => record.kind === 'agent-run').length +
282
483
  fresh.reserved.filter((record) => record.childKind === 'agent-run').length;
283
- if (agents >= (opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS))
484
+ if (agents >= agentLane().ceiling)
284
485
  return 'agent-capacity';
285
486
  }
286
487
  return reserveChild(opts.stateDir, {
@@ -312,6 +513,9 @@ export function createShiftLoop(opts) {
312
513
  opts.err(`[${c.workflow}/${c.order.run}] step '${c.order.step}' has failed ` +
313
514
  `${brake.count} time(s) in a row — braking for ${waitMs}ms and ` +
314
515
  `leaving this claim to lapse`);
516
+ brakeSweepDueAt = brakeSweepDueAt === undefined
517
+ ? brake.nextAllowedAt
518
+ : Math.min(brakeSweepDueAt, brake.nextAllowedAt);
315
519
  return 'braked';
316
520
  }
317
521
  let reservation;
@@ -331,15 +535,22 @@ export function createShiftLoop(opts) {
331
535
  startGate: reserved.gatePath,
332
536
  });
333
537
  cancel = spawned.cancel ?? spawned.terminate;
334
- const rec = finalizeChildReservation(opts.stateDir, reservation, {
335
- pid: spawned.pid,
336
- spawnedAt: opts.now(),
337
- kind: childKind,
338
- ...(childKind === 'agent-run' && c.defName !== undefined ? { def: c.defName } : {}),
339
- ...(childKind === 'agent-run' && c.defHash !== undefined ? { hash: c.defHash } : {}),
340
- ...(childKind === 'agent-run' ? { step: c.order.step } : {}),
341
- });
342
- startReservedChild(opts.stateDir, rec);
538
+ const dispatchLock = acquireDispatchLock(30_000, 'owenloop Shift dispatch');
539
+ let rec;
540
+ try {
541
+ rec = finalizeChildReservation(opts.stateDir, reservation, {
542
+ pid: spawned.pid,
543
+ spawnedAt: opts.now(),
544
+ kind: childKind,
545
+ ...(childKind === 'agent-run' && c.defName !== undefined ? { def: c.defName } : {}),
546
+ ...(childKind === 'agent-run' && c.defHash !== undefined ? { hash: c.defHash } : {}),
547
+ ...(childKind === 'agent-run' ? { step: c.order.step } : {}),
548
+ });
549
+ startReservedChild(opts.stateDir, rec);
550
+ }
551
+ finally {
552
+ releaseFileLock(dispatchLock);
553
+ }
343
554
  // Remember which step this run belongs to, so a later worker failure —
344
555
  // which names the run but not the fan-out key — can be charged to the
345
556
  // right brake key. NOTHING is counted here: a dispatch is not evidence of
@@ -362,7 +573,13 @@ export function createShiftLoop(opts) {
362
573
  cancel?.();
363
574
  if (reservation !== undefined) {
364
575
  try {
365
- cancelReservedChild(opts.stateDir, reservation);
576
+ const dispatchLock = acquireDispatchLock(30_000, 'owenloop Shift dispatch');
577
+ try {
578
+ cancelReservedChild(opts.stateDir, reservation);
579
+ }
580
+ finally {
581
+ releaseFileLock(dispatchLock);
582
+ }
366
583
  }
367
584
  catch (cleanupError) {
368
585
  opts.err(`[${c.workflow}/${c.order.run}] failed to cancel dispatch reservation: ${errMsg(cleanupError)}`);
@@ -383,26 +600,35 @@ export function createShiftLoop(opts) {
383
600
  }
384
601
  }
385
602
  function discardExpiredCandidate(candidate, queued) {
386
- if (monotonicNow() - candidate.requestStartedAt < MAX_PENDING_CANDIDATE_AGE_MS)
603
+ const holdMs = queued ? localQueueHoldMs() : MAX_PENDING_CANDIDATE_AGE_MS;
604
+ if (monotonicNow() - candidate.requestStartedAt < holdMs)
387
605
  return false;
388
606
  const prefix = queued ? 'queued claim' : 'claim';
389
- opts.err(`[${candidate.workflow}/${candidate.order.run}] ${prefix} expired before local dispatch — leaving the hub pickup window to re-offer it`);
607
+ const message = `${prefix} expired before local dispatch — handing the claim back to the hub`;
608
+ opts.err(`[${candidate.workflow}/${candidate.order.run}] ${message}`);
609
+ emit({
610
+ type: 'order-dropped',
611
+ workflow: candidate.workflow,
612
+ run: candidate.order.run,
613
+ step: candidate.order.step,
614
+ reason: 'claim-expired',
615
+ message,
616
+ });
617
+ releaseClaim(candidate.workflow, candidate.order.run);
390
618
  return true;
391
619
  }
392
620
  /** Dispatch already-claimed orders when local child capacity becomes free. */
393
621
  function drainPending(live, reserved) {
394
622
  let remaining = cap - live.length - reserved.length;
395
- if (remaining <= 0 || pendingCandidates.size === 0)
623
+ if (pendingCandidates.size === 0)
396
624
  return 0;
397
- const maxAgents = opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS;
398
- let agentRoom = maxAgents -
625
+ const { ceiling } = agentLane();
626
+ let agentRoom = ceiling -
399
627
  live.filter((r) => r.kind === 'agent-run').length -
400
628
  reserved.filter((r) => r.childKind === 'agent-run').length;
401
629
  const liveRuns = new Set([...live.map((r) => r.run), ...reserved.map((r) => r.run)]);
402
630
  let dispatched = 0;
403
631
  for (const [run, candidate] of pendingCandidates) {
404
- if (remaining <= 0)
405
- break;
406
632
  if (discardExpiredCandidate(candidate, true)) {
407
633
  pendingCandidates.delete(run);
408
634
  continue;
@@ -411,6 +637,10 @@ export function createShiftLoop(opts) {
411
637
  pendingCandidates.delete(run);
412
638
  continue;
413
639
  }
640
+ // Even at full capacity, keep scanning so a configured local hold is a
641
+ // real deadline rather than one deferred until another child exits.
642
+ if (remaining <= 0)
643
+ continue;
414
644
  // The agent lane's second budget can be full while the dispatch cap still
415
645
  // has room. Skip this entry and KEEP it queued (a command entry further
416
646
  // down the map is still dispatchable) — `continue`, never `break`, so one
@@ -418,10 +648,14 @@ export function createShiftLoop(opts) {
418
648
  if (candidate.kind === 'agent' && agentRoom <= 0)
419
649
  continue;
420
650
  const result = dispatchCandidate(candidate);
421
- if (result === 'total-capacity')
422
- break;
423
- if (result === 'agent-capacity')
651
+ if (result === 'total-capacity') {
652
+ remaining = 0;
424
653
  continue;
654
+ }
655
+ if (result === 'agent-capacity') {
656
+ agentRoom = 0;
657
+ continue;
658
+ }
425
659
  pendingCandidates.delete(run);
426
660
  if (result !== 'dispatched')
427
661
  continue;
@@ -464,6 +698,7 @@ export function createShiftLoop(opts) {
464
698
  if (!liveOrReserved.has(run))
465
699
  runBrakeKey.delete(run);
466
700
  }
701
+ pruneExpiredCapacityCooldown();
467
702
  /**
468
703
  * PHASE 4, for the work-dir reaper only. `polled` is the set of workflows
469
704
  * whose `whats_next` SUCCEEDED this sweep — a workflow whose call threw is
@@ -477,13 +712,17 @@ export function createShiftLoop(opts) {
477
712
  const polled = new Set();
478
713
  const openRuns = new Set();
479
714
  let complete = true;
715
+ let releasedForCapacity = false;
480
716
  let remaining = budget;
481
717
  const liveRuns = new Set([...live.map((r) => r.run), ...reserved.map((r) => r.run)]);
482
718
  // The agent lane's SECOND budget (D7). Counted from live records and durable
483
719
  // reservations, so a crash between reservation and spawn cannot overbook it.
484
720
  // A child or reservation that is reaped frees the slot on the next reconcile.
485
- const maxAgents = opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS;
486
- let agentRoom = maxAgents -
721
+ const { ceiling, requested, reserve } = agentLane();
722
+ const agentCapLabel = reserve > 0 && ceiling < requested
723
+ ? `${ceiling}, ${requested} requested minus a ${reserve}-slot exec reserve`
724
+ : `${ceiling}`;
725
+ let agentRoom = ceiling -
487
726
  live.filter((r) => r.kind === 'agent-run').length -
488
727
  reserved.filter((r) => r.childKind === 'agent-run').length;
489
728
  /**
@@ -504,7 +743,7 @@ export function createShiftLoop(opts) {
504
743
  }
505
744
  else {
506
745
  try {
507
- const inbox = await opts.hub.whatsNext();
746
+ const inbox = await opts.hub.whatsNext({ serve_capabilities: [...serveCapabilities] });
508
747
  instances = (inbox.instances ?? []).map((i) => i.workflow);
509
748
  }
510
749
  catch (e) {
@@ -518,14 +757,15 @@ export function createShiftLoop(opts) {
518
757
  // that omission is what distinguishes the untargeted inbox call from a
519
758
  // per-workflow one, since `HubErrorEvent.workflow` is optional.
520
759
  emit({ type: 'hub-error', op: 'whats_next', message: errMsg(e) });
521
- return { dispatched, polled, openRuns, complete: false };
760
+ return { dispatched, polled, openRuns, complete: false, releasedForCapacity };
522
761
  }
523
762
  }
524
763
  // Collect candidates, metering NEW dispatches and deduping in-flight.
525
764
  const claimed = new Set();
526
765
  const candidates = [];
527
766
  /**
528
- * Refuse one order and leave it for the hub pickup window.
767
+ * Refuse one order. Some reasons intentionally leave the claim for the hub
768
+ * pickup window; capacity reasons use `releaseOrder` below instead.
529
769
  *
530
770
  * Every refusal below is a DROPPED UNIT OF WORK, not a debug aside: the
531
771
  * shift declines an order the hub already handed it, and until now the only
@@ -539,11 +779,39 @@ export function createShiftLoop(opts) {
539
779
  emit({ type: 'order-dropped', workflow, run: order.run, step: order.step, reason, message });
540
780
  claimed.add(order.run);
541
781
  };
782
+ /**
783
+ * Refuse an order because this shift lacks capacity, then hand its claim
784
+ * back so a sibling daemon may dispatch it immediately.
785
+ */
786
+ const releaseOrder = (workflow, order, reason, message) => {
787
+ dropOrder(workflow, order, reason, message);
788
+ releaseClaim(workflow, order.run);
789
+ // Re-arm the next sweep, but only for the two TRANSIENT reasons — see
790
+ // `SweepResult.releasedForCapacity` for why `agent-lane-closed` is not
791
+ // one of them. The independent cooldown prevents a busy cursor from
792
+ // immediately re-claiming this stable step while the same capacity class
793
+ // remains unavailable; it neither replaces server Retry-After backoff
794
+ // nor the worker-failure brake.
795
+ if (reason === 'dispatch-cap-full' || reason === 'agent-cap-full') {
796
+ armCapacityCooldown(workflow, order, reason);
797
+ releasedForCapacity = true;
798
+ }
799
+ };
542
800
  for (const wf of instances) {
801
+ // Skipping means no targeted hub observation happened: leave this
802
+ // workflow out of `polled`/`openRuns` so the work-directory reaper never
803
+ // mistakes damping for an empty response. The inbox call and other
804
+ // workflows remain eligible.
805
+ if (workflowCapacityCooldownActive(wf, remaining, agentRoom))
806
+ continue;
543
807
  let res;
544
808
  const requestStartedAt = monotonicNow();
545
809
  try {
546
- res = await opts.hub.whatsNext({ workflow: wf, serve_crews: serveCrews });
810
+ res = await opts.hub.whatsNext({
811
+ workflow: wf,
812
+ serve_crews: serveCrews,
813
+ serve_capabilities: [...serveCapabilities],
814
+ });
547
815
  }
548
816
  catch (e) {
549
817
  if (isNonServableRace(e)) {
@@ -666,8 +934,13 @@ export function createShiftLoop(opts) {
666
934
  continue;
667
935
  }
668
936
  if (remaining <= 0) {
669
- pendingCandidates.set(order.run, candidate);
670
- opts.out(`[${wf}/${order.run}] at the dispatch cap (${cap}) — queued for local dispatch`);
937
+ if (localQueueHoldMs() > 0) {
938
+ pendingCandidates.set(order.run, candidate);
939
+ opts.out(`[${wf}/${order.run}] at the dispatch cap (${cap}) — queued for local dispatch`);
940
+ }
941
+ else {
942
+ releaseOrder(wf, order, 'dispatch-cap-full', `at the dispatch cap (${cap}) — handing the claim back to the hub`);
943
+ }
671
944
  }
672
945
  else {
673
946
  remaining--;
@@ -678,6 +951,7 @@ export function createShiftLoop(opts) {
678
951
  }
679
952
  // Agent wire routing is authoritative. The detached agent-run resolves its
680
953
  // own exact step and harness from the order digest, never from this cache.
954
+ // A re-offer already has a live child record; never release it.
681
955
  if (workerRuns.has(order.run))
682
956
  continue;
683
957
  const candidate = {
@@ -694,13 +968,27 @@ export function createShiftLoop(opts) {
694
968
  claimed.add(order.run);
695
969
  continue;
696
970
  }
971
+ if (ceiling === 0) {
972
+ releaseOrder(wf, order, 'agent-lane-closed', 'this shift runs no agent-run children (agent ceiling 0) — handing the claim back to the hub');
973
+ continue;
974
+ }
697
975
  if (!reoffer && remaining <= 0) {
698
- pendingCandidates.set(order.run, candidate);
699
- opts.out(`[${wf}/${order.run}] at the dispatch cap (${cap}) — queued for local dispatch`);
976
+ if (localQueueHoldMs() > 0) {
977
+ pendingCandidates.set(order.run, candidate);
978
+ opts.out(`[${wf}/${order.run}] at the dispatch cap (${cap}) — queued for local dispatch`);
979
+ }
980
+ else {
981
+ releaseOrder(wf, order, 'dispatch-cap-full', `at the dispatch cap (${cap}) — handing the claim back to the hub`);
982
+ }
700
983
  }
701
984
  else if (!reoffer && agentRoom <= 0) {
702
- pendingCandidates.set(order.run, candidate);
703
- opts.out(`[${wf}/${order.run}] at the agent-run cap (${maxAgents}) — queued for local dispatch`);
985
+ if (localQueueHoldMs() > 0) {
986
+ pendingCandidates.set(order.run, candidate);
987
+ opts.out(`[${wf}/${order.run}] at the agent-run cap (${agentCapLabel}) — queued for local dispatch`);
988
+ }
989
+ else {
990
+ releaseOrder(wf, order, 'agent-cap-full', `at the agent-run cap (${agentCapLabel}) — handing the claim back to the hub`);
991
+ }
704
992
  }
705
993
  else {
706
994
  if (!reoffer) {
@@ -725,14 +1013,19 @@ export function createShiftLoop(opts) {
725
1013
  continue;
726
1014
  }
727
1015
  if (result === 'total-capacity' || result === 'agent-capacity') {
728
- pendingCandidates.set(candidate.order.run, candidate);
729
1016
  const limit = result === 'total-capacity'
730
1017
  ? `dispatch cap (${cap})`
731
- : `agent-run cap (${opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS})`;
732
- opts.out(`[${candidate.workflow}/${candidate.order.run}] lost a shared-capacity race at the ${limit} — queued for local dispatch`);
1018
+ : `agent-run cap (${agentCapLabel})`;
1019
+ if (localQueueHoldMs() > 0) {
1020
+ pendingCandidates.set(candidate.order.run, candidate);
1021
+ opts.out(`[${candidate.workflow}/${candidate.order.run}] lost a shared-capacity race at the ${limit} — queued for local dispatch`);
1022
+ }
1023
+ else {
1024
+ releaseOrder(candidate.workflow, candidate.order, result === 'total-capacity' ? 'dispatch-cap-full' : 'agent-cap-full', `lost a shared-capacity race at the ${limit} — handing the claim back to the hub`);
1025
+ }
733
1026
  }
734
1027
  }
735
- return { dispatched, polled, openRuns, complete };
1028
+ return { dispatched, polled, openRuns, complete, releasedForCapacity };
736
1029
  }
737
1030
  /**
738
1031
  * PHASE 4 — remove the per-run work directories of runs that are over.
@@ -788,10 +1081,33 @@ export function createShiftLoop(opts) {
788
1081
  async function iteration() {
789
1082
  const backoffActiveAtStart = monotonicNow() < backoffUntil;
790
1083
  let rateLimitedThisIteration = false;
1084
+ // Hub roster cache refresh is low-priority operational work. Its absence is
1085
+ // always a degraded read for children, never a reason to stop dispatch.
1086
+ if (!backoffActiveAtStart &&
1087
+ opts.syncRosters !== undefined &&
1088
+ opts.rosterSyncIntervalMs !== undefined &&
1089
+ monotonicNow() - lastRosterSync >= opts.rosterSyncIntervalMs) {
1090
+ try {
1091
+ await withHubRosterSyncTimeout((signal) => opts.syncRosters(signal), opts.rosterSyncTimeoutMs);
1092
+ }
1093
+ catch (e) {
1094
+ rateLimitedThisIteration = noteServerBackoff(e);
1095
+ opts.err(`roster sync failed: ${errMsg(e)} (continuing)`);
1096
+ emit({ type: 'hub-error', op: 'roster_sync', message: `roster sync failed: ${errMsg(e)} (continuing)` });
1097
+ }
1098
+ finally {
1099
+ // This is a periodic attempt cadence, not a success-only retry loop. A
1100
+ // persistent non-429 failure must not make every 5s poll issue two more
1101
+ // hub calls and append another durable failure record.
1102
+ lastRosterSync = monotonicNow();
1103
+ refreshServeCapabilities();
1104
+ }
1105
+ }
791
1106
  // Presence when due (starts immediately — this shift exists to conduct).
792
1107
  // A server backoff suppresses every hub poll, including presence, while local
793
1108
  // reconciliation and queued dispatch continue below.
794
1109
  if (!backoffActiveAtStart &&
1110
+ !rateLimitedThisIteration &&
795
1111
  monotonicNow() - lastPresence >= opts.presenceIntervalMs) {
796
1112
  // `setShift` and `noteAttended` force the next ping by setting
797
1113
  // lastPresence to -Infinity. Either can land WHILE this ping is awaiting
@@ -804,6 +1120,7 @@ export function createShiftLoop(opts) {
804
1120
  await opts.hub.presencePing({
805
1121
  name: shiftName,
806
1122
  serve_crews: serveCrews,
1123
+ serve_capabilities: [...serveCapabilities],
807
1124
  ...(opts.shiftId !== undefined ? { shift_id: opts.shiftId } : {}),
808
1125
  ...(opts.startedAt !== undefined ? { started_at: opts.startedAt } : {}),
809
1126
  ...(attendedAt !== undefined ? { attended_at: attendedAt } : {}),
@@ -827,6 +1144,9 @@ export function createShiftLoop(opts) {
827
1144
  const reserved = inFlight.reserved;
828
1145
  const occupied = live.length + reserved.length;
829
1146
  const k = cap - occupied;
1147
+ const agentRoom = agentLane().ceiling -
1148
+ live.filter((record) => record.kind === 'agent-run').length -
1149
+ reserved.filter((record) => record.childKind === 'agent-run').length;
830
1150
  // Re-arm the at-capacity report the instant a slot frees, independently of
831
1151
  // whether this tick goes on to wake or sweep. The next stretch at capacity
832
1152
  // is a NEW episode and gets its own record.
@@ -859,9 +1179,23 @@ export function createShiftLoop(opts) {
859
1179
  // had no local capacity, or its sweep failed, the adopted cursor will be
860
1180
  // unchanged next time. `sweepOwed` preserves that unconsumed work signal.
861
1181
  let swept;
862
- if (wakeSucceeded && (changed || sweepOwed) && k > 0) {
1182
+ const brakeDue = brakeSweepDueAt !== undefined && monotonicNow() >= brakeSweepDueAt;
1183
+ // Keep a due cooldown armed while total capacity is zero: the outer gate
1184
+ // must still defer hub work, but an unchanged wake needs to retry as soon
1185
+ // as a total slot exists. `sweep()` removes expired entries only when it
1186
+ // can actually make the targeted decision.
1187
+ const capacityCooldownDue = capacityCooldownDueAt !== undefined && monotonicNow() >= capacityCooldownDueAt;
1188
+ const capacityBecameReady = capacityCooldownReady(k, agentRoom);
1189
+ if (wakeSucceeded && (changed || sweepOwed || brakeDue || capacityCooldownDue || capacityBecameReady) && k > 0) {
1190
+ // Clear before awaiting: a new brake armed during sweep must survive.
1191
+ if (brakeDue)
1192
+ brakeSweepDueAt = undefined;
863
1193
  swept = await sweep(k, live, reserved);
864
- sweepOwed = !swept.complete;
1194
+ // A COMPLETE sweep that handed work back for capacity is still unfinished
1195
+ // business. Without this the assignment below would clear the very signal
1196
+ // the release just raised, and the released order would wait for a cursor
1197
+ // change that a single-capability topology never produces.
1198
+ sweepOwed = !swept.complete || swept.releasedForCapacity;
865
1199
  }
866
1200
  else if (changed && k <= 0) {
867
1201
  sweepOwed = true;
@@ -936,15 +1270,19 @@ export function createShiftLoop(opts) {
936
1270
  return cap - current.live.length - current.reserved.length;
937
1271
  },
938
1272
  getCap: () => cap,
1273
+ agentCeiling: () => agentLane().ceiling,
939
1274
  setCap: (next) => {
940
1275
  cap = next;
941
1276
  },
942
1277
  getShift: () => ({ name: shiftName, serveCrews: [...serveCrews] }),
1278
+ getServeCapabilities: () => [...serveCapabilities],
943
1279
  setShift: (next) => {
944
1280
  if (next.name !== undefined)
945
1281
  shiftName = next.name;
946
- if (next.serveCrews !== undefined)
1282
+ if (next.serveCrews !== undefined) {
947
1283
  serveCrews = [...next.serveCrews];
1284
+ refreshServeCapabilities();
1285
+ }
948
1286
  lastPresence = Number.NEGATIVE_INFINITY; // D6: next iterate() pings immediately
949
1287
  presenceGeneration++; // survive an in-flight ping completing after this
950
1288
  return { name: shiftName, serveCrews: [...serveCrews] };
@@ -966,7 +1304,31 @@ export function createShiftLoop(opts) {
966
1304
  runBrakeKey.delete(run);
967
1305
  stepBrake.delete(brakeKey);
968
1306
  }
969
- removeChildRecord(opts.stateDir, run);
1307
+ // The hub's closed-run report is authoritative but carries no record identity.
1308
+ removeRecordUnderDispatchLock(run);
1309
+ },
1310
+ noteChildExited: (exit) => {
1311
+ pendingCandidates.delete(exit.run);
1312
+ let removed = false;
1313
+ try {
1314
+ removed = removeRecordUnderDispatchLock(exit.run, { pid: exit.pid }, 1_000, 'owenloop Shift exit reaper');
1315
+ }
1316
+ catch (e) {
1317
+ const retry = e instanceof FileLockTimeoutError
1318
+ ? ' (the pid reconciliation will retry)'
1319
+ : '';
1320
+ opts.err(`[${exit.workflow}/${exit.run}] failed to free the dispatch slot after the worker exited: ${errMsg(e)}${retry}`);
1321
+ return;
1322
+ }
1323
+ if (!removed)
1324
+ return;
1325
+ emit({
1326
+ type: 'reaped',
1327
+ workflow: exit.workflow,
1328
+ run: exit.run,
1329
+ kind: exit.kind,
1330
+ pid: exit.pid,
1331
+ });
970
1332
  },
971
1333
  noteWorkerFailure: (failure) => {
972
1334
  const brakeKey = runBrakeKey.get(failure.run);