owenloop 0.5.7 → 0.5.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/CHANGELOG.md +41 -0
  2. package/README.md +7 -8
  3. package/dist/packages/work/src/agent/capability-model.d.ts +64 -82
  4. package/dist/packages/work/src/agent/capability-model.js +94 -91
  5. package/dist/packages/work/src/agent/loop.d.ts +55 -26
  6. package/dist/packages/work/src/agent/loop.js +144 -50
  7. package/dist/packages/work/src/bundle/types.d.ts +3 -3
  8. package/dist/packages/work/src/consumed-verifier.js +6 -17
  9. package/dist/packages/work/src/exec/loop.d.ts +1 -1
  10. package/dist/packages/work/src/exec/loop.js +206 -35
  11. package/dist/packages/work/src/exec/receipt.d.ts +7 -7
  12. package/dist/packages/work/src/exec/receipt.js +6 -6
  13. package/dist/packages/work/src/harness/child-env.js +4 -5
  14. package/dist/packages/work/src/harness/claude.js +100 -0
  15. package/dist/packages/work/src/harness/codex.js +3 -4
  16. package/dist/packages/work/src/harness/session-store.d.ts +63 -3
  17. package/dist/packages/work/src/harness/session-store.js +104 -9
  18. package/dist/packages/work/src/hub/client.d.ts +19 -4
  19. package/dist/packages/work/src/hub/client.js +7 -3
  20. package/dist/packages/work/src/hub/types.d.ts +87 -3
  21. package/dist/packages/work/src/main.js +1 -0
  22. package/dist/packages/work/src/roles/agent-run.js +46 -43
  23. package/dist/packages/work/src/roles/exec.js +2 -0
  24. package/dist/packages/work/src/roles/sessions.d.ts +3 -2
  25. package/dist/packages/work/src/roles/sessions.js +9 -2
  26. package/dist/packages/work/src/roles/settings.js +3 -1
  27. package/dist/packages/work/src/roles/shift.js +14 -2
  28. package/dist/packages/work/src/roles/util.d.ts +1 -0
  29. package/dist/packages/work/src/roles/util.js +10 -0
  30. package/dist/packages/work/src/settings/hub-roster-cache.d.ts +68 -0
  31. package/dist/packages/work/src/settings/hub-roster-cache.js +370 -0
  32. package/dist/packages/work/src/settings/roster.d.ts +105 -0
  33. package/dist/packages/work/src/settings/roster.js +439 -0
  34. package/dist/packages/work/src/settings/serving.d.ts +17 -0
  35. package/dist/packages/work/src/settings/serving.js +79 -0
  36. package/dist/packages/work/src/settings/settings.d.ts +25 -24
  37. package/dist/packages/work/src/settings/settings.js +29 -33
  38. package/dist/packages/work/src/shift/loop.d.ts +108 -5
  39. package/dist/packages/work/src/shift/loop.js +299 -51
  40. package/dist/packages/work/src/shift/protocol.d.ts +7 -5
  41. package/dist/packages/work/src/shift/runtime.d.ts +15 -0
  42. package/dist/packages/work/src/shift/runtime.js +95 -13
  43. package/dist/packages/work/src/shift/server.js +2 -0
  44. package/dist/packages/work/src/shift/spawn.d.ts +18 -50
  45. package/dist/packages/work/src/shift/spawn.js +30 -65
  46. package/dist/packages/work/src/shift/state.d.ts +29 -2
  47. package/dist/packages/work/src/shift/state.js +45 -15
  48. package/dist/packages/work/src/usage.d.ts +1 -1
  49. package/dist/packages/work/src/usage.js +10 -5
  50. package/dist/packages/work/src/util/modifier-init.d.ts +7 -0
  51. package/dist/packages/work/src/util/modifier-init.js +73 -0
  52. package/dist/src/bundle/tar.d.ts +9 -4
  53. package/dist/src/bundle/tar.js +9 -4
  54. package/dist/src/capabilities.d.ts +50 -0
  55. package/dist/src/capabilities.js +38 -0
  56. package/dist/src/cli.d.ts +15 -1
  57. package/dist/src/cli.js +1099 -32
  58. package/dist/src/config-dir.d.ts +14 -19
  59. package/dist/src/config-dir.js +16 -24
  60. package/dist/src/credentials.d.ts +1 -1
  61. package/dist/src/credentials.js +1 -1
  62. package/dist/src/crypto/index.d.ts +1 -1
  63. package/dist/src/crypto/index.js +1 -1
  64. package/dist/src/crypto/org-root.d.ts +30 -12
  65. package/dist/src/crypto/org-root.js +162 -14
  66. package/dist/src/crypto/records.d.ts +3 -0
  67. package/dist/src/crypto/records.js +8 -0
  68. package/dist/src/crypto/trust-roots.d.ts +2 -2
  69. package/dist/src/crypto/trust-roots.js +2 -2
  70. package/dist/src/defs.d.ts +6 -1
  71. package/dist/src/defs.js +203 -8
  72. package/dist/src/engine.d.ts +55 -6
  73. package/dist/src/engine.js +172 -22
  74. package/dist/src/global-config.d.ts +4 -7
  75. package/dist/src/global-config.js +4 -7
  76. package/dist/src/hub.d.ts +207 -5
  77. package/dist/src/hub.js +263 -3
  78. package/dist/src/index.d.ts +5 -5
  79. package/dist/src/index.js +3 -3
  80. package/dist/src/mcp/serve.d.ts +22 -0
  81. package/dist/src/mcp/serve.js +88 -8
  82. package/dist/src/model.d.ts +2 -0
  83. package/dist/src/model.js +32 -25
  84. package/dist/src/order-resolver.d.ts +1 -0
  85. package/dist/src/order-resolver.js +8 -1
  86. package/dist/src/schemas/index.d.ts +19 -0
  87. package/dist/src/schemas/order.v1.schema.json +13 -2
  88. package/dist/src/store/pre-commit-verifier.js +1 -1
  89. package/dist/src/store.d.ts +5 -0
  90. package/dist/src/store.js +43 -8
  91. package/dist/src/types.d.ts +64 -7
  92. package/dist/src/work-settings.d.ts +2 -4
  93. package/dist/src/work-settings.js +2 -4
  94. package/docs/agent-runner.md +2 -2
  95. package/docs/authoring.md +90 -2
  96. package/docs/bundles.md +29 -3
  97. package/docs/cli.md +456 -120
  98. package/docs/crypto.md +32 -16
  99. package/docs/design.md +70 -1
  100. package/docs/routing.md +762 -0
  101. package/docs/shift-logs.md +37 -10
  102. package/docs/wire-contracts.md +50 -7
  103. package/examples/workflows/delivery.yaml +31 -14
  104. package/examples/workflows/ship.yaml +10 -2
  105. package/package.json +1 -1
  106. package/plugins/claude-code/plugin/.claude-plugin/plugin.json +1 -1
  107. package/plugins/claude-code/plugin/.mcp.json +1 -1
  108. package/plugins/claude-code/plugin/skills/author/SKILL.md +1 -2
  109. package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +1 -1
  110. package/plugins/codex/plugins/owenloop/.mcp.json +1 -1
  111. package/plugins/codex/plugins/owenloop/skills/author/SKILL.md +1 -2
@@ -27,9 +27,11 @@
27
27
  * liveInFlight counts every live child record, each one pid-probed (see
28
28
  * state.ts). When `k <= 0` the loop skips `whats_next` but remembers that the
29
29
  * adopted wake still needs a sweep. Orders already returned by `whats_next`
30
- * are already claimed, so over-cap orders enter an in-memory queue and dispatch
31
- * as soon as local child capacity frees; waiting for another hub wake would
32
- * strand those claims until pickup expiry.
30
+ * are already claimed. Orders this shift cannot dispatch are released back to
31
+ * the hub immediately, so another daemon can take them; `localQueueHoldMs`
32
+ * optionally retains them locally for a bounded window instead. The agent lane
33
+ * is additionally bounded by `cap − execReserve`, preserving room for an exec
34
+ * order while agents are saturated.
33
35
  *
34
36
  * RESILIENCE: presence/wake/whats_next failures are logged and never kill the
35
37
  * loop. `stop()` flips a flag checked between awaits; the in-flight sweep
@@ -53,21 +55,66 @@
53
55
  */
54
56
  import { join } from 'node:path';
55
57
  import { performance } from 'node:perf_hooks';
56
- import { acquireFileLockSync, releaseFileLock } from "../../../../src/lock.js";
58
+ import { acquireFileLockSync, FileLockTimeoutError, releaseFileLock, } from "../../../../src/lock.js";
57
59
  import { readDispatchBundle } from "../bundle/cache.js";
58
60
  import { HubError } from "../hub/types.js";
59
61
  import { isCommandStep, resolveCommandRouting } from "./routing.js";
60
- import { cancelReservedChild, finalizeChildReservation, reconcileInFlight, reserveChild, startReservedChild, removeChildRecord, ensureStateDir, } from "./state.js";
62
+ import { cancelReservedChild, finalizeChildReservation, reconcileInFlight, reserveChild, startReservedChild, removeChildRecord, settleChildGate, ensureStateDir, } from "./state.js";
61
63
  import { DEFAULT_WORK_DIR_TTL_MS, sweepWorkDirs as sweepWorkDirsImpl } from "../agent/workdir.js";
64
+ import { withHubRosterSyncTimeout } from "../settings/hub-roster-cache.js";
62
65
  import { sessionsPath } from "../harness/session-store.js";
63
66
  import { stampShiftEvent, } from "./protocol.js";
67
+ /** Run a state mutation while holding the lock that serializes dispatch state. */
68
+ export function withDispatchLock(stateDir, options = {}, fn) {
69
+ const dispatchLock = acquireFileLockSync(join(stateDir, '.dispatch.lock'), {
70
+ ...options.dispatchLockOptions,
71
+ waitMs: options.waitMs ?? 30_000,
72
+ label: options.label ?? 'owenloop Shift dispatch-state removal',
73
+ });
74
+ try {
75
+ return fn();
76
+ }
77
+ finally {
78
+ releaseFileLock(dispatchLock);
79
+ }
80
+ }
81
+ /**
82
+ * Remove one child record while holding the same lock that serializes durable
83
+ * reservation creation. The record is deliberately re-read by
84
+ * `removeChildRecord` only after acquisition, so a stale observer cannot erase
85
+ * a newer dispatch for the same run.
86
+ */
87
+ export function removeChildRecordUnderDispatchLock(stateDir, run, recordOptions, options = {}) {
88
+ return withDispatchLock(stateDir, options, () => removeChildRecord(stateDir, run, recordOptions));
89
+ }
90
+ /**
91
+ * Reconciliation callbacks for every unlocked Shift daemon path. Keep these
92
+ * together with the lock-held removal primitive so startup and the poll loop
93
+ * cannot silently diverge.
94
+ */
95
+ export function createLockedRemovalCallbacks(stateDir, options = {}) {
96
+ return {
97
+ removeAbandonedReservation: (reservation) => withDispatchLock(stateDir, options, () => cancelReservedChild(stateDir, reservation)),
98
+ removeDeadChild: (record) => withDispatchLock(stateDir, options, () => removeChildRecord(stateDir, record.run, { pid: record.pid })),
99
+ settleLiveChild: (record) => withDispatchLock(stateDir, options, () => settleChildGate(stateDir, record)),
100
+ };
101
+ }
64
102
  /** The `maxConcurrentAgents` fallback when the option is absent. */
65
103
  const DEFAULT_MAX_AGENTS = 4;
104
+ /** The `execReserve` fallback when the option is absent. */
105
+ const DEFAULT_EXEC_RESERVE = 1;
106
+ /** The `localQueueHoldMs` fallback when the option is absent. */
107
+ const DEFAULT_LOCAL_QUEUE_HOLD_MS = 0;
108
+ /** The hub reaps a never-contacted claim after this long. Every client-side
109
+ * re-offer latency must stay strictly under it. */
110
+ export const HUB_PICKUP_WINDOW_MS = 120_000;
111
+ /** Headroom for a detached child to make first contact after dispatch. */
112
+ const CHILD_FIRST_CONTACT_MARGIN_MS = 30_000;
66
113
  /**
67
114
  * The hub reaps a never-contacted claim after 120 seconds. Stop local queueing
68
115
  * after 90 seconds so a detached child retains 30 seconds to make first contact.
69
116
  */
70
- export const MAX_PENDING_CANDIDATE_AGE_MS = 90_000;
117
+ export const MAX_PENDING_CANDIDATE_AGE_MS = HUB_PICKUP_WINDOW_MS - CHILD_FIRST_CONTACT_MARGIN_MS;
71
118
  /**
72
119
  * THE PER-STEP FAILURE BRAKE.
73
120
  *
@@ -101,11 +148,11 @@ export const MAX_PENDING_CANDIDATE_AGE_MS = 90_000;
101
148
  * It is a rate limit, never a ban. A step that has not failed is dispatched
102
149
  * with zero delay, however many orders arrive for it at once.
103
150
  */
104
- const STEP_BRAKE_DELAYS_MS = [2_000, 8_000, 30_000, 120_000, 300_000];
151
+ export const STEP_BRAKE_DELAYS_MS = [2_000, 8_000, 30_000, 60_000, 90_000];
105
152
  /**
106
153
  * Treat a failure this long after the previous one as the start of a NEW
107
154
  * streak, resetting the count to zero first. Comfortably longer than the
108
- * longest delay in the table (300s), so a step failing at maximum backoff never
155
+ * longest delay in the table (90s), so a step failing at maximum backoff never
109
156
  * decays out of it, while an isolated transient long after an old streak starts
110
157
  * again at the shortest delay instead of inheriting a stale penalty.
111
158
  */
@@ -121,7 +168,7 @@ const STEP_BRAKE_FORGET_MS = 1_800_000;
121
168
  * worker failures. `count` is at least 1 at every call site (this is reached
122
169
  * only from the failure path), so the FIRST failure already costs 2s; past the
123
170
  * end of the table the last entry repeats, so a permanently-failing step
124
- * settles at one dispatch per five minutes instead of one per poll.
171
+ * settles at one dispatch per 90 seconds instead of one per poll.
125
172
  */
126
173
  function stepBrakeDelayMs(count) {
127
174
  const index = Math.min(count, STEP_BRAKE_DELAYS_MS.length) - 1;
@@ -146,6 +193,23 @@ function isNonServableRace(e) {
146
193
  export function createShiftLoop(opts) {
147
194
  let stopped = false;
148
195
  let cap = opts.cap;
196
+ /**
197
+ * The agent lane's live budget. Recomputed per call because `setCap`
198
+ * rewrites `cap` at runtime.
199
+ */
200
+ function agentLane() {
201
+ const requested = opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS;
202
+ const reserve = Math.min(opts.execReserve ?? DEFAULT_EXEC_RESERVE, Math.max(0, cap - 1));
203
+ return {
204
+ ceiling: Math.max(0, Math.min(requested, cap - reserve)),
205
+ requested,
206
+ reserve,
207
+ };
208
+ }
209
+ /** Effective local hold, capped below the hub pickup window. */
210
+ function localQueueHoldMs() {
211
+ return Math.min(Math.max(0, opts.localQueueHoldMs ?? DEFAULT_LOCAL_QUEUE_HOLD_MS), MAX_PENDING_CANDIDATE_AGE_MS);
212
+ }
149
213
  const isAlive = opts.isAlive;
150
214
  const monotonicNow = opts.monotonicNow ?? performance.now.bind(performance);
151
215
  // Live shift identity (MCP `clock_in`, D3-D7 of the plan). Seeded from opts,
@@ -153,9 +217,32 @@ export function createShiftLoop(opts) {
153
217
  // nor a caller of getShift/setShift can mutate the other's state afterward.
154
218
  let shiftName = opts.name;
155
219
  let serveCrews = [...opts.serveCrews];
220
+ let serveCapabilities = [];
221
+ let initial = true;
222
+ function refreshServeCapabilities() {
223
+ if (opts.computeServeCapabilities === undefined)
224
+ return;
225
+ let next;
226
+ try {
227
+ next = [...opts.computeServeCapabilities(serveCrews)];
228
+ }
229
+ catch {
230
+ return; // Keep the last known-good advertisement on a transient failure.
231
+ }
232
+ const changed = next.length !== serveCapabilities.length ||
233
+ next.some((capability, index) => capability !== serveCapabilities[index]);
234
+ serveCapabilities = next;
235
+ if (changed && !initial)
236
+ opts.out(`serving ${next.length === 0 ? '(nothing)' : next.join(', ')}`);
237
+ }
238
+ refreshServeCapabilities();
239
+ initial = false;
156
240
  // Park state persists across `iterate()` calls (the MCP park reuses it).
157
241
  let cursor;
158
242
  let lastPresence = Number.NEGATIVE_INFINITY;
243
+ // Runtime performs the required shift-start refresh before creating this
244
+ // loop. Start this cadence now so it is not immediately fetched twice.
245
+ let lastRosterSync = monotonicNow();
159
246
  let attendedAt;
160
247
  /**
161
248
  * Orders returned by `whats_next` are already claimed. If the hub returns more
@@ -168,6 +255,9 @@ export function createShiftLoop(opts) {
168
255
  /** A changed wake whose sweep was skipped or failed must be retried after the
169
256
  * cursor is adopted; otherwise the next unchanged wake hides that work. */
170
257
  let sweepOwed = false;
258
+ /** Earliest monotonic time at which a braked step becomes dispatchable again.
259
+ * Drives a sweep without a hub cursor move, so a brake never waits forever. */
260
+ let brakeSweepDueAt;
171
261
  /**
172
262
  * Has the current at-capacity EPISODE already produced a `capacity` event?
173
263
  *
@@ -213,10 +303,41 @@ export function createShiftLoop(opts) {
213
303
  opts.err(`shift event sink failed: ${errMsg(e)} (continuing)`);
214
304
  }
215
305
  }
306
+ /**
307
+ * Hand a claim back so another shift can be offered it immediately. This
308
+ * deliberately stays off the dispatch critical path; a failed release falls
309
+ * back to the existing pickup-window behavior and is observable.
310
+ */
311
+ function releaseClaim(workflow, run) {
312
+ void opts.hub.release({ workflow, run }).catch((e) => {
313
+ noteServerBackoff(e);
314
+ const message = errMsg(e);
315
+ opts.err(`[${workflow}/${run}] release failed: ${message} — leaving the hub pickup window to re-offer it`);
316
+ emit({ type: 'hub-error', op: 'release', workflow, message });
317
+ });
318
+ }
319
+ function acquireDispatchLock(waitMs, label) {
320
+ return acquireFileLockSync(join(opts.stateDir, '.dispatch.lock'), {
321
+ ...opts.dispatchLockOptions,
322
+ waitMs,
323
+ label,
324
+ });
325
+ }
326
+ const lockedRemovalCallbacks = createLockedRemovalCallbacks(opts.stateDir, {
327
+ dispatchLockOptions: opts.dispatchLockOptions,
328
+ });
329
+ function removeRecordUnderDispatchLock(run, recordOptions, waitMs = 30_000, label = 'owenloop Shift dispatch-state removal') {
330
+ return removeChildRecordUnderDispatchLock(opts.stateDir, run, recordOptions, {
331
+ dispatchLockOptions: opts.dispatchLockOptions,
332
+ waitMs,
333
+ label,
334
+ });
335
+ }
216
336
  function reconcile() {
217
337
  const result = reconcileInFlight(opts.stateDir, {
218
338
  ...(isAlive !== undefined ? { isAlive } : {}),
219
339
  now: opts.now(),
340
+ ...lockedRemovalCallbacks,
220
341
  });
221
342
  for (const rec of result.reaped) {
222
343
  emit({
@@ -263,10 +384,7 @@ export function createShiftLoop(opts) {
263
384
  */
264
385
  function reserveCandidate(c) {
265
386
  const childKind = c.kind === 'command' ? 'exec' : 'agent-run';
266
- const dispatchLock = acquireFileLockSync(join(opts.stateDir, '.dispatch.lock'), {
267
- waitMs: 30_000,
268
- label: 'owenloop Shift dispatch',
269
- });
387
+ const dispatchLock = acquireDispatchLock(30_000, 'owenloop Shift dispatch');
270
388
  try {
271
389
  const fresh = reconcileInFlight(opts.stateDir, {
272
390
  ...(isAlive === undefined ? {} : { isAlive }),
@@ -280,7 +398,7 @@ export function createShiftLoop(opts) {
280
398
  if (childKind === 'agent-run') {
281
399
  const agents = fresh.live.filter((record) => record.kind === 'agent-run').length +
282
400
  fresh.reserved.filter((record) => record.childKind === 'agent-run').length;
283
- if (agents >= (opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS))
401
+ if (agents >= agentLane().ceiling)
284
402
  return 'agent-capacity';
285
403
  }
286
404
  return reserveChild(opts.stateDir, {
@@ -312,6 +430,9 @@ export function createShiftLoop(opts) {
312
430
  opts.err(`[${c.workflow}/${c.order.run}] step '${c.order.step}' has failed ` +
313
431
  `${brake.count} time(s) in a row — braking for ${waitMs}ms and ` +
314
432
  `leaving this claim to lapse`);
433
+ brakeSweepDueAt = brakeSweepDueAt === undefined
434
+ ? brake.nextAllowedAt
435
+ : Math.min(brakeSweepDueAt, brake.nextAllowedAt);
315
436
  return 'braked';
316
437
  }
317
438
  let reservation;
@@ -331,15 +452,22 @@ export function createShiftLoop(opts) {
331
452
  startGate: reserved.gatePath,
332
453
  });
333
454
  cancel = spawned.cancel ?? spawned.terminate;
334
- const rec = finalizeChildReservation(opts.stateDir, reservation, {
335
- pid: spawned.pid,
336
- spawnedAt: opts.now(),
337
- kind: childKind,
338
- ...(childKind === 'agent-run' && c.defName !== undefined ? { def: c.defName } : {}),
339
- ...(childKind === 'agent-run' && c.defHash !== undefined ? { hash: c.defHash } : {}),
340
- ...(childKind === 'agent-run' ? { step: c.order.step } : {}),
341
- });
342
- startReservedChild(opts.stateDir, rec);
455
+ const dispatchLock = acquireDispatchLock(30_000, 'owenloop Shift dispatch');
456
+ let rec;
457
+ try {
458
+ rec = finalizeChildReservation(opts.stateDir, reservation, {
459
+ pid: spawned.pid,
460
+ spawnedAt: opts.now(),
461
+ kind: childKind,
462
+ ...(childKind === 'agent-run' && c.defName !== undefined ? { def: c.defName } : {}),
463
+ ...(childKind === 'agent-run' && c.defHash !== undefined ? { hash: c.defHash } : {}),
464
+ ...(childKind === 'agent-run' ? { step: c.order.step } : {}),
465
+ });
466
+ startReservedChild(opts.stateDir, rec);
467
+ }
468
+ finally {
469
+ releaseFileLock(dispatchLock);
470
+ }
343
471
  // Remember which step this run belongs to, so a later worker failure —
344
472
  // which names the run but not the fan-out key — can be charged to the
345
473
  // right brake key. NOTHING is counted here: a dispatch is not evidence of
@@ -362,7 +490,13 @@ export function createShiftLoop(opts) {
362
490
  cancel?.();
363
491
  if (reservation !== undefined) {
364
492
  try {
365
- cancelReservedChild(opts.stateDir, reservation);
493
+ const dispatchLock = acquireDispatchLock(30_000, 'owenloop Shift dispatch');
494
+ try {
495
+ cancelReservedChild(opts.stateDir, reservation);
496
+ }
497
+ finally {
498
+ releaseFileLock(dispatchLock);
499
+ }
366
500
  }
367
501
  catch (cleanupError) {
368
502
  opts.err(`[${c.workflow}/${c.order.run}] failed to cancel dispatch reservation: ${errMsg(cleanupError)}`);
@@ -383,26 +517,35 @@ export function createShiftLoop(opts) {
383
517
  }
384
518
  }
385
519
  function discardExpiredCandidate(candidate, queued) {
386
- if (monotonicNow() - candidate.requestStartedAt < MAX_PENDING_CANDIDATE_AGE_MS)
520
+ const holdMs = queued ? localQueueHoldMs() : MAX_PENDING_CANDIDATE_AGE_MS;
521
+ if (monotonicNow() - candidate.requestStartedAt < holdMs)
387
522
  return false;
388
523
  const prefix = queued ? 'queued claim' : 'claim';
389
- opts.err(`[${candidate.workflow}/${candidate.order.run}] ${prefix} expired before local dispatch — leaving the hub pickup window to re-offer it`);
524
+ const message = `${prefix} expired before local dispatch — handing the claim back to the hub`;
525
+ opts.err(`[${candidate.workflow}/${candidate.order.run}] ${message}`);
526
+ emit({
527
+ type: 'order-dropped',
528
+ workflow: candidate.workflow,
529
+ run: candidate.order.run,
530
+ step: candidate.order.step,
531
+ reason: 'claim-expired',
532
+ message,
533
+ });
534
+ releaseClaim(candidate.workflow, candidate.order.run);
390
535
  return true;
391
536
  }
392
537
  /** Dispatch already-claimed orders when local child capacity becomes free. */
393
538
  function drainPending(live, reserved) {
394
539
  let remaining = cap - live.length - reserved.length;
395
- if (remaining <= 0 || pendingCandidates.size === 0)
540
+ if (pendingCandidates.size === 0)
396
541
  return 0;
397
- const maxAgents = opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS;
398
- let agentRoom = maxAgents -
542
+ const { ceiling } = agentLane();
543
+ let agentRoom = ceiling -
399
544
  live.filter((r) => r.kind === 'agent-run').length -
400
545
  reserved.filter((r) => r.childKind === 'agent-run').length;
401
546
  const liveRuns = new Set([...live.map((r) => r.run), ...reserved.map((r) => r.run)]);
402
547
  let dispatched = 0;
403
548
  for (const [run, candidate] of pendingCandidates) {
404
- if (remaining <= 0)
405
- break;
406
549
  if (discardExpiredCandidate(candidate, true)) {
407
550
  pendingCandidates.delete(run);
408
551
  continue;
@@ -411,6 +554,10 @@ export function createShiftLoop(opts) {
411
554
  pendingCandidates.delete(run);
412
555
  continue;
413
556
  }
557
+ // Even at full capacity, keep scanning so a configured local hold is a
558
+ // real deadline rather than one deferred until another child exits.
559
+ if (remaining <= 0)
560
+ continue;
414
561
  // The agent lane's second budget can be full while the dispatch cap still
415
562
  // has room. Skip this entry and KEEP it queued (a command entry further
416
563
  // down the map is still dispatchable) — `continue`, never `break`, so one
@@ -418,10 +565,14 @@ export function createShiftLoop(opts) {
418
565
  if (candidate.kind === 'agent' && agentRoom <= 0)
419
566
  continue;
420
567
  const result = dispatchCandidate(candidate);
421
- if (result === 'total-capacity')
422
- break;
423
- if (result === 'agent-capacity')
568
+ if (result === 'total-capacity') {
569
+ remaining = 0;
424
570
  continue;
571
+ }
572
+ if (result === 'agent-capacity') {
573
+ agentRoom = 0;
574
+ continue;
575
+ }
425
576
  pendingCandidates.delete(run);
426
577
  if (result !== 'dispatched')
427
578
  continue;
@@ -482,8 +633,11 @@ export function createShiftLoop(opts) {
482
633
  // The agent lane's SECOND budget (D7). Counted from live records and durable
483
634
  // reservations, so a crash between reservation and spawn cannot overbook it.
484
635
  // A child or reservation that is reaped frees the slot on the next reconcile.
485
- const maxAgents = opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS;
486
- let agentRoom = maxAgents -
636
+ const { ceiling, requested, reserve } = agentLane();
637
+ const agentCapLabel = reserve > 0 && ceiling < requested
638
+ ? `${ceiling}, ${requested} requested minus a ${reserve}-slot exec reserve`
639
+ : `${ceiling}`;
640
+ let agentRoom = ceiling -
487
641
  live.filter((r) => r.kind === 'agent-run').length -
488
642
  reserved.filter((r) => r.childKind === 'agent-run').length;
489
643
  /**
@@ -504,7 +658,7 @@ export function createShiftLoop(opts) {
504
658
  }
505
659
  else {
506
660
  try {
507
- const inbox = await opts.hub.whatsNext();
661
+ const inbox = await opts.hub.whatsNext({ serve_capabilities: [...serveCapabilities] });
508
662
  instances = (inbox.instances ?? []).map((i) => i.workflow);
509
663
  }
510
664
  catch (e) {
@@ -525,7 +679,8 @@ export function createShiftLoop(opts) {
525
679
  const claimed = new Set();
526
680
  const candidates = [];
527
681
  /**
528
- * Refuse one order and leave it for the hub pickup window.
682
+ * Refuse one order. Some reasons intentionally leave the claim for the hub
683
+ * pickup window; capacity reasons use `releaseOrder` below instead.
529
684
  *
530
685
  * Every refusal below is a DROPPED UNIT OF WORK, not a debug aside: the
531
686
  * shift declines an order the hub already handed it, and until now the only
@@ -539,11 +694,23 @@ export function createShiftLoop(opts) {
539
694
  emit({ type: 'order-dropped', workflow, run: order.run, step: order.step, reason, message });
540
695
  claimed.add(order.run);
541
696
  };
697
+ /**
698
+ * Refuse an order because this shift lacks capacity, then hand its claim
699
+ * back so a sibling daemon may dispatch it immediately.
700
+ */
701
+ const releaseOrder = (workflow, order, reason, message) => {
702
+ dropOrder(workflow, order, reason, message);
703
+ releaseClaim(workflow, order.run);
704
+ };
542
705
  for (const wf of instances) {
543
706
  let res;
544
707
  const requestStartedAt = monotonicNow();
545
708
  try {
546
- res = await opts.hub.whatsNext({ workflow: wf, serve_crews: serveCrews });
709
+ res = await opts.hub.whatsNext({
710
+ workflow: wf,
711
+ serve_crews: serveCrews,
712
+ serve_capabilities: [...serveCapabilities],
713
+ });
547
714
  }
548
715
  catch (e) {
549
716
  if (isNonServableRace(e)) {
@@ -666,8 +833,13 @@ export function createShiftLoop(opts) {
666
833
  continue;
667
834
  }
668
835
  if (remaining <= 0) {
669
- pendingCandidates.set(order.run, candidate);
670
- opts.out(`[${wf}/${order.run}] at the dispatch cap (${cap}) — queued for local dispatch`);
836
+ if (localQueueHoldMs() > 0) {
837
+ pendingCandidates.set(order.run, candidate);
838
+ opts.out(`[${wf}/${order.run}] at the dispatch cap (${cap}) — queued for local dispatch`);
839
+ }
840
+ else {
841
+ releaseOrder(wf, order, 'dispatch-cap-full', `at the dispatch cap (${cap}) — handing the claim back to the hub`);
842
+ }
671
843
  }
672
844
  else {
673
845
  remaining--;
@@ -678,6 +850,7 @@ export function createShiftLoop(opts) {
678
850
  }
679
851
  // Agent wire routing is authoritative. The detached agent-run resolves its
680
852
  // own exact step and harness from the order digest, never from this cache.
853
+ // A re-offer already has a live child record; never release it.
681
854
  if (workerRuns.has(order.run))
682
855
  continue;
683
856
  const candidate = {
@@ -694,13 +867,27 @@ export function createShiftLoop(opts) {
694
867
  claimed.add(order.run);
695
868
  continue;
696
869
  }
870
+ if (ceiling === 0) {
871
+ releaseOrder(wf, order, 'agent-lane-closed', 'this shift runs no agent-run children (agent ceiling 0) — handing the claim back to the hub');
872
+ continue;
873
+ }
697
874
  if (!reoffer && remaining <= 0) {
698
- pendingCandidates.set(order.run, candidate);
699
- opts.out(`[${wf}/${order.run}] at the dispatch cap (${cap}) — queued for local dispatch`);
875
+ if (localQueueHoldMs() > 0) {
876
+ pendingCandidates.set(order.run, candidate);
877
+ opts.out(`[${wf}/${order.run}] at the dispatch cap (${cap}) — queued for local dispatch`);
878
+ }
879
+ else {
880
+ releaseOrder(wf, order, 'dispatch-cap-full', `at the dispatch cap (${cap}) — handing the claim back to the hub`);
881
+ }
700
882
  }
701
883
  else if (!reoffer && agentRoom <= 0) {
702
- pendingCandidates.set(order.run, candidate);
703
- opts.out(`[${wf}/${order.run}] at the agent-run cap (${maxAgents}) — queued for local dispatch`);
884
+ if (localQueueHoldMs() > 0) {
885
+ pendingCandidates.set(order.run, candidate);
886
+ opts.out(`[${wf}/${order.run}] at the agent-run cap (${agentCapLabel}) — queued for local dispatch`);
887
+ }
888
+ else {
889
+ releaseOrder(wf, order, 'agent-cap-full', `at the agent-run cap (${agentCapLabel}) — handing the claim back to the hub`);
890
+ }
704
891
  }
705
892
  else {
706
893
  if (!reoffer) {
@@ -725,11 +912,16 @@ export function createShiftLoop(opts) {
725
912
  continue;
726
913
  }
727
914
  if (result === 'total-capacity' || result === 'agent-capacity') {
728
- pendingCandidates.set(candidate.order.run, candidate);
729
915
  const limit = result === 'total-capacity'
730
916
  ? `dispatch cap (${cap})`
731
- : `agent-run cap (${opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS})`;
732
- opts.out(`[${candidate.workflow}/${candidate.order.run}] lost a shared-capacity race at the ${limit} — queued for local dispatch`);
917
+ : `agent-run cap (${agentCapLabel})`;
918
+ if (localQueueHoldMs() > 0) {
919
+ pendingCandidates.set(candidate.order.run, candidate);
920
+ opts.out(`[${candidate.workflow}/${candidate.order.run}] lost a shared-capacity race at the ${limit} — queued for local dispatch`);
921
+ }
922
+ else {
923
+ releaseOrder(candidate.workflow, candidate.order, result === 'total-capacity' ? 'dispatch-cap-full' : 'agent-cap-full', `lost a shared-capacity race at the ${limit} — handing the claim back to the hub`);
924
+ }
733
925
  }
734
926
  }
735
927
  return { dispatched, polled, openRuns, complete };
@@ -788,10 +980,33 @@ export function createShiftLoop(opts) {
788
980
  async function iteration() {
789
981
  const backoffActiveAtStart = monotonicNow() < backoffUntil;
790
982
  let rateLimitedThisIteration = false;
983
+ // Hub roster cache refresh is low-priority operational work. Its absence is
984
+ // always a degraded read for children, never a reason to stop dispatch.
985
+ if (!backoffActiveAtStart &&
986
+ opts.syncRosters !== undefined &&
987
+ opts.rosterSyncIntervalMs !== undefined &&
988
+ monotonicNow() - lastRosterSync >= opts.rosterSyncIntervalMs) {
989
+ try {
990
+ await withHubRosterSyncTimeout((signal) => opts.syncRosters(signal), opts.rosterSyncTimeoutMs);
991
+ }
992
+ catch (e) {
993
+ rateLimitedThisIteration = noteServerBackoff(e);
994
+ opts.err(`roster sync failed: ${errMsg(e)} (continuing)`);
995
+ emit({ type: 'hub-error', op: 'roster_sync', message: `roster sync failed: ${errMsg(e)} (continuing)` });
996
+ }
997
+ finally {
998
+ // This is a periodic attempt cadence, not a success-only retry loop. A
999
+ // persistent non-429 failure must not make every 5s poll issue two more
1000
+ // hub calls and append another durable failure record.
1001
+ lastRosterSync = monotonicNow();
1002
+ refreshServeCapabilities();
1003
+ }
1004
+ }
791
1005
  // Presence when due (starts immediately — this shift exists to conduct).
792
1006
  // A server backoff suppresses every hub poll, including presence, while local
793
1007
  // reconciliation and queued dispatch continue below.
794
1008
  if (!backoffActiveAtStart &&
1009
+ !rateLimitedThisIteration &&
795
1010
  monotonicNow() - lastPresence >= opts.presenceIntervalMs) {
796
1011
  // `setShift` and `noteAttended` force the next ping by setting
797
1012
  // lastPresence to -Infinity. Either can land WHILE this ping is awaiting
@@ -804,6 +1019,7 @@ export function createShiftLoop(opts) {
804
1019
  await opts.hub.presencePing({
805
1020
  name: shiftName,
806
1021
  serve_crews: serveCrews,
1022
+ serve_capabilities: [...serveCapabilities],
807
1023
  ...(opts.shiftId !== undefined ? { shift_id: opts.shiftId } : {}),
808
1024
  ...(opts.startedAt !== undefined ? { started_at: opts.startedAt } : {}),
809
1025
  ...(attendedAt !== undefined ? { attended_at: attendedAt } : {}),
@@ -859,7 +1075,11 @@ export function createShiftLoop(opts) {
859
1075
  // had no local capacity, or its sweep failed, the adopted cursor will be
860
1076
  // unchanged next time. `sweepOwed` preserves that unconsumed work signal.
861
1077
  let swept;
862
- if (wakeSucceeded && (changed || sweepOwed) && k > 0) {
1078
+ const brakeDue = brakeSweepDueAt !== undefined && monotonicNow() >= brakeSweepDueAt;
1079
+ if (wakeSucceeded && (changed || sweepOwed || brakeDue) && k > 0) {
1080
+ // Clear before awaiting: a new brake armed during sweep must survive.
1081
+ if (brakeDue)
1082
+ brakeSweepDueAt = undefined;
863
1083
  swept = await sweep(k, live, reserved);
864
1084
  sweepOwed = !swept.complete;
865
1085
  }
@@ -936,15 +1156,19 @@ export function createShiftLoop(opts) {
936
1156
  return cap - current.live.length - current.reserved.length;
937
1157
  },
938
1158
  getCap: () => cap,
1159
+ agentCeiling: () => agentLane().ceiling,
939
1160
  setCap: (next) => {
940
1161
  cap = next;
941
1162
  },
942
1163
  getShift: () => ({ name: shiftName, serveCrews: [...serveCrews] }),
1164
+ getServeCapabilities: () => [...serveCapabilities],
943
1165
  setShift: (next) => {
944
1166
  if (next.name !== undefined)
945
1167
  shiftName = next.name;
946
- if (next.serveCrews !== undefined)
1168
+ if (next.serveCrews !== undefined) {
947
1169
  serveCrews = [...next.serveCrews];
1170
+ refreshServeCapabilities();
1171
+ }
948
1172
  lastPresence = Number.NEGATIVE_INFINITY; // D6: next iterate() pings immediately
949
1173
  presenceGeneration++; // survive an in-flight ping completing after this
950
1174
  return { name: shiftName, serveCrews: [...serveCrews] };
@@ -966,7 +1190,31 @@ export function createShiftLoop(opts) {
966
1190
  runBrakeKey.delete(run);
967
1191
  stepBrake.delete(brakeKey);
968
1192
  }
969
- removeChildRecord(opts.stateDir, run);
1193
+ // The hub's closed-run report is authoritative but carries no record identity.
1194
+ removeRecordUnderDispatchLock(run);
1195
+ },
1196
+ noteChildExited: (exit) => {
1197
+ pendingCandidates.delete(exit.run);
1198
+ let removed = false;
1199
+ try {
1200
+ removed = removeRecordUnderDispatchLock(exit.run, { pid: exit.pid }, 1_000, 'owenloop Shift exit reaper');
1201
+ }
1202
+ catch (e) {
1203
+ const retry = e instanceof FileLockTimeoutError
1204
+ ? ' (the pid reconciliation will retry)'
1205
+ : '';
1206
+ opts.err(`[${exit.workflow}/${exit.run}] failed to free the dispatch slot after the worker exited: ${errMsg(e)}${retry}`);
1207
+ return;
1208
+ }
1209
+ if (!removed)
1210
+ return;
1211
+ emit({
1212
+ type: 'reaped',
1213
+ workflow: exit.workflow,
1214
+ run: exit.run,
1215
+ kind: exit.kind,
1216
+ pid: exit.pid,
1217
+ });
970
1218
  },
971
1219
  noteWorkerFailure: (failure) => {
972
1220
  const brakeKey = runBrakeKey.get(failure.run);
@@ -119,10 +119,10 @@ export interface CapacityEvent {
119
119
  inFlight: number;
120
120
  cap: number;
121
121
  }
122
- /** A hub call failed. `workflow` is present only for per-workflow calls. */
122
+ /** A hub call failed. `workflow` is present for per-workflow whats_next and release calls. */
123
123
  export interface HubErrorEvent {
124
124
  type: 'hub-error';
125
- op: 'wake' | 'whats_next';
125
+ op: 'wake' | 'whats_next' | 'roster_sync' | 'release';
126
126
  workflow?: string;
127
127
  message: string;
128
128
  }
@@ -133,8 +133,9 @@ export interface BundleMissEvent {
133
133
  def: string;
134
134
  }
135
135
  /**
136
- * The shift refused one order and left it for the hub pickup window. A dropped
137
- * unit of work, which is why it is a record and not a debug aside. `reason` is
136
+ * The shift refused one order. Capacity and expiry reasons hand its claim back
137
+ * to the hub; malformed and unsupported reasons leave it for the pickup
138
+ * window. A dropped unit of work is a record, not a debug aside. `reason` is
138
139
  * the stable machine discriminator; `message` is the human text.
139
140
  */
140
141
  export interface OrderDroppedEvent {
@@ -142,7 +143,7 @@ export interface OrderDroppedEvent {
142
143
  workflow: string;
143
144
  run: string;
144
145
  step: string;
145
- reason: 'malformed-digest' | 'malformed-worker' | 'unsupported-worker' | 'verification-failed' | 'metadata-unavailable';
146
+ reason: 'malformed-digest' | 'malformed-worker' | 'unsupported-worker' | 'verification-failed' | 'metadata-unavailable' | 'agent-lane-closed' | 'dispatch-cap-full' | 'agent-cap-full' | 'claim-expired';
146
147
  message: string;
147
148
  }
148
149
  /**
@@ -205,6 +206,7 @@ export interface ShiftStatus {
205
206
  cap: number;
206
207
  free: number;
207
208
  running: number;
209
+ agent_ceiling: number;
208
210
  attended_at: number | null;
209
211
  started_at: number;
210
212
  }