owenloop 0.5.0 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (211) hide show
  1. package/CHANGELOG.md +45 -0
  2. package/README.md +137 -35
  3. package/bin/owenloop.mjs +49 -1
  4. package/dist/packages/work/src/agent/brief.d.ts +8 -4
  5. package/dist/packages/work/src/agent/brief.js +7 -3
  6. package/dist/packages/work/src/agent/loop.d.ts +13 -7
  7. package/dist/packages/work/src/agent/loop.js +136 -9
  8. package/dist/packages/work/src/agent/model-policy.d.ts +49 -0
  9. package/dist/packages/work/src/agent/model-policy.js +138 -0
  10. package/dist/packages/work/src/bundle/cache.d.ts +7 -5
  11. package/dist/packages/work/src/bundle/cache.js +20 -6
  12. package/dist/packages/work/src/bundle/fetch.d.ts +7 -5
  13. package/dist/packages/work/src/bundle/fetch.js +27 -7
  14. package/dist/packages/work/src/bundle/types.d.ts +8 -4
  15. package/dist/packages/work/src/consumed-verifier.d.ts +45 -0
  16. package/dist/packages/work/src/consumed-verifier.js +205 -0
  17. package/dist/packages/work/src/exec/instructions.d.ts +99 -0
  18. package/dist/packages/work/src/exec/instructions.js +323 -0
  19. package/dist/packages/work/src/exec/loop.d.ts +12 -1
  20. package/dist/packages/work/src/exec/loop.js +137 -20
  21. package/dist/packages/work/src/exec/payload.d.ts +29 -0
  22. package/dist/packages/work/src/exec/payload.js +54 -0
  23. package/dist/packages/work/src/exec/receipt.d.ts +9 -6
  24. package/dist/packages/work/src/exec/receipt.js +12 -1
  25. package/dist/packages/work/src/exec/runner.d.ts +7 -1
  26. package/dist/packages/work/src/exec/runner.js +74 -3
  27. package/dist/packages/work/src/harness/child-env.js +7 -0
  28. package/dist/packages/work/src/harness/claude.d.ts +20 -8
  29. package/dist/packages/work/src/harness/claude.js +260 -58
  30. package/dist/packages/work/src/harness/codex.d.ts +5 -4
  31. package/dist/packages/work/src/harness/codex.js +169 -41
  32. package/dist/packages/work/src/harness/contract.d.ts +36 -11
  33. package/dist/packages/work/src/harness/fake.js +1 -0
  34. package/dist/packages/work/src/harness/jsonrpc-stdio.js +7 -0
  35. package/dist/packages/work/src/harness/permissions.d.ts +16 -3
  36. package/dist/packages/work/src/harness/permissions.js +102 -13
  37. package/dist/packages/work/src/harness/session-store.d.ts +27 -9
  38. package/dist/packages/work/src/harness/session-store.js +271 -67
  39. package/dist/packages/work/src/hold/mcp.d.ts +25 -5
  40. package/dist/packages/work/src/hold/mcp.js +126 -11
  41. package/dist/packages/work/src/hub/client.d.ts +2 -1
  42. package/dist/packages/work/src/hub/client.js +14 -1
  43. package/dist/packages/work/src/hub/types.d.ts +69 -33
  44. package/dist/packages/work/src/hub/types.js +8 -10
  45. package/dist/packages/work/src/mcp/server.js +1 -1
  46. package/dist/packages/work/src/owenloop-bin.d.ts +1 -0
  47. package/dist/packages/work/src/owenloop-bin.js +18 -0
  48. package/dist/packages/work/src/roles/agent-run.d.ts +12 -6
  49. package/dist/packages/work/src/roles/agent-run.js +116 -34
  50. package/dist/packages/work/src/roles/exec.d.ts +8 -0
  51. package/dist/packages/work/src/roles/exec.js +27 -2
  52. package/dist/packages/work/src/roles/hold.d.ts +4 -0
  53. package/dist/packages/work/src/roles/hold.js +44 -5
  54. package/dist/packages/work/src/roles/lint.d.ts +3 -4
  55. package/dist/packages/work/src/roles/lint.js +41 -7
  56. package/dist/packages/work/src/roles/settings.js +5 -0
  57. package/dist/packages/work/src/roles/shift.d.ts +1 -0
  58. package/dist/packages/work/src/roles/shift.js +23 -1
  59. package/dist/packages/work/src/settings/settings.d.ts +17 -1
  60. package/dist/packages/work/src/settings/settings.js +39 -0
  61. package/dist/packages/work/src/shift/loop.d.ts +14 -3
  62. package/dist/packages/work/src/shift/loop.js +447 -164
  63. package/dist/packages/work/src/shift/protocol.d.ts +4 -0
  64. package/dist/packages/work/src/shift/runtime.d.ts +2 -0
  65. package/dist/packages/work/src/shift/runtime.js +66 -3
  66. package/dist/packages/work/src/shift/spawn.d.ts +41 -18
  67. package/dist/packages/work/src/shift/spawn.js +89 -31
  68. package/dist/packages/work/src/shift/state.d.ts +53 -51
  69. package/dist/packages/work/src/shift/state.js +325 -102
  70. package/dist/packages/work/src/submit-proof.d.ts +36 -0
  71. package/dist/packages/work/src/submit-proof.js +93 -0
  72. package/dist/src/add.d.ts +64 -259
  73. package/dist/src/add.js +81 -615
  74. package/dist/src/archive.d.ts +97 -0
  75. package/dist/src/archive.js +591 -0
  76. package/dist/src/bundle/index.d.ts +64 -0
  77. package/dist/src/bundle/index.js +508 -0
  78. package/dist/src/bundle/manifest.d.ts +50 -0
  79. package/dist/src/bundle/manifest.js +435 -0
  80. package/dist/src/bundle/runtime.d.ts +26 -0
  81. package/dist/src/bundle/runtime.js +64 -0
  82. package/dist/src/bundle/tar.d.ts +62 -0
  83. package/dist/src/bundle/tar.js +238 -0
  84. package/dist/src/bundle/types.d.ts +105 -0
  85. package/dist/src/bundle/types.js +21 -0
  86. package/dist/src/cli.d.ts +90 -20
  87. package/dist/src/cli.js +2407 -365
  88. package/dist/src/credentials.d.ts +4 -3
  89. package/dist/src/credentials.js +21 -19
  90. package/dist/src/crypto/allowed-signers.d.ts +68 -0
  91. package/dist/src/crypto/allowed-signers.js +307 -0
  92. package/dist/src/crypto/canonical.d.ts +8 -0
  93. package/dist/src/crypto/canonical.js +92 -0
  94. package/dist/src/crypto/chain.d.ts +65 -0
  95. package/dist/src/crypto/chain.js +377 -0
  96. package/dist/src/crypto/dsse.d.ts +209 -0
  97. package/dist/src/crypto/dsse.js +282 -0
  98. package/dist/src/crypto/enrollment.d.ts +78 -0
  99. package/dist/src/crypto/enrollment.js +186 -0
  100. package/dist/src/crypto/index.d.ts +48 -0
  101. package/dist/src/crypto/index.js +44 -0
  102. package/dist/src/crypto/keys.d.ts +258 -0
  103. package/dist/src/crypto/keys.js +908 -0
  104. package/dist/src/crypto/org-root.d.ts +33 -0
  105. package/dist/src/crypto/org-root.js +78 -0
  106. package/dist/src/crypto/origin-rules.d.ts +56 -0
  107. package/dist/src/crypto/origin-rules.js +143 -0
  108. package/dist/src/crypto/policy-floor.d.ts +98 -0
  109. package/dist/src/crypto/policy-floor.js +353 -0
  110. package/dist/src/crypto/records.d.ts +316 -0
  111. package/dist/src/crypto/records.js +157 -0
  112. package/dist/src/crypto/scope.d.ts +49 -0
  113. package/dist/src/crypto/scope.js +83 -0
  114. package/dist/src/crypto/ssh.d.ts +171 -0
  115. package/dist/src/crypto/ssh.js +396 -0
  116. package/dist/src/crypto/submission.d.ts +33 -0
  117. package/dist/src/crypto/submission.js +47 -0
  118. package/dist/src/crypto/trust-roots.d.ts +29 -0
  119. package/dist/src/crypto/trust-roots.js +39 -0
  120. package/dist/src/crypto/verify-consumed.d.ts +67 -0
  121. package/dist/src/crypto/verify-consumed.js +259 -0
  122. package/dist/src/crypto/verify-origin.d.ts +50 -0
  123. package/dist/src/crypto/verify-origin.js +151 -0
  124. package/dist/src/crypto/verify-publication.d.ts +60 -0
  125. package/dist/src/crypto/verify-publication.js +150 -0
  126. package/dist/src/defs.d.ts +24 -1
  127. package/dist/src/defs.js +145 -25
  128. package/dist/src/engine.d.ts +99 -3
  129. package/dist/src/engine.js +238 -34
  130. package/dist/src/factory.d.ts +16 -0
  131. package/dist/src/factory.js +16 -4
  132. package/dist/src/global-config.d.ts +59 -0
  133. package/dist/src/global-config.js +84 -0
  134. package/dist/src/hub.d.ts +75 -21
  135. package/dist/src/hub.js +123 -40
  136. package/dist/src/index.d.ts +17 -2
  137. package/dist/src/index.js +20 -2
  138. package/dist/src/install.d.ts +538 -0
  139. package/dist/src/install.js +1981 -0
  140. package/dist/src/lock.d.ts +15 -57
  141. package/dist/src/lock.js +251 -143
  142. package/dist/src/mcp/serve.d.ts +46 -22
  143. package/dist/src/mcp/serve.js +167 -44
  144. package/dist/src/mcp/server.js +12 -1
  145. package/dist/src/model.d.ts +15 -0
  146. package/dist/src/model.js +111 -1
  147. package/dist/src/order-resolver.d.ts +184 -0
  148. package/dist/src/order-resolver.js +252 -0
  149. package/dist/src/package-version.d.ts +10 -0
  150. package/dist/src/package-version.js +42 -0
  151. package/dist/src/paths.d.ts +14 -1
  152. package/dist/src/paths.js +18 -0
  153. package/dist/src/schemas/enrollment-grant.v1.schema.json +90 -0
  154. package/dist/src/schemas/index.d.ts +1416 -0
  155. package/dist/src/schemas/index.js +31 -0
  156. package/dist/src/schemas/order.v1.schema.json +106 -0
  157. package/dist/src/schemas/origin.v1.schema.json +68 -0
  158. package/dist/src/schemas/policy-floor.v1.schema.json +29 -0
  159. package/dist/src/schemas/publication.v1.schema.json +31 -0
  160. package/dist/src/schemas/revocation.v1.schema.json +39 -0
  161. package/dist/src/schemas/submission.v1.schema.json +61 -0
  162. package/dist/src/store/def-source.d.ts +52 -0
  163. package/dist/src/store/def-source.js +335 -0
  164. package/dist/src/store/index-file.d.ts +69 -0
  165. package/dist/src/store/index-file.js +151 -0
  166. package/dist/src/store/index.d.ts +27 -0
  167. package/dist/src/store/index.js +22 -0
  168. package/dist/src/store/ingestor.d.ts +43 -0
  169. package/dist/src/store/ingestor.js +211 -0
  170. package/dist/src/store/install.d.ts +228 -0
  171. package/dist/src/store/install.js +555 -0
  172. package/dist/src/store/instruction-source.d.ts +49 -0
  173. package/dist/src/store/instruction-source.js +298 -0
  174. package/dist/src/store/origin-evidence.d.ts +21 -0
  175. package/dist/src/store/origin-evidence.js +66 -0
  176. package/dist/src/store/pre-commit-verifier.d.ts +68 -0
  177. package/dist/src/store/pre-commit-verifier.js +418 -0
  178. package/dist/src/store/resolve.d.ts +138 -0
  179. package/dist/src/store/resolve.js +363 -0
  180. package/dist/src/store/types.d.ts +288 -0
  181. package/dist/src/store/types.js +356 -0
  182. package/dist/src/types.d.ts +45 -11
  183. package/dist/src/untar.d.ts +8 -23
  184. package/dist/src/untar.js +10 -136
  185. package/docs/agent-runner.md +86 -0
  186. package/docs/authoring.md +57 -6
  187. package/docs/bundles.md +250 -0
  188. package/docs/cli.md +1171 -97
  189. package/docs/crypto.md +816 -0
  190. package/docs/design.md +342 -20
  191. package/docs/embedding.md +132 -25
  192. package/docs/wire-contracts.md +380 -0
  193. package/package.json +6 -1
  194. package/plugins/claude-code/.claude-plugin/marketplace.json +14 -0
  195. package/plugins/claude-code/plugin/.claude-plugin/plugin.json +8 -0
  196. package/plugins/claude-code/plugin/.mcp.json +14 -0
  197. package/plugins/claude-code/plugin/hooks/hooks.json +26 -0
  198. package/plugins/claude-code/plugin/hooks/session-end.sh +40 -0
  199. package/plugins/claude-code/plugin/hooks/session-start.sh +52 -0
  200. package/plugins/claude-code/plugin/skills/author/SKILL.md +181 -0
  201. package/plugins/claude-code/plugin/skills/conduct/SKILL.md +106 -0
  202. package/plugins/claude-code/plugin/skills/shift/SKILL.md +105 -0
  203. package/plugins/codex/.agents/plugins/marketplace.json +21 -0
  204. package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +7 -0
  205. package/plugins/codex/plugins/owenloop/.mcp.json +20 -0
  206. package/plugins/codex/plugins/owenloop/hooks/hooks.json +26 -0
  207. package/plugins/codex/plugins/owenloop/hooks/session-end.sh +40 -0
  208. package/plugins/codex/plugins/owenloop/hooks/session-start.sh +52 -0
  209. package/plugins/codex/plugins/owenloop/skills/author/SKILL.md +181 -0
  210. package/plugins/codex/plugins/owenloop/skills/conduct/SKILL.md +106 -0
  211. package/plugins/codex/plugins/owenloop/skills/shift/SKILL.md +105 -0
@@ -4,10 +4,10 @@
4
4
  * `createShiftLoop` is a standing Shift loop with every side-effecting
5
5
  * dependency injected (hub client, spawner, sleep/clock, output streams, dirs),
6
6
  * so the same core can run behind the local shift daemon. Per iteration it: pings
7
- * presence when due → cheap `wake(cursor)` pre-check → only when something
8
- * changed AND there is free capacity, sweeps `whats_next`, classifies/meters
9
- * orders, and DISPATCHES BY KIND → reaps work dirs → adopts the cursor → sleeps
10
- * the poll interval.
7
+ * presence when due → reconciles local children → dispatches any already-claimed
8
+ * orders waiting for local capacity → checks `wake(cursor)` → sweeps `whats_next`
9
+ * when the hub changed or a prior sweep was skipped/failed → classifies/meters
10
+ * new orders → reaps work dirs → adopts the cursor → sleeps the poll interval.
11
11
  *
12
12
  * DISPATCH SPLIT (D2 decision 3 — the D12 routing contract). BOTH kinds spawn a
13
13
  * detached child; there is exactly ONE way each kind runs, and no flag selects
@@ -15,9 +15,9 @@
15
15
  * - COMMAND orders spawn a detached `owenloop work exec <wf>/<run> --origin <url>`
16
16
  * child that self-leases (unchanged from C3).
17
17
  * - AGENT orders spawn a detached `owenloop work agent-run <wf>/<run>` child that
18
- * hosts the step agent itself through a harness adapter. It reads the step's
19
- * normalized spec straight out of the bundle cache and renders the brief in
20
- * process.
18
+ * hosts the step agent itself through a harness adapter. The child resolves the
19
+ * order-pinned instruction digest and selects the verified step and harness;
20
+ * Shift never substitutes definition-name cache metadata for modern orders.
21
21
  * - The shift makes NO first-contact `get_order` for any kind. It holds no
22
22
  * leases; the spawned child makes first contact and closes the B2 pickup
23
23
  * window. A failed hand-off lapses back via that window instead of sitting
@@ -25,24 +25,24 @@
25
25
  *
26
26
  * METERING (decision 3): free capacity `k = cap − liveInFlight`, where
27
27
  * liveInFlight counts every live child record, each one pid-probed (see
28
- * state.ts). When `k <= 0` the loop skips `whats_next` ENTIRELY (keeps wake +
29
- * presence). Metering is efficiency-only (engine race-safety + the hub's
30
- * pickup/lease TTLs are the correctness backstop), so this approximation is
31
- * acceptable.
28
+ * state.ts). When `k <= 0` the loop skips `whats_next` but remembers that the
29
+ * adopted wake still needs a sweep. Orders already returned by `whats_next`
30
+ * are already claimed, so over-cap orders enter an in-memory queue and dispatch
31
+ * as soon as local child capacity frees; waiting for another hub wake would
32
+ * strand those claims until pickup expiry.
32
33
  *
33
34
  * RESILIENCE: presence/wake/whats_next failures are logged and never kill the
34
35
  * loop. `stop()` flips a flag checked between awaits; the in-flight sweep
35
36
  * finishes, `run()` resolves 0, and no hub call is made afterward. Detached
36
37
  * children are never killed on shutdown — that is the drain semantic.
37
38
  *
38
- * PINNED-HASH DISPATCH (E, DD-4): `whats_next` serves only a def NAME (no hash,
39
- * no parent linkage), so the sweep resolves which cached bundle to serve via
40
- * `readDispatchBundle(cacheDir, defName)` instead of a bare latest read: 0 pins
41
- * → latest (unchanged); exactly 1 pinned hash → that frozen version (even when a
42
- * newer unpinned hash is cached); >1 conflicting pins across different cached
43
- * parents → refuse (no bundle + a warning), leave orders for the pickup window.
44
- * Everything downstream keys records by `bundle.def.hash`, so a pinned bundle
45
- * flows through `ChildRecord.hash` untouched.
39
+ * LEGACY PINNED-HASH DISPATCH (E, DD-4): the old wire shape omits both `worker`
40
+ * and `defDigest` and serves only a def NAME. Only that explicitly legacy shape
41
+ * uses `readDispatchBundle(cacheDir, defName)`: 0 pins → latest; exactly 1 pinned
42
+ * hash → that frozen version; >1 conflicting pins → refuse and leave the orders
43
+ * for the pickup window. Modern orders route from authoritative `worker` plus
44
+ * exact `defDigest`; modern command metadata is verified through
45
+ * `resolveOrderStep`, and modern agent metadata is resolved by `agent-run`.
46
46
  *
47
47
  * LIVE SHIFT IDENTITY: `name`/`serveCrews` on `ShiftLoopOptions` are INITIAL
48
48
  * values only. The loop holds them as live closure state (`getShift`/`setShift`
@@ -51,20 +51,39 @@
51
51
  * `setShift` also resets the presence timer so the new identity reaches the hub
52
52
  * on the very next `iterate()`.
53
53
  */
54
+ import { join } from 'node:path';
55
+ import { performance } from 'node:perf_hooks';
56
+ import { acquireFileLockSync, releaseFileLock } from "../../../../src/lock.js";
54
57
  import { readDispatchBundle } from "../bundle/cache.js";
58
+ import { HubError } from "../hub/types.js";
55
59
  import { isCommandStep, resolveCommandRouting } from "./routing.js";
56
- import { reconcileInFlight, writeChildRecord, removeChildRecord, ensureStateDir, } from "./state.js";
60
+ import { cancelReservedChild, finalizeChildReservation, reconcileInFlight, reserveChild, startReservedChild, removeChildRecord, ensureStateDir, } from "./state.js";
57
61
  import { DEFAULT_WORK_DIR_TTL_MS, sweepWorkDirs as sweepWorkDirsImpl } from "../agent/workdir.js";
58
62
  import { sessionsPath } from "../harness/session-store.js";
59
63
  /** The `maxConcurrentAgents` fallback when the option is absent. */
60
64
  const DEFAULT_MAX_AGENTS = 4;
65
+ /**
66
+ * The hub reaps a never-contacted claim after 120 seconds. Stop local queueing
67
+ * after 90 seconds so a detached child retains 30 seconds to make first contact.
68
+ */
69
+ export const MAX_PENDING_CANDIDATE_AGE_MS = 90_000;
61
70
  function errMsg(e) {
62
71
  return e instanceof Error ? e.message : String(e);
63
72
  }
73
+ /** The explicit hub refusal produced when an inbox row becomes terminal after
74
+ * the inbox snapshot but before its targeted serving call. This is a normal
75
+ * stale-candidate race, not a Shift failure worth retrying forever. */
76
+ function isNonServableRace(e) {
77
+ return e instanceof HubError
78
+ && e.status === 403
79
+ && e.code === 'forbidden'
80
+ && e.message.includes('non-running instance is not servable');
81
+ }
64
82
  export function createShiftLoop(opts) {
65
83
  let stopped = false;
66
84
  let cap = opts.cap;
67
85
  const isAlive = opts.isAlive;
86
+ const monotonicNow = opts.monotonicNow ?? performance.now.bind(performance);
68
87
  // Live shift identity (MCP `clock_in`, D3-D7 of the plan). Seeded from opts,
69
88
  // which are now INITIAL values only. Arrays are copied so neither the loop
70
89
  // nor a caller of getShift/setShift can mutate the other's state afterward.
@@ -74,6 +93,29 @@ export function createShiftLoop(opts) {
74
93
  let cursor;
75
94
  let lastPresence = Number.NEGATIVE_INFINITY;
76
95
  let attendedAt;
96
+ /**
97
+ * Orders returned by `whats_next` are already claimed. If the hub returns more
98
+ * orders than this process can spawn, retain those claims locally until a child
99
+ * exits and capacity opens. Waiting for another wake cannot work: no hub event
100
+ * is required when a local child exits, and `tick` will not re-return a claim
101
+ * that is already in flight.
102
+ */
103
+ const pendingCandidates = new Map();
104
+ /** A changed wake whose sweep was skipped or failed must be retried after the
105
+ * cursor is adopted; otherwise the next unchanged wake hides that work. */
106
+ let sweepOwed = false;
107
+ /** Earliest server-approved time for the next polling iteration. */
108
+ let backoffUntil = Number.NEGATIVE_INFINITY;
109
+ /** Bumped whenever a ping starts or is forced due, so a ping that completes
110
+ * after a force does not stamp the cadence over that force. */
111
+ let presenceGeneration = 0;
112
+ function noteServerBackoff(error) {
113
+ if (!(error instanceof HubError) || error.status !== 429)
114
+ return false;
115
+ const delay = error.retryAfterMs ?? opts.pollIntervalMs;
116
+ backoffUntil = Math.max(backoffUntil, monotonicNow() + delay);
117
+ return true;
118
+ }
77
119
  function emit(event) {
78
120
  if (opts.onEvent === undefined)
79
121
  return;
@@ -85,9 +127,10 @@ export function createShiftLoop(opts) {
85
127
  }
86
128
  }
87
129
  function reconcile() {
88
- // No clock is passed: reconciliation is purely pid-probed, so there is no
89
- // time-dependent decision for `opts.now` to influence.
90
- const result = reconcileInFlight(opts.stateDir, { ...(isAlive !== undefined ? { isAlive } : {}) });
130
+ const result = reconcileInFlight(opts.stateDir, {
131
+ ...(isAlive !== undefined ? { isAlive } : {}),
132
+ now: opts.now(),
133
+ });
91
134
  for (const rec of result.reaped) {
92
135
  emit({
93
136
  type: 'reaped',
@@ -97,8 +140,163 @@ export function createShiftLoop(opts) {
97
140
  pid: rec.pid,
98
141
  });
99
142
  }
143
+ for (const reservation of result.abandoned) {
144
+ opts.err(`[${reservation.workflow}/${reservation.run}] abandoned ${reservation.childKind} dispatch reservation expired and was cancelled`);
145
+ }
100
146
  return result;
101
147
  }
148
+ /**
149
+ * Recheck shared capacity and create the per-run reservation while holding one
150
+ * state-directory-wide lock. The lock ends before spawn: the durable
151
+ * reservation, not a long critical section, protects capacity during spawn.
152
+ */
153
+ function reserveCandidate(c) {
154
+ const childKind = c.kind === 'command' ? 'exec' : 'agent-run';
155
+ const dispatchLock = acquireFileLockSync(join(opts.stateDir, '.dispatch.lock'), {
156
+ waitMs: 30_000,
157
+ label: 'owenloop Shift dispatch',
158
+ });
159
+ try {
160
+ const fresh = reconcileInFlight(opts.stateDir, {
161
+ ...(isAlive === undefined ? {} : { isAlive }),
162
+ now: opts.now(),
163
+ });
164
+ if (fresh.live.some((record) => record.run === c.order.run) ||
165
+ fresh.reserved.some((record) => record.run === c.order.run))
166
+ return 'duplicate';
167
+ if (fresh.live.length + fresh.reserved.length >= cap)
168
+ return 'total-capacity';
169
+ if (childKind === 'agent-run') {
170
+ const agents = fresh.live.filter((record) => record.kind === 'agent-run').length +
171
+ fresh.reserved.filter((record) => record.childKind === 'agent-run').length;
172
+ if (agents >= (opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS))
173
+ return 'agent-capacity';
174
+ }
175
+ return reserveChild(opts.stateDir, {
176
+ workflow: c.workflow,
177
+ run: c.order.run,
178
+ reservedAt: opts.now(),
179
+ childKind,
180
+ step: c.order.step,
181
+ });
182
+ }
183
+ finally {
184
+ releaseFileLock(dispatchLock);
185
+ }
186
+ }
187
+ function dispatchCandidate(c) {
188
+ const childKind = c.kind === 'command' ? 'exec' : 'agent-run';
189
+ let reservation;
190
+ let cancel;
191
+ try {
192
+ const reserved = reserveCandidate(c);
193
+ if (typeof reserved === 'string')
194
+ return reserved;
195
+ reservation = reserved.reservation;
196
+ const spawned = opts.spawner({
197
+ workflow: c.workflow,
198
+ run: c.order.run,
199
+ step: c.order.step,
200
+ ...(childKind === 'agent-run' ? { kind: 'agent-run' } : {}),
201
+ startGate: reserved.gatePath,
202
+ });
203
+ cancel = spawned.cancel ?? spawned.terminate;
204
+ const rec = finalizeChildReservation(opts.stateDir, reservation, {
205
+ pid: spawned.pid,
206
+ spawnedAt: opts.now(),
207
+ kind: childKind,
208
+ ...(childKind === 'agent-run' && c.defName !== undefined ? { def: c.defName } : {}),
209
+ ...(childKind === 'agent-run' && c.defHash !== undefined ? { hash: c.defHash } : {}),
210
+ ...(childKind === 'agent-run' ? { step: c.order.step } : {}),
211
+ });
212
+ startReservedChild(opts.stateDir, rec);
213
+ emit({
214
+ type: 'dispatched',
215
+ workflow: c.workflow,
216
+ run: c.order.run,
217
+ step: c.order.step,
218
+ kind: childKind,
219
+ pid: spawned.pid,
220
+ });
221
+ opts.out(`dispatched ${c.kind === 'command' ? 'command' : 'agent-run'} ${c.workflow}/${c.order.run} ` +
222
+ `(step '${c.order.step}', pid ${spawned.pid})`);
223
+ return 'dispatched';
224
+ }
225
+ catch (e) {
226
+ cancel?.();
227
+ if (reservation !== undefined) {
228
+ try {
229
+ cancelReservedChild(opts.stateDir, reservation);
230
+ }
231
+ catch (cleanupError) {
232
+ opts.err(`[${c.workflow}/${c.order.run}] failed to cancel dispatch reservation: ${errMsg(cleanupError)}`);
233
+ }
234
+ }
235
+ const message = errMsg(e);
236
+ emit({
237
+ type: 'failed',
238
+ workflow: c.workflow,
239
+ run: c.order.run,
240
+ step: c.order.step,
241
+ kind: childKind,
242
+ message,
243
+ });
244
+ const prefix = c.kind === 'command' ? 'spawn' : 'agent-run spawn';
245
+ opts.err(`${prefix} for ${c.workflow}/${c.order.run} failed: ${message}`);
246
+ return 'failed';
247
+ }
248
+ }
249
+ function discardExpiredCandidate(candidate, queued) {
250
+ if (monotonicNow() - candidate.requestStartedAt < MAX_PENDING_CANDIDATE_AGE_MS)
251
+ return false;
252
+ const prefix = queued ? 'queued claim' : 'claim';
253
+ opts.err(`[${candidate.workflow}/${candidate.order.run}] ${prefix} expired before local dispatch — leaving the hub pickup window to re-offer it`);
254
+ return true;
255
+ }
256
+ /** Dispatch already-claimed orders when local child capacity becomes free. */
257
+ function drainPending(live, reserved) {
258
+ let remaining = cap - live.length - reserved.length;
259
+ if (remaining <= 0 || pendingCandidates.size === 0)
260
+ return 0;
261
+ const maxAgents = opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS;
262
+ let agentRoom = maxAgents -
263
+ live.filter((r) => r.kind === 'agent-run').length -
264
+ reserved.filter((r) => r.childKind === 'agent-run').length;
265
+ const liveRuns = new Set([...live.map((r) => r.run), ...reserved.map((r) => r.run)]);
266
+ let dispatched = 0;
267
+ for (const [run, candidate] of pendingCandidates) {
268
+ if (remaining <= 0)
269
+ break;
270
+ if (discardExpiredCandidate(candidate, true)) {
271
+ pendingCandidates.delete(run);
272
+ continue;
273
+ }
274
+ if (liveRuns.has(run)) {
275
+ pendingCandidates.delete(run);
276
+ continue;
277
+ }
278
+ // The agent lane's second budget can be full while the dispatch cap still
279
+ // has room. Skip this entry and KEEP it queued (a command entry further
280
+ // down the map is still dispatchable) — `continue`, never `break`, so one
281
+ // blocked agent does not head-of-line-block the whole queue.
282
+ if (candidate.kind === 'agent' && agentRoom <= 0)
283
+ continue;
284
+ const result = dispatchCandidate(candidate);
285
+ if (result === 'total-capacity')
286
+ break;
287
+ if (result === 'agent-capacity')
288
+ continue;
289
+ pendingCandidates.delete(run);
290
+ if (result !== 'dispatched')
291
+ continue;
292
+ dispatched++;
293
+ remaining--;
294
+ liveRuns.add(run);
295
+ if (candidate.kind === 'agent')
296
+ agentRoom--;
297
+ }
298
+ return dispatched;
299
+ }
102
300
  /**
103
301
  * One sweep. Collects dispatch candidates across the target instance(s)
104
302
  * within the free-capacity `budget`, then dispatches by kind: a command order
@@ -106,7 +304,7 @@ export function createShiftLoop(opts) {
106
304
  * `owenloop work agent-run` child. Returns how many children were spawned, plus the
107
305
  * polled/open-run sets the work-dir reaper needs.
108
306
  */
109
- async function sweep(budget, live) {
307
+ async function sweep(budget, live, reserved) {
110
308
  let dispatched = 0;
111
309
  /**
112
310
  * PHASE 4, for the work-dir reaper only. `polled` is the set of workflows
@@ -120,13 +318,16 @@ export function createShiftLoop(opts) {
120
318
  */
121
319
  const polled = new Set();
122
320
  const openRuns = new Set();
321
+ let complete = true;
123
322
  let remaining = budget;
124
- const liveRuns = new Set(live.map((r) => r.run));
125
- // The agent lane's SECOND budget (D7). Counted from the live records, so an
126
- // agent-run child that died frees its slot on the next reconcile exactly like
127
- // an exec child does.
323
+ const liveRuns = new Set([...live.map((r) => r.run), ...reserved.map((r) => r.run)]);
324
+ // The agent lane's SECOND budget (D7). Counted from live records and durable
325
+ // reservations, so a crash between reservation and spawn cannot overbook it.
326
+ // A child or reservation that is reaped frees the slot on the next reconcile.
128
327
  const maxAgents = opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS;
129
- let agentRoom = maxAgents - live.filter((r) => r.kind === 'agent-run').length;
328
+ let agentRoom = maxAgents -
329
+ live.filter((r) => r.kind === 'agent-run').length -
330
+ reserved.filter((r) => r.childKind === 'agent-run').length;
130
331
  /**
131
332
  * Runs that ALREADY have a live `agent-run` child. An agent-run child is a
132
333
  * PROCESS: re-dispatching one would put a second harness session on a single
@@ -134,7 +335,10 @@ export function createShiftLoop(opts) {
134
335
  * re-offer for a run a worker already holds is skipped outright, the way the
135
336
  * command lane does it.
136
337
  */
137
- const workerRuns = new Set(live.filter((r) => r.kind === 'agent-run').map((r) => r.run));
338
+ const workerRuns = new Set([
339
+ ...live.filter((r) => r.kind === 'agent-run').map((r) => r.run),
340
+ ...reserved.filter((r) => r.childKind === 'agent-run').map((r) => r.run),
341
+ ]);
138
342
  // Resolve which instances to poll.
139
343
  let instances;
140
344
  if (opts.workflow !== undefined) {
@@ -146,8 +350,9 @@ export function createShiftLoop(opts) {
146
350
  instances = (inbox.instances ?? []).map((i) => i.workflow);
147
351
  }
148
352
  catch (e) {
353
+ noteServerBackoff(e);
149
354
  opts.err(`inbox whats_next failed: ${errMsg(e)}`);
150
- return { dispatched, polled, openRuns };
355
+ return { dispatched, polled, openRuns, complete: false };
151
356
  }
152
357
  }
153
358
  // Collect candidates, metering NEW dispatches and deduping in-flight.
@@ -155,11 +360,25 @@ export function createShiftLoop(opts) {
155
360
  const candidates = [];
156
361
  for (const wf of instances) {
157
362
  let res;
363
+ const requestStartedAt = monotonicNow();
158
364
  try {
159
365
  res = await opts.hub.whatsNext({ workflow: wf, serve_crews: serveCrews });
160
366
  }
161
367
  catch (e) {
368
+ if (isNonServableRace(e)) {
369
+ // Treat the targeted call as a successful empty observation for the
370
+ // work-dir reaper and consume this wake. A later hub event may list
371
+ // new work, but this stale terminal row must not force every poll to
372
+ // repeat the same ForbiddenError.
373
+ polled.add(wf);
374
+ opts.out(`[${wf}] skipped stale terminal inbox candidate`);
375
+ continue;
376
+ }
377
+ const rateLimited = noteServerBackoff(e);
378
+ complete = false;
162
379
  opts.err(`whats_next for ${wf} failed: ${errMsg(e)}`);
380
+ if (rateLimited)
381
+ break;
163
382
  continue;
164
383
  }
165
384
  const orders = res.orders ?? [];
@@ -167,153 +386,171 @@ export function createShiftLoop(opts) {
167
386
  for (const o of orders)
168
387
  openRuns.add(o.run);
169
388
  const defName = res.def;
170
- // Dispatch selection (DD-4): honor a UNIQUE pinned hash for this def name;
171
- // 0 pins → latest (unchanged); conflicting pins → refuse (null + warning).
172
- const dispatch = defName !== undefined ? readDispatchBundle(opts.cacheDir, defName) : { bundle: null };
389
+ const legacyOrders = orders.filter((order) => !Object.prototype.hasOwnProperty.call(order, 'worker') &&
390
+ !Object.prototype.hasOwnProperty.call(order, 'defDigest'));
391
+ // The definition-name cache exists only for the explicitly identified old
392
+ // wire shape. Modern orders route from `worker` and exact `defDigest`.
393
+ const dispatch = legacyOrders.length > 0 && defName !== undefined
394
+ ? readDispatchBundle(opts.cacheDir, defName)
395
+ : { bundle: null };
173
396
  const bundle = dispatch.bundle;
174
- if (orders.length > 0 && dispatch.warning !== undefined) {
397
+ if (legacyOrders.length > 0 && dispatch.warning !== undefined) {
175
398
  opts.err(`[${wf}] ${dispatch.warning}`);
176
399
  }
177
- else if (orders.length > 0 && bundle === null && defName !== undefined) {
178
- opts.err(`no cached bundle for def '${defName}' — run \`owenloop work prepare ${defName}\` (agent orders left for pickup)`);
400
+ else if (legacyOrders.length > 0 && bundle === null && defName !== undefined) {
401
+ opts.err(`no cached bundle for def '${defName}' — run \`owenloop work prepare ${defName}\` (legacy orders left for pickup)`);
179
402
  }
180
403
  for (const order of orders) {
181
404
  if (claimed.has(order.run))
182
405
  continue; // seen this sweep
183
- const step = bundle?.def.steps.find((s) => s.name === order.step);
184
- const isCmd = step !== undefined && isCommandStep(step);
406
+ if (pendingCandidates.has(order.run))
407
+ continue; // already claimed and queued locally
185
408
  const reoffer = liveRuns.has(order.run);
186
- if (isCmd) {
187
- // A command already in flight (its exec child lives) is never respawned.
409
+ const workerPresent = Object.prototype.hasOwnProperty.call(order, 'worker');
410
+ const digestPresent = Object.prototype.hasOwnProperty.call(order, 'defDigest');
411
+ const legacy = !workerPresent && !digestPresent;
412
+ let step;
413
+ let kind;
414
+ let candidateDefName;
415
+ let candidateDefHash;
416
+ if (legacy) {
417
+ step = bundle?.def.steps.find((candidate) => candidate.name === order.step);
418
+ if (step === undefined || bundle === null)
419
+ continue;
420
+ kind = isCommandStep(step) ? 'command' : 'agent';
421
+ candidateDefName = defName;
422
+ candidateDefHash = bundle.def.hash;
423
+ }
424
+ else {
425
+ if (typeof order.defDigest !== 'string' || order.defDigest.trim() === '') {
426
+ opts.err(`[${wf}/${order.run}] malformed modern work order: non-empty defDigest is required whenever modern routing metadata is present — leaving for manual pickup`);
427
+ claimed.add(order.run);
428
+ continue;
429
+ }
430
+ // The deployed projection omits `worker` for the default executor. That
431
+ // shape is a modern AGENT order because `defDigest` pins the exact step;
432
+ // only an explicitly authored executor produces `worker` on the wire.
433
+ if (!workerPresent || order.worker === 'agent') {
434
+ kind = 'agent';
435
+ }
436
+ else if (order.worker === 'command') {
437
+ kind = 'command';
438
+ try {
439
+ step = await opts.resolveOrderStep?.(order);
440
+ }
441
+ catch (error) {
442
+ opts.err(`[${wf}/${order.run}] exact command instructions for digest '${order.defDigest}' failed verification: ${errMsg(error)} — leaving for manual pickup`);
443
+ claimed.add(order.run);
444
+ continue;
445
+ }
446
+ if (step === undefined) {
447
+ opts.err(`[${wf}/${order.run}] exact command routing metadata for digest '${order.defDigest}' is unavailable — leaving for manual pickup`);
448
+ claimed.add(order.run);
449
+ continue;
450
+ }
451
+ }
452
+ else if (typeof order.worker !== 'string' || order.worker.trim() === '') {
453
+ opts.err(`[${wf}/${order.run}] malformed modern work order: worker must be absent, 'agent', or 'command' — leaving for manual pickup`);
454
+ claimed.add(order.run);
455
+ continue;
456
+ }
457
+ else {
458
+ opts.err(`[${wf}/${order.run}] unsupported worker '${order.worker}' — leaving for manual pickup`);
459
+ claimed.add(order.run);
460
+ continue;
461
+ }
462
+ }
463
+ if (kind === 'command') {
464
+ // A command already in flight (or durably reserved) is never respawned.
188
465
  if (reoffer)
189
466
  continue;
190
- const r = resolveCommandRouting(opts.commandRouting, step);
191
- for (const w of r.warnings)
192
- opts.err(`[${wf}/${order.run}] ${w}`);
193
- if (!r.autoDispatch) {
194
- opts.out(`[${wf}/${order.run}] command step '${order.step}' routed to Shift — leaving for pickup window`);
467
+ if (step === undefined)
195
468
  continue;
469
+ const routing = resolveCommandRouting(opts.commandRouting, step);
470
+ for (const warning of routing.warnings)
471
+ opts.err(`[${wf}/${order.run}] ${warning}`);
472
+ if (!routing.autoDispatch) {
473
+ opts.out(`[${wf}/${order.run}] command step '${order.step}' routed to manual — leaving for pickup window`);
474
+ continue;
475
+ }
476
+ const candidate = {
477
+ order,
478
+ workflow: wf,
479
+ step,
480
+ defName: candidateDefName,
481
+ defHash: candidateDefHash,
482
+ kind,
483
+ requestStartedAt,
484
+ reoffer,
485
+ };
486
+ if (discardExpiredCandidate(candidate, false)) {
487
+ claimed.add(order.run);
488
+ continue;
489
+ }
490
+ if (remaining <= 0) {
491
+ pendingCandidates.set(order.run, candidate);
492
+ opts.out(`[${wf}/${order.run}] at the dispatch cap (${cap}) — queued for local dispatch`);
493
+ }
494
+ else {
495
+ remaining--;
496
+ candidates.push(candidate);
196
497
  }
197
- if (remaining <= 0)
198
- continue; // out of free capacity for new work
199
- remaining--;
200
- candidates.push({ order, workflow: wf, step, defName, defHash: bundle?.def.hash, kind: 'command', reoffer });
201
498
  claimed.add(order.run);
202
499
  continue;
203
500
  }
204
- // Agent order. An order a live worker already holds is never re-dispatched
205
- // (see `workerRuns`) — that would double-brief one claim.
501
+ // Agent wire routing is authoritative. The detached agent-run resolves its
502
+ // own exact step and harness from the order digest, never from this cache.
206
503
  if (workerRuns.has(order.run))
207
504
  continue;
208
- // No resolved bundle ⇒ no dispatch. Two distinct causes, one outcome:
209
- // - nothing cached for this def name (the operator never ran
210
- // `owenloop work prepare`), so there is no def+hash for the `agent-run`
211
- // child to read `steps/<step>.json` from; and
212
- // - conflicting pins across cached parents (DD-4), where refusing is
213
- // the point — spawning a child that picks its own bundle would make
214
- // the very version guess DD-4 forbids.
215
- // Either way the order is LEFT for the hub's pickup window. The one
216
- // warning per sweep was already written above.
217
- if (bundle === null)
505
+ const candidate = {
506
+ order,
507
+ workflow: wf,
508
+ step: legacy ? step : undefined,
509
+ defName: candidateDefName,
510
+ defHash: candidateDefHash,
511
+ kind,
512
+ requestStartedAt,
513
+ reoffer,
514
+ };
515
+ if (discardExpiredCandidate(candidate, false)) {
516
+ claimed.add(order.run);
218
517
  continue;
219
- // A re-offer replaces its (stale) record without consuming a new slot;
220
- // a genuinely new agent order needs free capacity.
221
- if (!reoffer) {
222
- if (remaining <= 0)
223
- continue;
224
- // The agent budget applies IN ADDITION to the global one. Skipped
225
- // candidates are simply left alone — the hub re-offers them on a later
226
- // sweep once a worker child exits.
227
- if (agentRoom <= 0) {
228
- opts.out(`[${wf}/${order.run}] at the agent-run cap (${maxAgents}) — leaving for a later sweep`);
229
- continue;
518
+ }
519
+ if (!reoffer && remaining <= 0) {
520
+ pendingCandidates.set(order.run, candidate);
521
+ opts.out(`[${wf}/${order.run}] at the dispatch cap (${cap}) — queued for local dispatch`);
522
+ }
523
+ else if (!reoffer && agentRoom <= 0) {
524
+ pendingCandidates.set(order.run, candidate);
525
+ opts.out(`[${wf}/${order.run}] at the agent-run cap (${maxAgents}) — queued for local dispatch`);
526
+ }
527
+ else {
528
+ if (!reoffer) {
529
+ remaining--;
530
+ agentRoom--;
230
531
  }
231
- remaining--;
232
- agentRoom--;
532
+ candidates.push(candidate);
233
533
  }
234
- candidates.push({ order, workflow: wf, step, defName, defHash: bundle?.def.hash, kind: 'agent', reoffer });
235
534
  claimed.add(order.run);
236
535
  }
237
536
  }
238
- if (candidates.length === 0)
239
- return { dispatched, polled, openRuns };
240
- // Dispatch by kind.
241
- for (const c of candidates) {
242
- if (c.kind === 'command') {
243
- try {
244
- const { pid } = opts.spawner({ workflow: c.workflow, run: c.order.run });
245
- const rec = { workflow: c.workflow, run: c.order.run, pid, spawnedAt: opts.now(), kind: 'exec' };
246
- writeChildRecord(opts.stateDir, rec);
247
- dispatched++;
248
- emit({
249
- type: 'dispatched',
250
- workflow: c.workflow,
251
- run: c.order.run,
252
- step: c.order.step,
253
- kind: 'exec',
254
- pid,
255
- });
256
- opts.out(`dispatched command ${c.workflow}/${c.order.run} (step '${c.order.step}', pid ${pid})`);
257
- }
258
- catch (e) {
259
- const message = errMsg(e);
260
- emit({
261
- type: 'failed',
262
- workflow: c.workflow,
263
- run: c.order.run,
264
- step: c.order.step,
265
- kind: 'exec',
266
- message,
267
- });
268
- opts.err(`spawn for ${c.workflow}/${c.order.run} failed: ${message}`);
269
- }
537
+ for (const candidate of candidates) {
538
+ if (discardExpiredCandidate(candidate, false))
270
539
  continue;
271
- }
272
- // Agent: spawn a detached `agent-run` child that hosts the step agent.
273
- // This is the ONLY agent path — no flag, no alternative.
274
- try {
275
- const { pid } = opts.spawner({
276
- workflow: c.workflow,
277
- run: c.order.run,
278
- kind: 'agent-run',
279
- ...(c.step?.harness !== undefined && c.step.harness !== '' ? { harness: c.step.harness } : {}),
280
- });
281
- const rec = {
282
- workflow: c.workflow,
283
- run: c.order.run,
284
- pid,
285
- spawnedAt: opts.now(),
286
- kind: 'agent-run',
287
- ...(c.defName !== undefined ? { def: c.defName } : {}),
288
- ...(c.defHash !== undefined ? { hash: c.defHash } : {}),
289
- step: c.order.step,
290
- };
291
- writeChildRecord(opts.stateDir, rec);
540
+ const result = dispatchCandidate(candidate);
541
+ if (result === 'dispatched') {
292
542
  dispatched++;
293
- emit({
294
- type: 'dispatched',
295
- workflow: c.workflow,
296
- run: c.order.run,
297
- step: c.order.step,
298
- kind: 'agent-run',
299
- pid,
300
- });
301
- opts.out(`dispatched agent-run ${c.workflow}/${c.order.run} (step '${c.order.step}', pid ${pid})`);
543
+ continue;
302
544
  }
303
- catch (e) {
304
- const message = errMsg(e);
305
- emit({
306
- type: 'failed',
307
- workflow: c.workflow,
308
- run: c.order.run,
309
- step: c.order.step,
310
- kind: 'agent-run',
311
- message,
312
- });
313
- opts.err(`agent-run spawn for ${c.workflow}/${c.order.run} failed: ${message}`);
545
+ if (result === 'total-capacity' || result === 'agent-capacity') {
546
+ pendingCandidates.set(candidate.order.run, candidate);
547
+ const limit = result === 'total-capacity'
548
+ ? `dispatch cap (${cap})`
549
+ : `agent-run cap (${opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS})`;
550
+ opts.out(`[${candidate.workflow}/${candidate.order.run}] lost a shared-capacity race at the ${limit} — queued for local dispatch`);
314
551
  }
315
552
  }
316
- return { dispatched, polled, openRuns };
553
+ return { dispatched, polled, openRuns, complete };
317
554
  }
318
555
  /**
319
556
  * PHASE 4 — remove the per-run work directories of runs that are over.
@@ -367,8 +604,20 @@ export function createShiftLoop(opts) {
367
604
  opts.out(`reaped work directory ${dir}`);
368
605
  }
369
606
  async function iteration() {
607
+ const backoffActiveAtStart = monotonicNow() < backoffUntil;
608
+ let rateLimitedThisIteration = false;
370
609
  // Presence when due (starts immediately — this shift exists to conduct).
371
- if (opts.now() - lastPresence >= opts.presenceIntervalMs) {
610
+ // A server backoff suppresses every hub poll, including presence, while local
611
+ // reconciliation and queued dispatch continue below.
612
+ if (!backoffActiveAtStart &&
613
+ monotonicNow() - lastPresence >= opts.presenceIntervalMs) {
614
+ // `setShift` and `noteAttended` force the next ping by setting
615
+ // lastPresence to -Infinity. Either can land WHILE this ping is awaiting
616
+ // the hub, so record the generation first: on completion, only stamp the
617
+ // cadence if no force arrived in the meantime. Stamping unconditionally
618
+ // would swallow that sentinel and defer the newly recorded attendance or
619
+ // identity by a full presenceIntervalMs.
620
+ const generation = ++presenceGeneration;
372
621
  try {
373
622
  await opts.hub.presencePing({
374
623
  name: shiftName,
@@ -377,32 +626,58 @@ export function createShiftLoop(opts) {
377
626
  ...(opts.startedAt !== undefined ? { started_at: opts.startedAt } : {}),
378
627
  ...(attendedAt !== undefined ? { attended_at: attendedAt } : {}),
379
628
  });
380
- lastPresence = opts.now();
629
+ if (generation === presenceGeneration)
630
+ lastPresence = monotonicNow();
381
631
  }
382
632
  catch (e) {
633
+ rateLimitedThisIteration = noteServerBackoff(e);
383
634
  opts.err(`presence ping failed: ${errMsg(e)} (continuing)`);
384
635
  }
385
636
  }
386
- // Reconcile in-flight (startup recovery + per-sweep reap).
387
- const { live } = reconcile();
388
- const k = cap - live.length;
389
- // Cheap wake pre-check. On failure, stay put this tick (don't sweep).
637
+ // Reconcile in-flight first. A child exit is local state, not a hub event,
638
+ // so use the freed slot to dispatch claims retained from an earlier sweep
639
+ // before consulting the wake cursor.
640
+ let inFlight = reconcile();
641
+ let dispatched = drainPending(inFlight.live, inFlight.reserved);
642
+ if (dispatched > 0)
643
+ inFlight = reconcile();
644
+ const live = inFlight.live;
645
+ const reserved = inFlight.reserved;
646
+ const occupied = live.length + reserved.length;
647
+ const k = cap - occupied;
648
+ // Retry-After suppresses every remaining hub call in the iteration that
649
+ // received it, and every hub call in later iterations before the deadline.
650
+ // Local child reconciliation and queued-claim dispatch above still run.
651
+ if (backoffActiveAtStart ||
652
+ rateLimitedThisIteration ||
653
+ monotonicNow() < backoffUntil) {
654
+ return dispatched;
655
+ }
656
+ // Cheap wake pre-check. A failed call also suppresses a forced retry for this
657
+ // tick: a rate-limited wake should not immediately be followed by whats_next.
390
658
  let changed = false;
659
+ let wakeSucceeded = false;
391
660
  try {
392
661
  const w = await opts.hub.wake(cursor);
393
662
  changed = w.changed;
394
663
  cursor = w.cursor;
664
+ wakeSucceeded = true;
395
665
  }
396
666
  catch (e) {
667
+ noteServerBackoff(e);
397
668
  opts.err(`wake failed: ${errMsg(e)} (retrying next tick)`);
398
669
  }
399
- // Sweep only when something changed AND there is free capacity.
670
+ // A changed cursor is not sufficient by itself: if the prior changed tick
671
+ // had no local capacity, or its sweep failed, the adopted cursor will be
672
+ // unchanged next time. `sweepOwed` preserves that unconsumed work signal.
400
673
  let swept;
401
- if (changed && k > 0) {
402
- swept = await sweep(k, live);
674
+ if (wakeSucceeded && (changed || sweepOwed) && k > 0) {
675
+ swept = await sweep(k, live, reserved);
676
+ sweepOwed = !swept.complete;
403
677
  }
404
- else if (changed) {
405
- opts.out(`at capacity (${live.length}/${cap} in flight) — skipping whats_next this tick`);
678
+ else if (changed && k <= 0) {
679
+ sweepOwed = true;
680
+ opts.out(`at capacity (${occupied}/${cap} in flight) — deferring whats_next until capacity is free`);
406
681
  }
407
682
  const liveAfter = reconcile().live;
408
683
  // PHASE 4 — the work-directory reaper.
@@ -415,7 +690,8 @@ export function createShiftLoop(opts) {
415
690
  // orders" and delete live work. Skipping costs nothing: the next sweeping
416
691
  // tick reaps whatever is still reapable.
417
692
  reapWorkDirs(swept, liveAfter);
418
- return swept?.dispatched ?? 0;
693
+ dispatched += swept?.dispatched ?? 0;
694
+ return dispatched;
419
695
  }
420
696
  async function run() {
421
697
  ensureStateDir(opts.stateDir);
@@ -429,7 +705,8 @@ export function createShiftLoop(opts) {
429
705
  await iteration();
430
706
  if (stopped)
431
707
  return 0;
432
- await opts.sleep(opts.pollIntervalMs);
708
+ const delay = Math.max(opts.pollIntervalMs, backoffUntil - monotonicNow());
709
+ await opts.sleep(delay);
433
710
  // Re-check after the park so a stop during sleep makes no further hub call.
434
711
  if (stopped)
435
712
  return 0;
@@ -442,7 +719,10 @@ export function createShiftLoop(opts) {
442
719
  run,
443
720
  stop,
444
721
  iterate: iteration,
445
- freeCapacity: () => cap - reconcile().live.length,
722
+ freeCapacity: () => {
723
+ const current = reconcile();
724
+ return cap - current.live.length - current.reserved.length;
725
+ },
446
726
  getCap: () => cap,
447
727
  setCap: (next) => {
448
728
  cap = next;
@@ -454,14 +734,17 @@ export function createShiftLoop(opts) {
454
734
  if (next.serveCrews !== undefined)
455
735
  serveCrews = [...next.serveCrews];
456
736
  lastPresence = Number.NEGATIVE_INFINITY; // D6: next iterate() pings immediately
737
+ presenceGeneration++; // survive an in-flight ping completing after this
457
738
  return { name: shiftName, serveCrews: [...serveCrews] };
458
739
  },
459
740
  noteAttended: (at) => {
460
741
  attendedAt = at;
461
742
  lastPresence = Number.NEGATIVE_INFINITY;
743
+ presenceGeneration++; // survive an in-flight ping completing after this
462
744
  },
463
745
  getAttendedAt: () => attendedAt,
464
746
  noteRunEnded: (run) => {
747
+ pendingCandidates.delete(run);
465
748
  removeChildRecord(opts.stateDir, run);
466
749
  },
467
750
  };