owenloop 0.5.0 → 0.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (229) hide show
  1. package/CHANGELOG.md +67 -0
  2. package/README.md +144 -35
  3. package/bin/owenloop.mjs +49 -1
  4. package/dist/packages/work/src/agent/brief.d.ts +29 -4
  5. package/dist/packages/work/src/agent/brief.js +49 -4
  6. package/dist/packages/work/src/agent/capability-model.d.ts +123 -0
  7. package/dist/packages/work/src/agent/capability-model.js +148 -0
  8. package/dist/packages/work/src/agent/loop.d.ts +56 -7
  9. package/dist/packages/work/src/agent/loop.js +166 -9
  10. package/dist/packages/work/src/bundle/cache.d.ts +7 -5
  11. package/dist/packages/work/src/bundle/cache.js +20 -6
  12. package/dist/packages/work/src/bundle/fetch.d.ts +7 -5
  13. package/dist/packages/work/src/bundle/fetch.js +27 -7
  14. package/dist/packages/work/src/bundle/types.d.ts +8 -4
  15. package/dist/packages/work/src/consumed-verifier.d.ts +45 -0
  16. package/dist/packages/work/src/consumed-verifier.js +205 -0
  17. package/dist/packages/work/src/exec/instructions.d.ts +99 -0
  18. package/dist/packages/work/src/exec/instructions.js +323 -0
  19. package/dist/packages/work/src/exec/loop.d.ts +29 -1
  20. package/dist/packages/work/src/exec/loop.js +313 -35
  21. package/dist/packages/work/src/exec/payload.d.ts +29 -0
  22. package/dist/packages/work/src/exec/payload.js +54 -0
  23. package/dist/packages/work/src/exec/receipt.d.ts +9 -6
  24. package/dist/packages/work/src/exec/receipt.js +12 -1
  25. package/dist/packages/work/src/exec/runner.d.ts +7 -1
  26. package/dist/packages/work/src/exec/runner.js +84 -3
  27. package/dist/packages/work/src/harness/child-env.js +7 -0
  28. package/dist/packages/work/src/harness/claude.d.ts +20 -8
  29. package/dist/packages/work/src/harness/claude.js +260 -58
  30. package/dist/packages/work/src/harness/codex.d.ts +5 -4
  31. package/dist/packages/work/src/harness/codex.js +169 -41
  32. package/dist/packages/work/src/harness/contract.d.ts +36 -11
  33. package/dist/packages/work/src/harness/fake.js +1 -0
  34. package/dist/packages/work/src/harness/jsonrpc-stdio.js +7 -0
  35. package/dist/packages/work/src/harness/permissions.d.ts +16 -3
  36. package/dist/packages/work/src/harness/permissions.js +102 -13
  37. package/dist/packages/work/src/harness/session-store.d.ts +30 -10
  38. package/dist/packages/work/src/harness/session-store.js +317 -89
  39. package/dist/packages/work/src/hold/mcp.d.ts +25 -5
  40. package/dist/packages/work/src/hold/mcp.js +134 -12
  41. package/dist/packages/work/src/hub/client.d.ts +8 -1
  42. package/dist/packages/work/src/hub/client.js +15 -1
  43. package/dist/packages/work/src/hub/types.d.ts +137 -33
  44. package/dist/packages/work/src/hub/types.js +8 -10
  45. package/dist/packages/work/src/mcp/server.js +1 -1
  46. package/dist/packages/work/src/owenloop-bin.d.ts +1 -0
  47. package/dist/packages/work/src/owenloop-bin.js +18 -0
  48. package/dist/packages/work/src/roles/agent-run.d.ts +12 -6
  49. package/dist/packages/work/src/roles/agent-run.js +115 -34
  50. package/dist/packages/work/src/roles/exec.d.ts +8 -0
  51. package/dist/packages/work/src/roles/exec.js +27 -2
  52. package/dist/packages/work/src/roles/hold.d.ts +6 -0
  53. package/dist/packages/work/src/roles/hold.js +83 -13
  54. package/dist/packages/work/src/roles/lint.d.ts +3 -4
  55. package/dist/packages/work/src/roles/lint.js +41 -7
  56. package/dist/packages/work/src/roles/settings.js +19 -2
  57. package/dist/packages/work/src/roles/shift.d.ts +1 -0
  58. package/dist/packages/work/src/roles/shift.js +36 -2
  59. package/dist/packages/work/src/settings/settings.d.ts +70 -1
  60. package/dist/packages/work/src/settings/settings.js +90 -4
  61. package/dist/packages/work/src/shift/logretention.d.ts +253 -0
  62. package/dist/packages/work/src/shift/logretention.js +407 -0
  63. package/dist/packages/work/src/shift/logsink.d.ts +14 -0
  64. package/dist/packages/work/src/shift/logsink.js +98 -0
  65. package/dist/packages/work/src/shift/loop.d.ts +32 -4
  66. package/dist/packages/work/src/shift/loop.js +687 -166
  67. package/dist/packages/work/src/shift/protocol.d.ts +128 -1
  68. package/dist/packages/work/src/shift/protocol.js +14 -0
  69. package/dist/packages/work/src/shift/runtime.d.ts +16 -0
  70. package/dist/packages/work/src/shift/runtime.js +291 -5
  71. package/dist/packages/work/src/shift/server.d.ts +13 -0
  72. package/dist/packages/work/src/shift/server.js +61 -36
  73. package/dist/packages/work/src/shift/spawn.d.ts +115 -19
  74. package/dist/packages/work/src/shift/spawn.js +234 -30
  75. package/dist/packages/work/src/shift/state.d.ts +76 -49
  76. package/dist/packages/work/src/shift/state.js +350 -103
  77. package/dist/packages/work/src/shift/truncate.d.ts +35 -0
  78. package/dist/packages/work/src/shift/truncate.js +73 -0
  79. package/dist/packages/work/src/submit-proof.d.ts +61 -0
  80. package/dist/packages/work/src/submit-proof.js +119 -0
  81. package/dist/packages/work/src/submit-value.d.ts +48 -0
  82. package/dist/packages/work/src/submit-value.js +118 -0
  83. package/dist/packages/work/src/usage.d.ts +1 -1
  84. package/dist/packages/work/src/usage.js +11 -0
  85. package/dist/src/add.d.ts +64 -259
  86. package/dist/src/add.js +81 -615
  87. package/dist/src/archive.d.ts +97 -0
  88. package/dist/src/archive.js +591 -0
  89. package/dist/src/bundle/index.d.ts +64 -0
  90. package/dist/src/bundle/index.js +508 -0
  91. package/dist/src/bundle/manifest.d.ts +50 -0
  92. package/dist/src/bundle/manifest.js +435 -0
  93. package/dist/src/bundle/runtime.d.ts +26 -0
  94. package/dist/src/bundle/runtime.js +64 -0
  95. package/dist/src/bundle/tar.d.ts +62 -0
  96. package/dist/src/bundle/tar.js +238 -0
  97. package/dist/src/bundle/types.d.ts +105 -0
  98. package/dist/src/bundle/types.js +21 -0
  99. package/dist/src/capabilities.d.ts +93 -0
  100. package/dist/src/capabilities.js +101 -0
  101. package/dist/src/cli.d.ts +90 -20
  102. package/dist/src/cli.js +2421 -374
  103. package/dist/src/credentials.d.ts +4 -3
  104. package/dist/src/credentials.js +21 -19
  105. package/dist/src/crypto/allowed-signers.d.ts +68 -0
  106. package/dist/src/crypto/allowed-signers.js +307 -0
  107. package/dist/src/crypto/canonical.d.ts +8 -0
  108. package/dist/src/crypto/canonical.js +92 -0
  109. package/dist/src/crypto/chain.d.ts +65 -0
  110. package/dist/src/crypto/chain.js +377 -0
  111. package/dist/src/crypto/dsse.d.ts +209 -0
  112. package/dist/src/crypto/dsse.js +282 -0
  113. package/dist/src/crypto/enrollment.d.ts +78 -0
  114. package/dist/src/crypto/enrollment.js +186 -0
  115. package/dist/src/crypto/index.d.ts +48 -0
  116. package/dist/src/crypto/index.js +44 -0
  117. package/dist/src/crypto/keys.d.ts +258 -0
  118. package/dist/src/crypto/keys.js +908 -0
  119. package/dist/src/crypto/org-root.d.ts +33 -0
  120. package/dist/src/crypto/org-root.js +78 -0
  121. package/dist/src/crypto/origin-rules.d.ts +56 -0
  122. package/dist/src/crypto/origin-rules.js +143 -0
  123. package/dist/src/crypto/policy-floor.d.ts +98 -0
  124. package/dist/src/crypto/policy-floor.js +353 -0
  125. package/dist/src/crypto/records.d.ts +319 -0
  126. package/dist/src/crypto/records.js +168 -0
  127. package/dist/src/crypto/scope.d.ts +49 -0
  128. package/dist/src/crypto/scope.js +83 -0
  129. package/dist/src/crypto/ssh.d.ts +171 -0
  130. package/dist/src/crypto/ssh.js +396 -0
  131. package/dist/src/crypto/submission.d.ts +33 -0
  132. package/dist/src/crypto/submission.js +47 -0
  133. package/dist/src/crypto/trust-roots.d.ts +29 -0
  134. package/dist/src/crypto/trust-roots.js +39 -0
  135. package/dist/src/crypto/verify-consumed.d.ts +67 -0
  136. package/dist/src/crypto/verify-consumed.js +259 -0
  137. package/dist/src/crypto/verify-origin.d.ts +50 -0
  138. package/dist/src/crypto/verify-origin.js +151 -0
  139. package/dist/src/crypto/verify-publication.d.ts +60 -0
  140. package/dist/src/crypto/verify-publication.js +150 -0
  141. package/dist/src/defs.d.ts +24 -1
  142. package/dist/src/defs.js +335 -27
  143. package/dist/src/engine.d.ts +211 -6
  144. package/dist/src/engine.js +459 -59
  145. package/dist/src/factory.d.ts +16 -0
  146. package/dist/src/factory.js +16 -4
  147. package/dist/src/global-config.d.ts +59 -0
  148. package/dist/src/global-config.js +84 -0
  149. package/dist/src/hub.d.ts +102 -32
  150. package/dist/src/hub.js +147 -58
  151. package/dist/src/index.d.ts +19 -2
  152. package/dist/src/index.js +23 -2
  153. package/dist/src/install.d.ts +538 -0
  154. package/dist/src/install.js +1981 -0
  155. package/dist/src/lock.d.ts +15 -57
  156. package/dist/src/lock.js +251 -143
  157. package/dist/src/mcp/serve.d.ts +46 -22
  158. package/dist/src/mcp/serve.js +174 -44
  159. package/dist/src/mcp/server.js +12 -1
  160. package/dist/src/model.d.ts +51 -0
  161. package/dist/src/model.js +157 -1
  162. package/dist/src/order-resolver.d.ts +184 -0
  163. package/dist/src/order-resolver.js +252 -0
  164. package/dist/src/package-version.d.ts +10 -0
  165. package/dist/src/package-version.js +42 -0
  166. package/dist/src/paths.d.ts +14 -1
  167. package/dist/src/paths.js +18 -0
  168. package/dist/src/schemas/enrollment-grant.v1.schema.json +90 -0
  169. package/dist/src/schemas/index.d.ts +1433 -0
  170. package/dist/src/schemas/index.js +31 -0
  171. package/dist/src/schemas/order.v1.schema.json +120 -0
  172. package/dist/src/schemas/origin.v1.schema.json +68 -0
  173. package/dist/src/schemas/policy-floor.v1.schema.json +29 -0
  174. package/dist/src/schemas/publication.v1.schema.json +31 -0
  175. package/dist/src/schemas/revocation.v1.schema.json +39 -0
  176. package/dist/src/schemas/submission.v1.schema.json +61 -0
  177. package/dist/src/store/def-source.d.ts +52 -0
  178. package/dist/src/store/def-source.js +335 -0
  179. package/dist/src/store/index-file.d.ts +69 -0
  180. package/dist/src/store/index-file.js +151 -0
  181. package/dist/src/store/index.d.ts +27 -0
  182. package/dist/src/store/index.js +22 -0
  183. package/dist/src/store/ingestor.d.ts +43 -0
  184. package/dist/src/store/ingestor.js +211 -0
  185. package/dist/src/store/install.d.ts +228 -0
  186. package/dist/src/store/install.js +555 -0
  187. package/dist/src/store/instruction-source.d.ts +49 -0
  188. package/dist/src/store/instruction-source.js +298 -0
  189. package/dist/src/store/origin-evidence.d.ts +21 -0
  190. package/dist/src/store/origin-evidence.js +66 -0
  191. package/dist/src/store/pre-commit-verifier.d.ts +68 -0
  192. package/dist/src/store/pre-commit-verifier.js +418 -0
  193. package/dist/src/store/resolve.d.ts +138 -0
  194. package/dist/src/store/resolve.js +363 -0
  195. package/dist/src/store/types.d.ts +288 -0
  196. package/dist/src/store/types.js +356 -0
  197. package/dist/src/store.d.ts +33 -1
  198. package/dist/src/store.js +57 -4
  199. package/dist/src/types.d.ts +146 -12
  200. package/dist/src/untar.d.ts +8 -23
  201. package/dist/src/untar.js +10 -136
  202. package/docs/agent-runner.md +86 -0
  203. package/docs/authoring.md +64 -6
  204. package/docs/bundles.md +350 -0
  205. package/docs/cli.md +1298 -106
  206. package/docs/crypto.md +832 -0
  207. package/docs/design.md +342 -20
  208. package/docs/embedding.md +132 -25
  209. package/docs/shift-logs.md +642 -0
  210. package/docs/wire-contracts.md +395 -0
  211. package/package.json +6 -1
  212. package/plugins/claude-code/.claude-plugin/marketplace.json +14 -0
  213. package/plugins/claude-code/plugin/.claude-plugin/plugin.json +8 -0
  214. package/plugins/claude-code/plugin/.mcp.json +14 -0
  215. package/plugins/claude-code/plugin/hooks/hooks.json +26 -0
  216. package/plugins/claude-code/plugin/hooks/session-end.sh +40 -0
  217. package/plugins/claude-code/plugin/hooks/session-start.sh +52 -0
  218. package/plugins/claude-code/plugin/skills/author/SKILL.md +181 -0
  219. package/plugins/claude-code/plugin/skills/conduct/SKILL.md +106 -0
  220. package/plugins/claude-code/plugin/skills/shift/SKILL.md +105 -0
  221. package/plugins/codex/.agents/plugins/marketplace.json +21 -0
  222. package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +7 -0
  223. package/plugins/codex/plugins/owenloop/.mcp.json +20 -0
  224. package/plugins/codex/plugins/owenloop/hooks/hooks.json +26 -0
  225. package/plugins/codex/plugins/owenloop/hooks/session-end.sh +40 -0
  226. package/plugins/codex/plugins/owenloop/hooks/session-start.sh +52 -0
  227. package/plugins/codex/plugins/owenloop/skills/author/SKILL.md +181 -0
  228. package/plugins/codex/plugins/owenloop/skills/conduct/SKILL.md +106 -0
  229. package/plugins/codex/plugins/owenloop/skills/shift/SKILL.md +105 -0
@@ -4,10 +4,10 @@
4
4
  * `createShiftLoop` is a standing Shift loop with every side-effecting
5
5
  * dependency injected (hub client, spawner, sleep/clock, output streams, dirs),
6
6
  * so the same core can run behind the local shift daemon. Per iteration it: pings
7
- * presence when due → cheap `wake(cursor)` pre-check → only when something
8
- * changed AND there is free capacity, sweeps `whats_next`, classifies/meters
9
- * orders, and DISPATCHES BY KIND → reaps work dirs → adopts the cursor → sleeps
10
- * the poll interval.
7
+ * presence when due → reconciles local children → dispatches any already-claimed
8
+ * orders waiting for local capacity → checks `wake(cursor)` → sweeps `whats_next`
9
+ * when the hub changed or a prior sweep was skipped/failed → classifies/meters
10
+ * new orders → reaps work dirs → adopts the cursor → sleeps the poll interval.
11
11
  *
12
12
  * DISPATCH SPLIT (D2 decision 3 — the D12 routing contract). BOTH kinds spawn a
13
13
  * detached child; there is exactly ONE way each kind runs, and no flag selects
@@ -15,9 +15,9 @@
15
15
  * - COMMAND orders spawn a detached `owenloop work exec <wf>/<run> --origin <url>`
16
16
  * child that self-leases (unchanged from C3).
17
17
  * - AGENT orders spawn a detached `owenloop work agent-run <wf>/<run>` child that
18
- * hosts the step agent itself through a harness adapter. It reads the step's
19
- * normalized spec straight out of the bundle cache and renders the brief in
20
- * process.
18
+ * hosts the step agent itself through a harness adapter. The child resolves the
19
+ * order-pinned instruction digest and selects the verified step and harness;
20
+ * Shift never substitutes definition-name cache metadata for modern orders.
21
21
  * - The shift makes NO first-contact `get_order` for any kind. It holds no
22
22
  * leases; the spawned child makes first contact and closes the B2 pickup
23
23
  * window. A failed hand-off lapses back via that window instead of sitting
@@ -25,24 +25,24 @@
25
25
  *
26
26
  * METERING (decision 3): free capacity `k = cap − liveInFlight`, where
27
27
  * liveInFlight counts every live child record, each one pid-probed (see
28
- * state.ts). When `k <= 0` the loop skips `whats_next` ENTIRELY (keeps wake +
29
- * presence). Metering is efficiency-only (engine race-safety + the hub's
30
- * pickup/lease TTLs are the correctness backstop), so this approximation is
31
- * acceptable.
28
+ * state.ts). When `k <= 0` the loop skips `whats_next` but remembers that the
29
+ * adopted wake still needs a sweep. Orders already returned by `whats_next`
30
+ * are already claimed, so over-cap orders enter an in-memory queue and dispatch
31
+ * as soon as local child capacity frees; waiting for another hub wake would
32
+ * strand those claims until pickup expiry.
32
33
  *
33
34
  * RESILIENCE: presence/wake/whats_next failures are logged and never kill the
34
35
  * loop. `stop()` flips a flag checked between awaits; the in-flight sweep
35
36
  * finishes, `run()` resolves 0, and no hub call is made afterward. Detached
36
37
  * children are never killed on shutdown — that is the drain semantic.
37
38
  *
38
- * PINNED-HASH DISPATCH (E, DD-4): `whats_next` serves only a def NAME (no hash,
39
- * no parent linkage), so the sweep resolves which cached bundle to serve via
40
- * `readDispatchBundle(cacheDir, defName)` instead of a bare latest read: 0 pins
41
- * → latest (unchanged); exactly 1 pinned hash → that frozen version (even when a
42
- * newer unpinned hash is cached); >1 conflicting pins across different cached
43
- * parents → refuse (no bundle + a warning), leave orders for the pickup window.
44
- * Everything downstream keys records by `bundle.def.hash`, so a pinned bundle
45
- * flows through `ChildRecord.hash` untouched.
39
+ * LEGACY PINNED-HASH DISPATCH (E, DD-4): the old wire shape omits both `worker`
40
+ * and `defDigest` and serves only a def NAME. Only that explicitly legacy shape
41
+ * uses `readDispatchBundle(cacheDir, defName)`: 0 pins → latest; exactly 1 pinned
42
+ * hash → that frozen version; >1 conflicting pins → refuse and leave the orders
43
+ * for the pickup window. Modern orders route from authoritative `worker` plus
44
+ * exact `defDigest`; modern command metadata is verified through
45
+ * `resolveOrderStep`, and modern agent metadata is resolved by `agent-run`.
46
46
  *
47
47
  * LIVE SHIFT IDENTITY: `name`/`serveCrews` on `ShiftLoopOptions` are INITIAL
48
48
  * values only. The loop holds them as live closure state (`getShift`/`setShift`
@@ -51,20 +51,103 @@
51
51
  * `setShift` also resets the presence timer so the new identity reaches the hub
52
52
  * on the very next `iterate()`.
53
53
  */
54
+ import { join } from 'node:path';
55
+ import { performance } from 'node:perf_hooks';
56
+ import { acquireFileLockSync, releaseFileLock } from "../../../../src/lock.js";
54
57
  import { readDispatchBundle } from "../bundle/cache.js";
58
+ import { HubError } from "../hub/types.js";
55
59
  import { isCommandStep, resolveCommandRouting } from "./routing.js";
56
- import { reconcileInFlight, writeChildRecord, removeChildRecord, ensureStateDir, } from "./state.js";
60
+ import { cancelReservedChild, finalizeChildReservation, reconcileInFlight, reserveChild, startReservedChild, removeChildRecord, ensureStateDir, } from "./state.js";
57
61
  import { DEFAULT_WORK_DIR_TTL_MS, sweepWorkDirs as sweepWorkDirsImpl } from "../agent/workdir.js";
58
62
  import { sessionsPath } from "../harness/session-store.js";
63
+ import { stampShiftEvent, } from "./protocol.js";
59
64
  /** The `maxConcurrentAgents` fallback when the option is absent. */
60
65
  const DEFAULT_MAX_AGENTS = 4;
66
+ /**
67
+ * The hub reaps a never-contacted claim after 120 seconds. Stop local queueing
68
+ * after 90 seconds so a detached child retains 30 seconds to make first contact.
69
+ */
70
+ export const MAX_PENDING_CANDIDATE_AGE_MS = 90_000;
71
+ /**
72
+ * THE PER-STEP FAILURE BRAKE.
73
+ *
74
+ * Every in-flight guard in this file keys on the RUN id — `liveRuns`,
75
+ * `workerRuns`, `claimed`, `pendingCandidates`. That is correct for the case
76
+ * they were written for: one claim, one child. It is blind to the case where a
77
+ * step fails the same way forever.
78
+ *
79
+ * Observed 2026-08-12: `local-check` of one instance was dispatched five-plus
80
+ * times in seconds under a DIFFERENT run id each time
81
+ * (run_fd94d88a…, run_5ec4d807…, run_d8cdf03f…, run_67c8e2db…, run_072755ba…).
82
+ * The child refused its consumed artifact, exited non-zero WITHOUT submitting,
83
+ * the hub released the claim, and the next poll handed the same step back under
84
+ * a fresh run id. A fresh run id matches none of the run-keyed guards, so
85
+ * nothing bounded the respawns.
86
+ *
87
+ * So the brake is keyed on the identity that actually repeats — the STEP, as
88
+ * `(workflow, step, key)`. `key` is part of the identity because a fanned-out
89
+ * step has one legitimately concurrent order per key, and those must not brake
90
+ * each other.
91
+ *
92
+ * WHAT IS COUNTED, AND WHY IT IS NOT DISPATCHES. The event counted against that
93
+ * key is a WORKER FAILURE — a child that exited non-zero, reported into the
94
+ * loop through `noteWorkerFailure` — never a dispatch. Counting dispatches
95
+ * conflates a storm with legitimate concurrency: the hub can offer several
96
+ * DISTINCT runs of one step in a single sweep, and metering the second through
97
+ * fifth of those would throttle healthy work. A failure is direct evidence that
98
+ * running this step again right now is unlikely to help; a dispatch is evidence
99
+ * of nothing.
100
+ *
101
+ * It is a rate limit, never a ban. A step that has not failed is dispatched
102
+ * with zero delay, however many orders arrive for it at once.
103
+ */
104
+ const STEP_BRAKE_DELAYS_MS = [2_000, 8_000, 30_000, 120_000, 300_000];
105
+ /**
106
+ * Treat a failure this long after the previous one as the start of a NEW
107
+ * streak, resetting the count to zero first. Comfortably longer than the
108
+ * longest delay in the table (300s), so a step failing at maximum backoff never
109
+ * decays out of it, while an isolated transient long after an old streak starts
110
+ * again at the shortest delay instead of inheriting a stale penalty.
111
+ */
112
+ const STEP_BRAKE_DECAY_MS = 600_000;
113
+ /**
114
+ * Forget a step's brake entry once this long has passed since its window
115
+ * closed, so the map cannot grow for the life of the shift. Pure housekeeping —
116
+ * `STEP_BRAKE_DECAY_MS` is what governs behaviour.
117
+ */
118
+ const STEP_BRAKE_FORGET_MS = 1_800_000;
119
+ /**
120
+ * How long to refuse the next dispatch of a step after `count` consecutive
121
+ * worker failures. `count` is at least 1 at every call site (this is reached
122
+ * only from the failure path), so the FIRST failure already costs 2s; past the
123
+ * end of the table the last entry repeats, so a permanently-failing step
124
+ * settles at one dispatch per five minutes instead of one per poll.
125
+ */
126
+ function stepBrakeDelayMs(count) {
127
+ const index = Math.min(count, STEP_BRAKE_DELAYS_MS.length) - 1;
128
+ return STEP_BRAKE_DELAYS_MS[index] ?? 0;
129
+ }
130
+ /** The identity a storm repeats on. `key` is '' for an unfanned step. */
131
+ function stepBrakeKey(workflow, step, key) {
132
+ return `${workflow}\u0000${step}\u0000${key ?? ''}`;
133
+ }
61
134
  function errMsg(e) {
62
135
  return e instanceof Error ? e.message : String(e);
63
136
  }
137
+ /** The explicit hub refusal produced when an inbox row becomes terminal after
138
+ * the inbox snapshot but before its targeted serving call. This is a normal
139
+ * stale-candidate race, not a Shift failure worth retrying forever. */
140
+ function isNonServableRace(e) {
141
+ return e instanceof HubError
142
+ && e.status === 403
143
+ && e.code === 'forbidden'
144
+ && e.message.includes('non-running instance is not servable');
145
+ }
64
146
  export function createShiftLoop(opts) {
65
147
  let stopped = false;
66
148
  let cap = opts.cap;
67
149
  const isAlive = opts.isAlive;
150
+ const monotonicNow = opts.monotonicNow ?? performance.now.bind(performance);
68
151
  // Live shift identity (MCP `clock_in`, D3-D7 of the plan). Seeded from opts,
69
152
  // which are now INITIAL values only. Arrays are copied so neither the loop
70
153
  // nor a caller of getShift/setShift can mutate the other's state afterward.
@@ -74,20 +157,67 @@ export function createShiftLoop(opts) {
74
157
  let cursor;
75
158
  let lastPresence = Number.NEGATIVE_INFINITY;
76
159
  let attendedAt;
77
- function emit(event) {
160
+ /**
161
+ * Orders returned by `whats_next` are already claimed. If the hub returns more
162
+ * orders than this process can spawn, retain those claims locally until a child
163
+ * exits and capacity opens. Waiting for another wake cannot work: no hub event
164
+ * is required when a local child exits, and `tick` will not re-return a claim
165
+ * that is already in flight.
166
+ */
167
+ const pendingCandidates = new Map();
168
+ /** A changed wake whose sweep was skipped or failed must be retried after the
169
+ * cursor is adopted; otherwise the next unchanged wake hides that work. */
170
+ let sweepOwed = false;
171
+ /**
172
+ * Has the current at-capacity EPISODE already produced a `capacity` event?
173
+ *
174
+ * Set when the event fires, cleared the moment a slot is free again, so one
175
+ * unbroken stretch at capacity yields exactly one record no matter how many
176
+ * ticks it spans. See the emit site for why the console line next to it is
177
+ * deliberately NOT edge-triggered.
178
+ */
179
+ let atCapReported = false;
180
+ /** Earliest server-approved time for the next polling iteration. */
181
+ let backoffUntil = Number.NEGATIVE_INFINITY;
182
+ /** Bumped whenever a ping starts or is forced due, so a ping that completes
183
+ * after a force does not stamp the cadence over that force. */
184
+ let presenceGeneration = 0;
185
+ function noteServerBackoff(error) {
186
+ if (!(error instanceof HubError) || error.status !== 429)
187
+ return false;
188
+ const delay = error.retryAfterMs ?? opts.pollIntervalMs;
189
+ backoffUntil = Math.max(backoffUntil, monotonicNow() + delay);
190
+ return true;
191
+ }
192
+ /**
193
+ * Fan one observation to every consumer, stamped.
194
+ *
195
+ * THE SINGLE ENVELOPE-STAMPING POINT for everything the loop observes. Call
196
+ * sites build a `ShiftEventBody` — payload only — and this function adds the
197
+ * timestamp and the shift identity, so no construction site can forget them
198
+ * and no consumer can receive a half-stamped record. `shiftName` is read live
199
+ * rather than captured, because `clock_in` can rename a shift mid-run and the
200
+ * record should say what the shift was called when it was written.
201
+ *
202
+ * A THROWING CONSUMER MUST NOT KILL THE SHIFT. The report goes to `opts.err`
203
+ * as free text and stays free text: it is the sink's own failure report, and
204
+ * turning it into an event would write it to the sink that just failed.
205
+ */
206
+ function emit(body) {
78
207
  if (opts.onEvent === undefined)
79
208
  return;
80
209
  try {
81
- opts.onEvent(event);
210
+ opts.onEvent(stampShiftEvent(body, { name: shiftName, id: opts.shiftId ?? '' }, opts.now()));
82
211
  }
83
212
  catch (e) {
84
213
  opts.err(`shift event sink failed: ${errMsg(e)} (continuing)`);
85
214
  }
86
215
  }
87
216
  function reconcile() {
88
- // No clock is passed: reconciliation is purely pid-probed, so there is no
89
- // time-dependent decision for `opts.now` to influence.
90
- const result = reconcileInFlight(opts.stateDir, { ...(isAlive !== undefined ? { isAlive } : {}) });
217
+ const result = reconcileInFlight(opts.stateDir, {
218
+ ...(isAlive !== undefined ? { isAlive } : {}),
219
+ now: opts.now(),
220
+ });
91
221
  for (const rec of result.reaped) {
92
222
  emit({
93
223
  type: 'reaped',
@@ -97,8 +227,210 @@ export function createShiftLoop(opts) {
97
227
  pid: rec.pid,
98
228
  });
99
229
  }
230
+ for (const reservation of result.abandoned) {
231
+ opts.err(`[${reservation.workflow}/${reservation.run}] abandoned ${reservation.childKind} dispatch reservation expired and was cancelled`);
232
+ }
100
233
  return result;
101
234
  }
235
+ /**
236
+ * Per-step failure history for the brake. Keyed by `(workflow, step, key)` —
237
+ * see `stepBrakeKey`. `count` is the current consecutive-failure streak for
238
+ * that step, `failedAt` the monotonic instant of the most recent failure (the
239
+ * input to the `STEP_BRAKE_DECAY_MS` streak reset), and `nextAllowedAt` the
240
+ * monotonic instant the next spawn of that step becomes permitted.
241
+ */
242
+ const stepBrake = new Map();
243
+ /**
244
+ * Brake key of the step each dispatched run belongs to.
245
+ *
246
+ * A `WorkerFailure` carries the run id and the step name but NOT the fan-out
247
+ * key, and the brake identity includes the key — so the mapping is recorded
248
+ * here at dispatch, when the whole order is in hand, rather than
249
+ * reconstructed from the failure. Written synchronously in
250
+ * `dispatchCandidate` immediately after the spawn returns, which always
251
+ * precedes the child's `exit` event (that fires on a later tick), so a
252
+ * failure can never arrive before its own entry exists.
253
+ *
254
+ * Entries are removed when the run's failure is noted, when the run ends
255
+ * (`noteRunEnded`), and for any run no longer live at sweep time — so this
256
+ * map tracks live dispatches only and cannot grow without bound.
257
+ */
258
+ const runBrakeKey = new Map();
259
+ /**
260
+ * Recheck shared capacity and create the per-run reservation while holding one
261
+ * state-directory-wide lock. The lock ends before spawn: the durable
262
+ * reservation, not a long critical section, protects capacity during spawn.
263
+ */
264
+ function reserveCandidate(c) {
265
+ const childKind = c.kind === 'command' ? 'exec' : 'agent-run';
266
+ const dispatchLock = acquireFileLockSync(join(opts.stateDir, '.dispatch.lock'), {
267
+ waitMs: 30_000,
268
+ label: 'owenloop Shift dispatch',
269
+ });
270
+ try {
271
+ const fresh = reconcileInFlight(opts.stateDir, {
272
+ ...(isAlive === undefined ? {} : { isAlive }),
273
+ now: opts.now(),
274
+ });
275
+ if (fresh.live.some((record) => record.run === c.order.run) ||
276
+ fresh.reserved.some((record) => record.run === c.order.run))
277
+ return 'duplicate';
278
+ if (fresh.live.length + fresh.reserved.length >= cap)
279
+ return 'total-capacity';
280
+ if (childKind === 'agent-run') {
281
+ const agents = fresh.live.filter((record) => record.kind === 'agent-run').length +
282
+ fresh.reserved.filter((record) => record.childKind === 'agent-run').length;
283
+ if (agents >= (opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS))
284
+ return 'agent-capacity';
285
+ }
286
+ return reserveChild(opts.stateDir, {
287
+ workflow: c.workflow,
288
+ run: c.order.run,
289
+ reservedAt: opts.now(),
290
+ childKind,
291
+ step: c.order.step,
292
+ });
293
+ }
294
+ finally {
295
+ releaseFileLock(dispatchLock);
296
+ }
297
+ }
298
+ function dispatchCandidate(c) {
299
+ const childKind = c.kind === 'command' ? 'exec' : 'agent-run';
300
+ // THE BRAKE. This function is the single choke point every spawn passes
301
+ // through — both the sweep's dispatch loop and `drainPending` reach a child
302
+ // only through here — so gating it here cannot be bypassed by either path.
303
+ // Braking BEFORE `reserveCandidate` means no reservation, no state-dir
304
+ // write and no lock contention for an order this shift will not run. The
305
+ // claim already taken by `whats_next` is deliberately left to lapse: the
306
+ // hub reaps a never-contacted claim after 120s, which throttles the
307
+ // re-offer rate on the hub side too.
308
+ const brakeKey = stepBrakeKey(c.workflow, c.order.step, c.order.key);
309
+ const brake = stepBrake.get(brakeKey);
310
+ if (brake !== undefined && monotonicNow() < brake.nextAllowedAt) {
311
+ const waitMs = Math.round(brake.nextAllowedAt - monotonicNow());
312
+ opts.err(`[${c.workflow}/${c.order.run}] step '${c.order.step}' has failed ` +
313
+ `${brake.count} time(s) in a row — braking for ${waitMs}ms and ` +
314
+ `leaving this claim to lapse`);
315
+ return 'braked';
316
+ }
317
+ let reservation;
318
+ let cancel;
319
+ try {
320
+ const reserved = reserveCandidate(c);
321
+ if (typeof reserved === 'string')
322
+ return reserved;
323
+ reservation = reserved.reservation;
324
+ const spawned = opts.spawner({
325
+ workflow: c.workflow,
326
+ run: c.order.run,
327
+ step: c.order.step,
328
+ ...(childKind === 'agent-run' ? { kind: 'agent-run' } : {}),
329
+ startGate: reserved.gatePath,
330
+ });
331
+ cancel = spawned.cancel ?? spawned.terminate;
332
+ const rec = finalizeChildReservation(opts.stateDir, reservation, {
333
+ pid: spawned.pid,
334
+ spawnedAt: opts.now(),
335
+ kind: childKind,
336
+ ...(childKind === 'agent-run' && c.defName !== undefined ? { def: c.defName } : {}),
337
+ ...(childKind === 'agent-run' && c.defHash !== undefined ? { hash: c.defHash } : {}),
338
+ ...(childKind === 'agent-run' ? { step: c.order.step } : {}),
339
+ });
340
+ startReservedChild(opts.stateDir, rec);
341
+ // Remember which step this run belongs to, so a later worker failure —
342
+ // which names the run but not the fan-out key — can be charged to the
343
+ // right brake key. NOTHING is counted here: a dispatch is not evidence of
344
+ // trouble, and counting it would meter the legitimately concurrent runs
345
+ // of one step that a single sweep can offer.
346
+ runBrakeKey.set(c.order.run, brakeKey);
347
+ emit({
348
+ type: 'dispatched',
349
+ workflow: c.workflow,
350
+ run: c.order.run,
351
+ step: c.order.step,
352
+ kind: childKind,
353
+ pid: spawned.pid,
354
+ });
355
+ opts.out(`dispatched ${c.kind === 'command' ? 'command' : 'agent-run'} ${c.workflow}/${c.order.run} ` +
356
+ `(step '${c.order.step}', pid ${spawned.pid})`);
357
+ return 'dispatched';
358
+ }
359
+ catch (e) {
360
+ cancel?.();
361
+ if (reservation !== undefined) {
362
+ try {
363
+ cancelReservedChild(opts.stateDir, reservation);
364
+ }
365
+ catch (cleanupError) {
366
+ opts.err(`[${c.workflow}/${c.order.run}] failed to cancel dispatch reservation: ${errMsg(cleanupError)}`);
367
+ }
368
+ }
369
+ const message = errMsg(e);
370
+ emit({
371
+ type: 'failed',
372
+ workflow: c.workflow,
373
+ run: c.order.run,
374
+ step: c.order.step,
375
+ kind: childKind,
376
+ message,
377
+ });
378
+ const prefix = c.kind === 'command' ? 'spawn' : 'agent-run spawn';
379
+ opts.err(`${prefix} for ${c.workflow}/${c.order.run} failed: ${message}`);
380
+ return 'failed';
381
+ }
382
+ }
383
+ function discardExpiredCandidate(candidate, queued) {
384
+ if (monotonicNow() - candidate.requestStartedAt < MAX_PENDING_CANDIDATE_AGE_MS)
385
+ return false;
386
+ const prefix = queued ? 'queued claim' : 'claim';
387
+ opts.err(`[${candidate.workflow}/${candidate.order.run}] ${prefix} expired before local dispatch — leaving the hub pickup window to re-offer it`);
388
+ return true;
389
+ }
390
+ /** Dispatch already-claimed orders when local child capacity becomes free. */
391
+ function drainPending(live, reserved) {
392
+ let remaining = cap - live.length - reserved.length;
393
+ if (remaining <= 0 || pendingCandidates.size === 0)
394
+ return 0;
395
+ const maxAgents = opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS;
396
+ let agentRoom = maxAgents -
397
+ live.filter((r) => r.kind === 'agent-run').length -
398
+ reserved.filter((r) => r.childKind === 'agent-run').length;
399
+ const liveRuns = new Set([...live.map((r) => r.run), ...reserved.map((r) => r.run)]);
400
+ let dispatched = 0;
401
+ for (const [run, candidate] of pendingCandidates) {
402
+ if (remaining <= 0)
403
+ break;
404
+ if (discardExpiredCandidate(candidate, true)) {
405
+ pendingCandidates.delete(run);
406
+ continue;
407
+ }
408
+ if (liveRuns.has(run)) {
409
+ pendingCandidates.delete(run);
410
+ continue;
411
+ }
412
+ // The agent lane's second budget can be full while the dispatch cap still
413
+ // has room. Skip this entry and KEEP it queued (a command entry further
414
+ // down the map is still dispatchable) — `continue`, never `break`, so one
415
+ // blocked agent does not head-of-line-block the whole queue.
416
+ if (candidate.kind === 'agent' && agentRoom <= 0)
417
+ continue;
418
+ const result = dispatchCandidate(candidate);
419
+ if (result === 'total-capacity')
420
+ break;
421
+ if (result === 'agent-capacity')
422
+ continue;
423
+ pendingCandidates.delete(run);
424
+ if (result !== 'dispatched')
425
+ continue;
426
+ dispatched++;
427
+ remaining--;
428
+ liveRuns.add(run);
429
+ if (candidate.kind === 'agent')
430
+ agentRoom--;
431
+ }
432
+ return dispatched;
433
+ }
102
434
  /**
103
435
  * One sweep. Collects dispatch candidates across the target instance(s)
104
436
  * within the free-capacity `budget`, then dispatches by kind: a command order
@@ -106,8 +438,30 @@ export function createShiftLoop(opts) {
106
438
  * `owenloop work agent-run` child. Returns how many children were spawned, plus the
107
439
  * polled/open-run sets the work-dir reaper needs.
108
440
  */
109
- async function sweep(budget, live) {
441
+ async function sweep(budget, live, reserved) {
110
442
  let dispatched = 0;
443
+ // Housekeeping for both brake maps, once per sweep rather than on write, so
444
+ // neither can grow for the life of a long-lived shift.
445
+ //
446
+ // `stepBrake`: forget an entry whose window closed STEP_BRAKE_FORGET_MS
447
+ // ago. Behaviour is already governed by STEP_BRAKE_DECAY_MS, which resets a
448
+ // stale streak; this only reclaims the memory.
449
+ //
450
+ // `runBrakeKey`: drop every run that is no longer live. A run whose child
451
+ // exited CLEANLY reports no failure, so nothing else would ever remove its
452
+ // entry. A non-zero exit is charged by `noteWorkerFailure`, which the
453
+ // child's own `exit` event drives on the next event-loop tick — always well
454
+ // before the next sweep, which waits a whole poll interval. If that order
455
+ // ever did invert, the cost is one uncharged failure, never a wrong charge.
456
+ for (const [key, entry] of stepBrake) {
457
+ if (monotonicNow() - entry.nextAllowedAt > STEP_BRAKE_FORGET_MS)
458
+ stepBrake.delete(key);
459
+ }
460
+ const liveOrReserved = new Set([...live.map((r) => r.run), ...reserved.map((r) => r.run)]);
461
+ for (const run of runBrakeKey.keys()) {
462
+ if (!liveOrReserved.has(run))
463
+ runBrakeKey.delete(run);
464
+ }
111
465
  /**
112
466
  * PHASE 4, for the work-dir reaper only. `polled` is the set of workflows
113
467
  * whose `whats_next` SUCCEEDED this sweep — a workflow whose call threw is
@@ -120,13 +474,16 @@ export function createShiftLoop(opts) {
120
474
  */
121
475
  const polled = new Set();
122
476
  const openRuns = new Set();
477
+ let complete = true;
123
478
  let remaining = budget;
124
- const liveRuns = new Set(live.map((r) => r.run));
125
- // The agent lane's SECOND budget (D7). Counted from the live records, so an
126
- // agent-run child that died frees its slot on the next reconcile exactly like
127
- // an exec child does.
479
+ const liveRuns = new Set([...live.map((r) => r.run), ...reserved.map((r) => r.run)]);
480
+ // The agent lane's SECOND budget (D7). Counted from live records and durable
481
+ // reservations, so a crash between reservation and spawn cannot overbook it.
482
+ // A child or reservation that is reaped frees the slot on the next reconcile.
128
483
  const maxAgents = opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS;
129
- let agentRoom = maxAgents - live.filter((r) => r.kind === 'agent-run').length;
484
+ let agentRoom = maxAgents -
485
+ live.filter((r) => r.kind === 'agent-run').length -
486
+ reserved.filter((r) => r.childKind === 'agent-run').length;
130
487
  /**
131
488
  * Runs that ALREADY have a live `agent-run` child. An agent-run child is a
132
489
  * PROCESS: re-dispatching one would put a second harness session on a single
@@ -134,7 +491,10 @@ export function createShiftLoop(opts) {
134
491
  * re-offer for a run a worker already holds is skipped outright, the way the
135
492
  * command lane does it.
136
493
  */
137
- const workerRuns = new Set(live.filter((r) => r.kind === 'agent-run').map((r) => r.run));
494
+ const workerRuns = new Set([
495
+ ...live.filter((r) => r.kind === 'agent-run').map((r) => r.run),
496
+ ...reserved.filter((r) => r.childKind === 'agent-run').map((r) => r.run),
497
+ ]);
138
498
  // Resolve which instances to poll.
139
499
  let instances;
140
500
  if (opts.workflow !== undefined) {
@@ -146,20 +506,59 @@ export function createShiftLoop(opts) {
146
506
  instances = (inbox.instances ?? []).map((i) => i.workflow);
147
507
  }
148
508
  catch (e) {
509
+ noteServerBackoff(e);
149
510
  opts.err(`inbox whats_next failed: ${errMsg(e)}`);
150
- return { dispatched, polled, openRuns };
511
+ // THE THIRD HUB-CALL FAILURE PATH, and the most consequential: this one
512
+ // aborts the WHOLE sweep, because the shift never learned which
513
+ // workflows to poll. The targeted `whats_next` below loses one workflow
514
+ // and the `wake` above loses one tick; this loses everything. It is
515
+ // recorded on the same terms as the other two, with `workflow` absent —
516
+ // that omission is what distinguishes the untargeted inbox call from a
517
+ // per-workflow one, since `HubErrorEvent.workflow` is optional.
518
+ emit({ type: 'hub-error', op: 'whats_next', message: errMsg(e) });
519
+ return { dispatched, polled, openRuns, complete: false };
151
520
  }
152
521
  }
153
522
  // Collect candidates, metering NEW dispatches and deduping in-flight.
154
523
  const claimed = new Set();
155
524
  const candidates = [];
525
+ /**
526
+ * Refuse one order and leave it for the hub pickup window.
527
+ *
528
+ * Every refusal below is a DROPPED UNIT OF WORK, not a debug aside: the
529
+ * shift declines an order the hub already handed it, and until now the only
530
+ * trace was a stderr line a dispatched shift discards. One helper, so the
531
+ * console text and the record cannot drift apart, and so a future refusal
532
+ * cannot be added as text-only by accident. `reason` is the stable machine
533
+ * discriminator; `message` is the same human text as before, unchanged.
534
+ */
535
+ const dropOrder = (workflow, order, reason, message) => {
536
+ opts.err(`[${workflow}/${order.run}] ${message}`);
537
+ emit({ type: 'order-dropped', workflow, run: order.run, step: order.step, reason, message });
538
+ claimed.add(order.run);
539
+ };
156
540
  for (const wf of instances) {
157
541
  let res;
542
+ const requestStartedAt = monotonicNow();
158
543
  try {
159
544
  res = await opts.hub.whatsNext({ workflow: wf, serve_crews: serveCrews });
160
545
  }
161
546
  catch (e) {
547
+ if (isNonServableRace(e)) {
548
+ // Treat the targeted call as a successful empty observation for the
549
+ // work-dir reaper and consume this wake. A later hub event may list
550
+ // new work, but this stale terminal row must not force every poll to
551
+ // repeat the same ForbiddenError.
552
+ polled.add(wf);
553
+ opts.out(`[${wf}] skipped stale terminal inbox candidate`);
554
+ continue;
555
+ }
556
+ const rateLimited = noteServerBackoff(e);
557
+ complete = false;
162
558
  opts.err(`whats_next for ${wf} failed: ${errMsg(e)}`);
559
+ emit({ type: 'hub-error', op: 'whats_next', workflow: wf, message: errMsg(e) });
560
+ if (rateLimited)
561
+ break;
163
562
  continue;
164
563
  }
165
564
  const orders = res.orders ?? [];
@@ -167,153 +566,171 @@ export function createShiftLoop(opts) {
167
566
  for (const o of orders)
168
567
  openRuns.add(o.run);
169
568
  const defName = res.def;
170
- // Dispatch selection (DD-4): honor a UNIQUE pinned hash for this def name;
171
- // 0 pins → latest (unchanged); conflicting pins → refuse (null + warning).
172
- const dispatch = defName !== undefined ? readDispatchBundle(opts.cacheDir, defName) : { bundle: null };
569
+ const legacyOrders = orders.filter((order) => !Object.prototype.hasOwnProperty.call(order, 'worker') &&
570
+ !Object.prototype.hasOwnProperty.call(order, 'defDigest'));
571
+ // The definition-name cache exists only for the explicitly identified old
572
+ // wire shape. Modern orders route from `worker` and exact `defDigest`.
573
+ const dispatch = legacyOrders.length > 0 && defName !== undefined
574
+ ? readDispatchBundle(opts.cacheDir, defName)
575
+ : { bundle: null };
173
576
  const bundle = dispatch.bundle;
174
- if (orders.length > 0 && dispatch.warning !== undefined) {
577
+ if (legacyOrders.length > 0 && dispatch.warning !== undefined) {
175
578
  opts.err(`[${wf}] ${dispatch.warning}`);
176
579
  }
177
- else if (orders.length > 0 && bundle === null && defName !== undefined) {
178
- opts.err(`no cached bundle for def '${defName}' — run \`owenloop work prepare ${defName}\` (agent orders left for pickup)`);
580
+ else if (legacyOrders.length > 0 && bundle === null && defName !== undefined) {
581
+ opts.err(`no cached bundle for def '${defName}' — run \`owenloop work prepare ${defName}\` (legacy orders left for pickup)`);
582
+ emit({ type: 'bundle-miss', workflow: wf, def: defName });
179
583
  }
180
584
  for (const order of orders) {
181
585
  if (claimed.has(order.run))
182
586
  continue; // seen this sweep
183
- const step = bundle?.def.steps.find((s) => s.name === order.step);
184
- const isCmd = step !== undefined && isCommandStep(step);
587
+ if (pendingCandidates.has(order.run))
588
+ continue; // already claimed and queued locally
185
589
  const reoffer = liveRuns.has(order.run);
186
- if (isCmd) {
187
- // A command already in flight (its exec child lives) is never respawned.
590
+ const workerPresent = Object.prototype.hasOwnProperty.call(order, 'worker');
591
+ const digestPresent = Object.prototype.hasOwnProperty.call(order, 'defDigest');
592
+ const legacy = !workerPresent && !digestPresent;
593
+ let step;
594
+ let kind;
595
+ let candidateDefName;
596
+ let candidateDefHash;
597
+ if (legacy) {
598
+ step = bundle?.def.steps.find((candidate) => candidate.name === order.step);
599
+ if (step === undefined || bundle === null)
600
+ continue;
601
+ kind = isCommandStep(step) ? 'command' : 'agent';
602
+ candidateDefName = defName;
603
+ candidateDefHash = bundle.def.hash;
604
+ }
605
+ else {
606
+ if (typeof order.defDigest !== 'string' || order.defDigest.trim() === '') {
607
+ dropOrder(wf, order, 'malformed-digest', 'malformed modern work order: non-empty defDigest is required whenever modern routing metadata is present — leaving for manual pickup');
608
+ continue;
609
+ }
610
+ // The deployed projection omits `worker` for the default executor. That
611
+ // shape is a modern AGENT order because `defDigest` pins the exact step;
612
+ // only an explicitly authored executor produces `worker` on the wire.
613
+ if (!workerPresent || order.worker === 'agent') {
614
+ kind = 'agent';
615
+ }
616
+ else if (order.worker === 'command') {
617
+ kind = 'command';
618
+ try {
619
+ step = await opts.resolveOrderStep?.(order);
620
+ }
621
+ catch (error) {
622
+ dropOrder(wf, order, 'verification-failed', `exact command instructions for digest '${order.defDigest}' failed verification: ${errMsg(error)} — leaving for manual pickup`);
623
+ continue;
624
+ }
625
+ if (step === undefined) {
626
+ dropOrder(wf, order, 'metadata-unavailable', `exact command routing metadata for digest '${order.defDigest}' is unavailable — leaving for manual pickup`);
627
+ continue;
628
+ }
629
+ }
630
+ else if (typeof order.worker !== 'string' || order.worker.trim() === '') {
631
+ dropOrder(wf, order, 'malformed-worker', "malformed modern work order: worker must be absent, 'agent', or 'command' — leaving for manual pickup");
632
+ continue;
633
+ }
634
+ else {
635
+ dropOrder(wf, order, 'unsupported-worker', `unsupported worker '${order.worker}' — leaving for manual pickup`);
636
+ continue;
637
+ }
638
+ }
639
+ if (kind === 'command') {
640
+ // A command already in flight (or durably reserved) is never respawned.
188
641
  if (reoffer)
189
642
  continue;
190
- const r = resolveCommandRouting(opts.commandRouting, step);
191
- for (const w of r.warnings)
192
- opts.err(`[${wf}/${order.run}] ${w}`);
193
- if (!r.autoDispatch) {
194
- opts.out(`[${wf}/${order.run}] command step '${order.step}' routed to Shift — leaving for pickup window`);
643
+ if (step === undefined)
644
+ continue;
645
+ const routing = resolveCommandRouting(opts.commandRouting, step);
646
+ for (const warning of routing.warnings)
647
+ opts.err(`[${wf}/${order.run}] ${warning}`);
648
+ if (!routing.autoDispatch) {
649
+ opts.out(`[${wf}/${order.run}] command step '${order.step}' routed to manual — leaving for pickup window`);
650
+ continue;
651
+ }
652
+ const candidate = {
653
+ order,
654
+ workflow: wf,
655
+ step,
656
+ defName: candidateDefName,
657
+ defHash: candidateDefHash,
658
+ kind,
659
+ requestStartedAt,
660
+ reoffer,
661
+ };
662
+ if (discardExpiredCandidate(candidate, false)) {
663
+ claimed.add(order.run);
195
664
  continue;
196
665
  }
197
- if (remaining <= 0)
198
- continue; // out of free capacity for new work
199
- remaining--;
200
- candidates.push({ order, workflow: wf, step, defName, defHash: bundle?.def.hash, kind: 'command', reoffer });
666
+ if (remaining <= 0) {
667
+ pendingCandidates.set(order.run, candidate);
668
+ opts.out(`[${wf}/${order.run}] at the dispatch cap (${cap}) — queued for local dispatch`);
669
+ }
670
+ else {
671
+ remaining--;
672
+ candidates.push(candidate);
673
+ }
201
674
  claimed.add(order.run);
202
675
  continue;
203
676
  }
204
- // Agent order. An order a live worker already holds is never re-dispatched
205
- // (see `workerRuns`) — that would double-brief one claim.
677
+ // Agent wire routing is authoritative. The detached agent-run resolves its
678
+ // own exact step and harness from the order digest, never from this cache.
206
679
  if (workerRuns.has(order.run))
207
680
  continue;
208
- // No resolved bundle ⇒ no dispatch. Two distinct causes, one outcome:
209
- // - nothing cached for this def name (the operator never ran
210
- // `owenloop work prepare`), so there is no def+hash for the `agent-run`
211
- // child to read `steps/<step>.json` from; and
212
- // - conflicting pins across cached parents (DD-4), where refusing is
213
- // the point — spawning a child that picks its own bundle would make
214
- // the very version guess DD-4 forbids.
215
- // Either way the order is LEFT for the hub's pickup window. The one
216
- // warning per sweep was already written above.
217
- if (bundle === null)
681
+ const candidate = {
682
+ order,
683
+ workflow: wf,
684
+ step: legacy ? step : undefined,
685
+ defName: candidateDefName,
686
+ defHash: candidateDefHash,
687
+ kind,
688
+ requestStartedAt,
689
+ reoffer,
690
+ };
691
+ if (discardExpiredCandidate(candidate, false)) {
692
+ claimed.add(order.run);
218
693
  continue;
219
- // A re-offer replaces its (stale) record without consuming a new slot;
220
- // a genuinely new agent order needs free capacity.
221
- if (!reoffer) {
222
- if (remaining <= 0)
223
- continue;
224
- // The agent budget applies IN ADDITION to the global one. Skipped
225
- // candidates are simply left alone — the hub re-offers them on a later
226
- // sweep once a worker child exits.
227
- if (agentRoom <= 0) {
228
- opts.out(`[${wf}/${order.run}] at the agent-run cap (${maxAgents}) — leaving for a later sweep`);
229
- continue;
694
+ }
695
+ if (!reoffer && remaining <= 0) {
696
+ pendingCandidates.set(order.run, candidate);
697
+ opts.out(`[${wf}/${order.run}] at the dispatch cap (${cap}) — queued for local dispatch`);
698
+ }
699
+ else if (!reoffer && agentRoom <= 0) {
700
+ pendingCandidates.set(order.run, candidate);
701
+ opts.out(`[${wf}/${order.run}] at the agent-run cap (${maxAgents}) — queued for local dispatch`);
702
+ }
703
+ else {
704
+ if (!reoffer) {
705
+ remaining--;
706
+ agentRoom--;
230
707
  }
231
- remaining--;
232
- agentRoom--;
708
+ candidates.push(candidate);
233
709
  }
234
- candidates.push({ order, workflow: wf, step, defName, defHash: bundle?.def.hash, kind: 'agent', reoffer });
235
710
  claimed.add(order.run);
236
711
  }
237
712
  }
238
- if (candidates.length === 0)
239
- return { dispatched, polled, openRuns };
240
- // Dispatch by kind.
241
- for (const c of candidates) {
242
- if (c.kind === 'command') {
243
- try {
244
- const { pid } = opts.spawner({ workflow: c.workflow, run: c.order.run });
245
- const rec = { workflow: c.workflow, run: c.order.run, pid, spawnedAt: opts.now(), kind: 'exec' };
246
- writeChildRecord(opts.stateDir, rec);
247
- dispatched++;
248
- emit({
249
- type: 'dispatched',
250
- workflow: c.workflow,
251
- run: c.order.run,
252
- step: c.order.step,
253
- kind: 'exec',
254
- pid,
255
- });
256
- opts.out(`dispatched command ${c.workflow}/${c.order.run} (step '${c.order.step}', pid ${pid})`);
257
- }
258
- catch (e) {
259
- const message = errMsg(e);
260
- emit({
261
- type: 'failed',
262
- workflow: c.workflow,
263
- run: c.order.run,
264
- step: c.order.step,
265
- kind: 'exec',
266
- message,
267
- });
268
- opts.err(`spawn for ${c.workflow}/${c.order.run} failed: ${message}`);
269
- }
713
+ // 'braked' and 'failed' are deliberately NOT queued into pendingCandidates:
714
+ // both mean this shift is not going to run that order now, and re-queuing a
715
+ // braked candidate would just re-dispatch it on the next drain, defeating
716
+ // the brake. The claim lapses through the hub's pickup window instead.
717
+ for (const candidate of candidates) {
718
+ if (discardExpiredCandidate(candidate, false))
270
719
  continue;
271
- }
272
- // Agent: spawn a detached `agent-run` child that hosts the step agent.
273
- // This is the ONLY agent path — no flag, no alternative.
274
- try {
275
- const { pid } = opts.spawner({
276
- workflow: c.workflow,
277
- run: c.order.run,
278
- kind: 'agent-run',
279
- ...(c.step?.harness !== undefined && c.step.harness !== '' ? { harness: c.step.harness } : {}),
280
- });
281
- const rec = {
282
- workflow: c.workflow,
283
- run: c.order.run,
284
- pid,
285
- spawnedAt: opts.now(),
286
- kind: 'agent-run',
287
- ...(c.defName !== undefined ? { def: c.defName } : {}),
288
- ...(c.defHash !== undefined ? { hash: c.defHash } : {}),
289
- step: c.order.step,
290
- };
291
- writeChildRecord(opts.stateDir, rec);
720
+ const result = dispatchCandidate(candidate);
721
+ if (result === 'dispatched') {
292
722
  dispatched++;
293
- emit({
294
- type: 'dispatched',
295
- workflow: c.workflow,
296
- run: c.order.run,
297
- step: c.order.step,
298
- kind: 'agent-run',
299
- pid,
300
- });
301
- opts.out(`dispatched agent-run ${c.workflow}/${c.order.run} (step '${c.order.step}', pid ${pid})`);
723
+ continue;
302
724
  }
303
- catch (e) {
304
- const message = errMsg(e);
305
- emit({
306
- type: 'failed',
307
- workflow: c.workflow,
308
- run: c.order.run,
309
- step: c.order.step,
310
- kind: 'agent-run',
311
- message,
312
- });
313
- opts.err(`agent-run spawn for ${c.workflow}/${c.order.run} failed: ${message}`);
725
+ if (result === 'total-capacity' || result === 'agent-capacity') {
726
+ pendingCandidates.set(candidate.order.run, candidate);
727
+ const limit = result === 'total-capacity'
728
+ ? `dispatch cap (${cap})`
729
+ : `agent-run cap (${opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS})`;
730
+ opts.out(`[${candidate.workflow}/${candidate.order.run}] lost a shared-capacity race at the ${limit} — queued for local dispatch`);
314
731
  }
315
732
  }
316
- return { dispatched, polled, openRuns };
733
+ return { dispatched, polled, openRuns, complete };
317
734
  }
318
735
  /**
319
736
  * PHASE 4 — remove the per-run work directories of runs that are over.
@@ -367,8 +784,20 @@ export function createShiftLoop(opts) {
367
784
  opts.out(`reaped work directory ${dir}`);
368
785
  }
369
786
  async function iteration() {
787
+ const backoffActiveAtStart = monotonicNow() < backoffUntil;
788
+ let rateLimitedThisIteration = false;
370
789
  // Presence when due (starts immediately — this shift exists to conduct).
371
- if (opts.now() - lastPresence >= opts.presenceIntervalMs) {
790
+ // A server backoff suppresses every hub poll, including presence, while local
791
+ // reconciliation and queued dispatch continue below.
792
+ if (!backoffActiveAtStart &&
793
+ monotonicNow() - lastPresence >= opts.presenceIntervalMs) {
794
+ // `setShift` and `noteAttended` force the next ping by setting
795
+ // lastPresence to -Infinity. Either can land WHILE this ping is awaiting
796
+ // the hub, so record the generation first: on completion, only stamp the
797
+ // cadence if no force arrived in the meantime. Stamping unconditionally
798
+ // would swallow that sentinel and defer the newly recorded attendance or
799
+ // identity by a full presenceIntervalMs.
800
+ const generation = ++presenceGeneration;
372
801
  try {
373
802
  await opts.hub.presencePing({
374
803
  name: shiftName,
@@ -377,32 +806,88 @@ export function createShiftLoop(opts) {
377
806
  ...(opts.startedAt !== undefined ? { started_at: opts.startedAt } : {}),
378
807
  ...(attendedAt !== undefined ? { attended_at: attendedAt } : {}),
379
808
  });
380
- lastPresence = opts.now();
809
+ if (generation === presenceGeneration)
810
+ lastPresence = monotonicNow();
381
811
  }
382
812
  catch (e) {
813
+ rateLimitedThisIteration = noteServerBackoff(e);
383
814
  opts.err(`presence ping failed: ${errMsg(e)} (continuing)`);
384
815
  }
385
816
  }
386
- // Reconcile in-flight (startup recovery + per-sweep reap).
387
- const { live } = reconcile();
388
- const k = cap - live.length;
389
- // Cheap wake pre-check. On failure, stay put this tick (don't sweep).
817
+ // Reconcile in-flight first. A child exit is local state, not a hub event,
818
+ // so use the freed slot to dispatch claims retained from an earlier sweep
819
+ // before consulting the wake cursor.
820
+ let inFlight = reconcile();
821
+ let dispatched = drainPending(inFlight.live, inFlight.reserved);
822
+ if (dispatched > 0)
823
+ inFlight = reconcile();
824
+ const live = inFlight.live;
825
+ const reserved = inFlight.reserved;
826
+ const occupied = live.length + reserved.length;
827
+ const k = cap - occupied;
828
+ // Re-arm the at-capacity report the instant a slot frees, independently of
829
+ // whether this tick goes on to wake or sweep. The next stretch at capacity
830
+ // is a NEW episode and gets its own record.
831
+ if (k > 0)
832
+ atCapReported = false;
833
+ // Retry-After suppresses every remaining hub call in the iteration that
834
+ // received it, and every hub call in later iterations before the deadline.
835
+ // Local child reconciliation and queued-claim dispatch above still run.
836
+ if (backoffActiveAtStart ||
837
+ rateLimitedThisIteration ||
838
+ monotonicNow() < backoffUntil) {
839
+ return dispatched;
840
+ }
841
+ // Cheap wake pre-check. A failed call also suppresses a forced retry for this
842
+ // tick: a rate-limited wake should not immediately be followed by whats_next.
390
843
  let changed = false;
844
+ let wakeSucceeded = false;
391
845
  try {
392
846
  const w = await opts.hub.wake(cursor);
393
847
  changed = w.changed;
394
848
  cursor = w.cursor;
849
+ wakeSucceeded = true;
395
850
  }
396
851
  catch (e) {
852
+ noteServerBackoff(e);
397
853
  opts.err(`wake failed: ${errMsg(e)} (retrying next tick)`);
854
+ emit({ type: 'hub-error', op: 'wake', message: errMsg(e) });
398
855
  }
399
- // Sweep only when something changed AND there is free capacity.
856
+ // A changed cursor is not sufficient by itself: if the prior changed tick
857
+ // had no local capacity, or its sweep failed, the adopted cursor will be
858
+ // unchanged next time. `sweepOwed` preserves that unconsumed work signal.
400
859
  let swept;
401
- if (changed && k > 0) {
402
- swept = await sweep(k, live);
860
+ if (wakeSucceeded && (changed || sweepOwed) && k > 0) {
861
+ swept = await sweep(k, live, reserved);
862
+ sweepOwed = !swept.complete;
403
863
  }
404
- else if (changed) {
405
- opts.out(`at capacity (${live.length}/${cap} in flight) — skipping whats_next this tick`);
864
+ else if (changed && k <= 0) {
865
+ sweepOwed = true;
866
+ opts.out(`at capacity (${occupied}/${cap} in flight) — deferring whats_next until capacity is free`);
867
+ // EDGE-TRIGGERED: at most one record per at-capacity EPISODE, not per
868
+ // tick. At MOST, because this branch needs `changed` — an episode during
869
+ // which no wake reports a changed cursor produces no record at all. The
870
+ // absence of one therefore says nothing about occupancy; `dispatched` and
871
+ // `reaped` are the unconditional pair a reader should count.
872
+ //
873
+ // This branch runs on every changed wake for as long as the shift stays
874
+ // full, which on a busy hub is every poll interval. The console line above
875
+ // is a live tail an operator is watching, and repeating it is how it says
876
+ // "still stuck" — so it stays level-triggered and unchanged. The event has
877
+ // one durable consumer that repetition actively harms: `shift.log`, which
878
+ // this change does not rotate. Entering the state is the fact worth
879
+ // recording; staying in it is recoverable from the next record's `ts`.
880
+ //
881
+ // The record's ROUTING is decided elsewhere, and the two are separate
882
+ // concerns. `runtime.ts` withholds `capacity` from the socket entirely
883
+ // (`FILE_ONLY_EVENTS`) because a parked `shift next` must not be woken by
884
+ // news that nothing happened. Edge-triggering keeps the FILE readable;
885
+ // that set is what keeps the SOCKET correct. Neither substitutes for the
886
+ // other — the loop emits, `consumeEvent` decides who hears it.
887
+ if (!atCapReported) {
888
+ atCapReported = true;
889
+ emit({ type: 'capacity', inFlight: occupied, cap });
890
+ }
406
891
  }
407
892
  const liveAfter = reconcile().live;
408
893
  // PHASE 4 — the work-directory reaper.
@@ -415,7 +900,8 @@ export function createShiftLoop(opts) {
415
900
  // orders" and delete live work. Skipping costs nothing: the next sweeping
416
901
  // tick reaps whatever is still reapable.
417
902
  reapWorkDirs(swept, liveAfter);
418
- return swept?.dispatched ?? 0;
903
+ dispatched += swept?.dispatched ?? 0;
904
+ return dispatched;
419
905
  }
420
906
  async function run() {
421
907
  ensureStateDir(opts.stateDir);
@@ -429,7 +915,8 @@ export function createShiftLoop(opts) {
429
915
  await iteration();
430
916
  if (stopped)
431
917
  return 0;
432
- await opts.sleep(opts.pollIntervalMs);
918
+ const delay = Math.max(opts.pollIntervalMs, backoffUntil - monotonicNow());
919
+ await opts.sleep(delay);
433
920
  // Re-check after the park so a stop during sleep makes no further hub call.
434
921
  if (stopped)
435
922
  return 0;
@@ -442,7 +929,10 @@ export function createShiftLoop(opts) {
442
929
  run,
443
930
  stop,
444
931
  iterate: iteration,
445
- freeCapacity: () => cap - reconcile().live.length,
932
+ freeCapacity: () => {
933
+ const current = reconcile();
934
+ return cap - current.live.length - current.reserved.length;
935
+ },
446
936
  getCap: () => cap,
447
937
  setCap: (next) => {
448
938
  cap = next;
@@ -454,15 +944,46 @@ export function createShiftLoop(opts) {
454
944
  if (next.serveCrews !== undefined)
455
945
  serveCrews = [...next.serveCrews];
456
946
  lastPresence = Number.NEGATIVE_INFINITY; // D6: next iterate() pings immediately
947
+ presenceGeneration++; // survive an in-flight ping completing after this
457
948
  return { name: shiftName, serveCrews: [...serveCrews] };
458
949
  },
459
950
  noteAttended: (at) => {
460
951
  attendedAt = at;
461
952
  lastPresence = Number.NEGATIVE_INFINITY;
953
+ presenceGeneration++; // survive an in-flight ping completing after this
462
954
  },
463
955
  getAttendedAt: () => attendedAt,
464
956
  noteRunEnded: (run) => {
957
+ pendingCandidates.delete(run);
958
+ // A run that ENDED is a run that progressed — the shift's MCP `submit`
959
+ // tool calls this only when the hub reports the submit closed the run.
960
+ // That clears the step's failure streak outright, so one green run wipes
961
+ // out an earlier storm's penalty instead of leaving it to decay.
962
+ const brakeKey = runBrakeKey.get(run);
963
+ if (brakeKey !== undefined) {
964
+ runBrakeKey.delete(run);
965
+ stepBrake.delete(brakeKey);
966
+ }
465
967
  removeChildRecord(opts.stateDir, run);
466
968
  },
969
+ noteWorkerFailure: (failure) => {
970
+ const brakeKey = runBrakeKey.get(failure.run);
971
+ if (brakeKey === undefined)
972
+ return; // not ours, or already accounted for
973
+ runBrakeKey.delete(failure.run);
974
+ const at = monotonicNow();
975
+ const prior = stepBrake.get(brakeKey);
976
+ // A failure long after the previous one starts a NEW streak rather than
977
+ // extending a stale one — see `STEP_BRAKE_DECAY_MS`.
978
+ const priorCount = prior !== undefined && at - prior.failedAt < STEP_BRAKE_DECAY_MS
979
+ ? prior.count
980
+ : 0;
981
+ const count = priorCount + 1;
982
+ stepBrake.set(brakeKey, {
983
+ count,
984
+ failedAt: at,
985
+ nextAllowedAt: at + stepBrakeDelayMs(count),
986
+ });
987
+ },
467
988
  };
468
989
  }