owenloop 0.5.1 → 0.5.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/CHANGELOG.md +30 -0
  2. package/README.md +9 -2
  3. package/dist/packages/work/src/agent/brief.d.ts +21 -0
  4. package/dist/packages/work/src/agent/brief.js +42 -1
  5. package/dist/packages/work/src/agent/capability-model.d.ts +123 -0
  6. package/dist/packages/work/src/agent/capability-model.js +148 -0
  7. package/dist/packages/work/src/agent/loop.d.ts +47 -4
  8. package/dist/packages/work/src/agent/loop.js +83 -53
  9. package/dist/packages/work/src/exec/loop.d.ts +17 -0
  10. package/dist/packages/work/src/exec/loop.js +195 -34
  11. package/dist/packages/work/src/exec/runner.js +15 -5
  12. package/dist/packages/work/src/harness/claude.js +69 -9
  13. package/dist/packages/work/src/harness/codex.js +56 -8
  14. package/dist/packages/work/src/harness/contract.d.ts +65 -1
  15. package/dist/packages/work/src/harness/contract.js +56 -0
  16. package/dist/packages/work/src/harness/session-store.d.ts +3 -1
  17. package/dist/packages/work/src/harness/session-store.js +46 -22
  18. package/dist/packages/work/src/hold/mcp.js +9 -2
  19. package/dist/packages/work/src/hub/client.d.ts +7 -1
  20. package/dist/packages/work/src/hub/client.js +1 -0
  21. package/dist/packages/work/src/hub/types.d.ts +71 -3
  22. package/dist/packages/work/src/roles/agent-run.js +8 -9
  23. package/dist/packages/work/src/roles/hold.d.ts +2 -0
  24. package/dist/packages/work/src/roles/hold.js +39 -8
  25. package/dist/packages/work/src/roles/settings.js +18 -6
  26. package/dist/packages/work/src/roles/shift.js +13 -1
  27. package/dist/packages/work/src/settings/settings.d.ts +60 -7
  28. package/dist/packages/work/src/settings/settings.js +69 -22
  29. package/dist/packages/work/src/shift/logretention.d.ts +253 -0
  30. package/dist/packages/work/src/shift/logretention.js +407 -0
  31. package/dist/packages/work/src/shift/logsink.d.ts +14 -0
  32. package/dist/packages/work/src/shift/logsink.js +98 -0
  33. package/dist/packages/work/src/shift/loop.d.ts +18 -1
  34. package/dist/packages/work/src/shift/loop.js +250 -12
  35. package/dist/packages/work/src/shift/protocol.d.ts +124 -1
  36. package/dist/packages/work/src/shift/protocol.js +14 -0
  37. package/dist/packages/work/src/shift/runtime.d.ts +14 -0
  38. package/dist/packages/work/src/shift/runtime.js +229 -6
  39. package/dist/packages/work/src/shift/server.d.ts +13 -0
  40. package/dist/packages/work/src/shift/server.js +61 -36
  41. package/dist/packages/work/src/shift/spawn.d.ts +78 -5
  42. package/dist/packages/work/src/shift/spawn.js +158 -12
  43. package/dist/packages/work/src/shift/state.d.ts +25 -0
  44. package/dist/packages/work/src/shift/state.js +29 -5
  45. package/dist/packages/work/src/shift/truncate.d.ts +35 -0
  46. package/dist/packages/work/src/shift/truncate.js +73 -0
  47. package/dist/packages/work/src/submit-proof.d.ts +31 -6
  48. package/dist/packages/work/src/submit-proof.js +33 -7
  49. package/dist/packages/work/src/submit-value.d.ts +48 -0
  50. package/dist/packages/work/src/submit-value.js +118 -0
  51. package/dist/packages/work/src/usage.d.ts +1 -1
  52. package/dist/packages/work/src/usage.js +11 -0
  53. package/dist/src/bundle/runtime.d.ts +1 -1
  54. package/dist/src/bundle/runtime.js +9 -0
  55. package/dist/src/capabilities.d.ts +93 -0
  56. package/dist/src/capabilities.js +101 -0
  57. package/dist/src/cli.js +37 -11
  58. package/dist/src/crypto/records.d.ts +3 -0
  59. package/dist/src/crypto/records.js +11 -0
  60. package/dist/src/defs.js +192 -4
  61. package/dist/src/engine.d.ts +112 -3
  62. package/dist/src/engine.js +224 -28
  63. package/dist/src/hub.d.ts +27 -11
  64. package/dist/src/hub.js +24 -18
  65. package/dist/src/index.d.ts +3 -1
  66. package/dist/src/index.js +4 -1
  67. package/dist/src/mcp/serve.js +11 -4
  68. package/dist/src/model.d.ts +36 -0
  69. package/dist/src/model.js +46 -0
  70. package/dist/src/schemas/index.d.ts +17 -0
  71. package/dist/src/schemas/order.v1.schema.json +17 -3
  72. package/dist/src/store.d.ts +33 -1
  73. package/dist/src/store.js +57 -4
  74. package/dist/src/types.d.ts +102 -2
  75. package/docs/agent-runner.md +26 -1
  76. package/docs/authoring.md +7 -0
  77. package/docs/bundles.md +103 -3
  78. package/docs/cli.md +127 -9
  79. package/docs/crypto.md +23 -7
  80. package/docs/shift-logs.md +642 -0
  81. package/docs/wire-contracts.md +34 -19
  82. package/package.json +1 -1
  83. package/plugins/claude-code/plugin/.claude-plugin/plugin.json +1 -1
  84. package/plugins/claude-code/plugin/.mcp.json +1 -1
  85. package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +1 -1
  86. package/plugins/codex/plugins/owenloop/.mcp.json +1 -1
  87. package/dist/packages/work/src/agent/model-policy.d.ts +0 -49
  88. package/dist/packages/work/src/agent/model-policy.js +0 -138
@@ -0,0 +1,98 @@
1
+ /**
2
+ * The shift's structured on-disk log — `<log-dir>/shift.log`, JSON Lines.
3
+ *
4
+ * This is the SECOND consumer of the `ShiftEvent` contract (the Unix socket is
5
+ * the first), and it is deliberately not a new logging abstraction: it receives
6
+ * the same stamped records `emit()` fans to the socket, serializes each as one
7
+ * line, and appends. A future hub uploader becomes the THIRD consumer of those
8
+ * same records by tailing this file — not by rewriting this code.
9
+ *
10
+ * ── WHY `appendFileSync`, NOT `createWriteStream` ──
11
+ *
12
+ * Shift events are low volume — a handful per dispatch, reap, failure, or gate
13
+ * — so throughput is not the constraint. Crash durability is. A stream's
14
+ * buffered lines are lost when the process dies abruptly, which is exactly the
15
+ * crash the log exists to explain. `appendFileSync` holds NO descriptor between
16
+ * writes: every line is on disk before `write()` returns, and there is nothing
17
+ * to flush or close at shutdown. That is why this module exposes no `close()` —
18
+ * there is no open resource to release, and a no-op `close()` would only invite
19
+ * a caller to believe ordering or durability depended on calling it.
20
+ *
21
+ * ── WHY ONE SYSCALL PER LINE ──
22
+ *
23
+ * Each record is serialized to a complete `…\n` string and written with a
24
+ * SINGLE `appendFileSync` call, and two shifts sharing a log directory both
25
+ * open with `O_APPEND`, so in practice lines stay individually parseable and
26
+ * the `shiftId` on each record says which process wrote it.
27
+ *
28
+ * BE PRECISE ABOUT WHERE THAT GUARANTEE COMES FROM: it is a property of an
29
+ * `O_APPEND` `write()` to a REGULAR FILE ON A LOCAL FILESYSTEM, not of
30
+ * `appendFileSync` itself. `appendFileSync` loops on a short write, and each
31
+ * iteration of that loop is a separate `write()` a second process could append
32
+ * between. Under the sizes involved (one JSON Lines record, bounded at
33
+ * `MAX_RESPONSE_LINE_BYTES`) a local filesystem does not short-write, which is
34
+ * why the property holds — but a reader on NFS, or one tailing a file being
35
+ * written by a process that died mid-line, can still see a partial trailing
36
+ * line. Parse whole lines and treat a trailing fragment as not-yet-written;
37
+ * `docs/shift-logs.md` states this as the uploader contract.
38
+ *
39
+ * ── WHY A WRITE FAILURE IS NOT AN ERROR THE CALLER SEES ──
40
+ *
41
+ * `write()` never throws. A shift that cannot log must still dispatch work: a
42
+ * full disk is not a reason to stop delivering. The failure is reported ONCE
43
+ * through `err` — the report is latched, not the writing — so a broken sink
44
+ * cannot spam one line per event, while a transient failure (a disk that frees
45
+ * up, a directory an operator recreates) starts logging again on its own.
46
+ */
47
+ import { appendFileSync } from 'node:fs';
48
+ import { MAX_RESPONSE_LINE_BYTES } from "./protocol.js";
49
+ import { truncateEventToBytes } from "./truncate.js";
50
+ /** Serialized bytes of one JSON Lines record, newline included. */
51
+ function lineBytes(event) {
52
+ return Buffer.byteLength(`${JSON.stringify(event)}\n`, 'utf8');
53
+ }
54
+ export function createShiftLogSink(opts) {
55
+ const append = opts.append ?? ((path, line) => {
56
+ // MODE 0600, OWNER ONLY, and it takes effect only on the call that CREATES
57
+ // the file — every later append leaves the existing mode alone. `shift.log`
58
+ // carries the same class of data as `<run>.log`: `order-dropped` and
59
+ // `hub-error` records quote hub and workflow messages verbatim, so the file
60
+ // is owner-only for the same reason `exec/loop.ts` writes its artifact JSON
61
+ // 0600. Without the option this would be 0666 & ~umask — 0644 in practice.
62
+ appendFileSync(path, line, { mode: 0o600 });
63
+ });
64
+ let reported = false;
65
+ return {
66
+ write(event) {
67
+ let line;
68
+ try {
69
+ // Bound the line by the SAME ceiling the socket applies, so the file
70
+ // never carries a record the wire protocol would have refused and an
71
+ // uploader can size its reads from one documented number.
72
+ const bounded = truncateEventToBytes(event, MAX_RESPONSE_LINE_BYTES, lineBytes);
73
+ line = `${JSON.stringify(bounded)}\n`;
74
+ }
75
+ catch (e) {
76
+ // Serialization itself failed (a cyclic or unserializable field). There
77
+ // is no line to write, and no partial write happened.
78
+ if (!reported) {
79
+ reported = true;
80
+ opts.err(`shift log sink could not serialize an event: ${errMsg(e)} (logging continues)`);
81
+ }
82
+ return;
83
+ }
84
+ try {
85
+ append(opts.path, line);
86
+ }
87
+ catch (e) {
88
+ if (!reported) {
89
+ reported = true;
90
+ opts.err(`shift log sink write to ${opts.path} failed: ${errMsg(e)} (continuing without it)`);
91
+ }
92
+ }
93
+ },
94
+ };
95
+ }
96
+ function errMsg(err) {
97
+ return err instanceof Error ? err.message : String(err);
98
+ }
@@ -4,7 +4,7 @@ import type { FetchedStep } from '../bundle/types.ts';
4
4
  import { type Liveness } from './state.ts';
5
5
  import { sweepWorkDirs as sweepWorkDirsImpl } from '../agent/workdir.ts';
6
6
  import type { Spawner } from './spawn.ts';
7
- import type { ShiftEvent } from './protocol.ts';
7
+ import { type ShiftEvent } from './protocol.ts';
8
8
  export interface ShiftLoopOptions {
9
9
  hub: HubClient;
10
10
  spawner: Spawner;
@@ -122,6 +122,23 @@ export interface ShiftLoop {
122
122
  * next reconcile's pid probe frees the slot anyway.
123
123
  */
124
124
  noteRunEnded(run: string): void;
125
+ /**
126
+ * A dispatched child exited non-zero. Charges one failure against that run's
127
+ * STEP and arms the brake window — see `STEP_BRAKE_DELAYS_MS`.
128
+ *
129
+ * The shift runtime already builds this exact failure record for its daemon
130
+ * event and its stderr line; this is the third consumer. The loop cannot
131
+ * observe child exits itself (children are detached with `stdio: ignore`),
132
+ * so without this call the loop has no failure signal at all and re-dispatch
133
+ * is bounded by nothing.
134
+ *
135
+ * A failure for a run this loop never dispatched (a foreign or already-reaped
136
+ * run) is ignored rather than guessed at: the fan-out key is not on the
137
+ * failure record, so there is no sound way to pick a brake key for it.
138
+ */
139
+ noteWorkerFailure(failure: {
140
+ run: string;
141
+ }): void;
125
142
  }
126
143
  /**
127
144
  * The hub reaps a never-contacted claim after 120 seconds. Stop local queueing
@@ -60,6 +60,7 @@ import { isCommandStep, resolveCommandRouting } from "./routing.js";
60
60
  import { cancelReservedChild, finalizeChildReservation, reconcileInFlight, reserveChild, startReservedChild, removeChildRecord, ensureStateDir, } from "./state.js";
61
61
  import { DEFAULT_WORK_DIR_TTL_MS, sweepWorkDirs as sweepWorkDirsImpl } from "../agent/workdir.js";
62
62
  import { sessionsPath } from "../harness/session-store.js";
63
+ import { stampShiftEvent, } from "./protocol.js";
63
64
  /** The `maxConcurrentAgents` fallback when the option is absent. */
64
65
  const DEFAULT_MAX_AGENTS = 4;
65
66
  /**
@@ -67,6 +68,69 @@ const DEFAULT_MAX_AGENTS = 4;
67
68
  * after 90 seconds so a detached child retains 30 seconds to make first contact.
68
69
  */
69
70
  export const MAX_PENDING_CANDIDATE_AGE_MS = 90_000;
71
+ /**
72
+ * THE PER-STEP FAILURE BRAKE.
73
+ *
74
+ * Every in-flight guard in this file keys on the RUN id — `liveRuns`,
75
+ * `workerRuns`, `claimed`, `pendingCandidates`. That is correct for the case
76
+ * they were written for: one claim, one child. It is blind to the case where a
77
+ * step fails the same way forever.
78
+ *
79
+ * Observed 2026-08-12: `local-check` of one instance was dispatched five-plus
80
+ * times in seconds under a DIFFERENT run id each time
81
+ * (run_fd94d88a…, run_5ec4d807…, run_d8cdf03f…, run_67c8e2db…, run_072755ba…).
82
+ * The child refused its consumed artifact, exited non-zero WITHOUT submitting,
83
+ * the hub released the claim, and the next poll handed the same step back under
84
+ * a fresh run id. A fresh run id matches none of the run-keyed guards, so
85
+ * nothing bounded the respawns.
86
+ *
87
+ * So the brake is keyed on the identity that actually repeats — the STEP, as
88
+ * `(workflow, step, key)`. `key` is part of the identity because a fanned-out
89
+ * step has one legitimately concurrent order per key, and those must not brake
90
+ * each other.
91
+ *
92
+ * WHAT IS COUNTED, AND WHY IT IS NOT DISPATCHES. The event counted against that
93
+ * key is a WORKER FAILURE — a child that exited non-zero, reported into the
94
+ * loop through `noteWorkerFailure` — never a dispatch. Counting dispatches
95
+ * conflates a storm with legitimate concurrency: the hub can offer several
96
+ * DISTINCT runs of one step in a single sweep, and metering the second through
97
+ * fifth of those would throttle healthy work. A failure is direct evidence that
98
+ * running this step again right now is unlikely to help; a dispatch is evidence
99
+ * of nothing.
100
+ *
101
+ * It is a rate limit, never a ban. A step that has not failed is dispatched
102
+ * with zero delay, however many orders arrive for it at once.
103
+ */
104
+ const STEP_BRAKE_DELAYS_MS = [2_000, 8_000, 30_000, 120_000, 300_000];
105
+ /**
106
+ * Treat a failure this long after the previous one as the start of a NEW
107
+ * streak, resetting the count to zero first. Comfortably longer than the
108
+ * longest delay in the table (300s), so a step failing at maximum backoff never
109
+ * decays out of it, while an isolated transient long after an old streak starts
110
+ * again at the shortest delay instead of inheriting a stale penalty.
111
+ */
112
+ const STEP_BRAKE_DECAY_MS = 600_000;
113
+ /**
114
+ * Forget a step's brake entry once this long has passed since its window
115
+ * closed, so the map cannot grow for the life of the shift. Pure housekeeping —
116
+ * `STEP_BRAKE_DECAY_MS` is what governs behaviour.
117
+ */
118
+ const STEP_BRAKE_FORGET_MS = 1_800_000;
119
+ /**
120
+ * How long to refuse the next dispatch of a step after `count` consecutive
121
+ * worker failures. `count` is at least 1 at every call site (this is reached
122
+ * only from the failure path), so the FIRST failure already costs 2s; past the
123
+ * end of the table the last entry repeats, so a permanently-failing step
124
+ * settles at one dispatch per five minutes instead of one per poll.
125
+ */
126
+ function stepBrakeDelayMs(count) {
127
+ const index = Math.min(count, STEP_BRAKE_DELAYS_MS.length) - 1;
128
+ return STEP_BRAKE_DELAYS_MS[index] ?? 0;
129
+ }
130
+ /** The identity a storm repeats on. `key` is '' for an unfanned step. */
131
+ function stepBrakeKey(workflow, step, key) {
132
+ return `${workflow}\u0000${step}\u0000${key ?? ''}`;
133
+ }
70
134
  function errMsg(e) {
71
135
  return e instanceof Error ? e.message : String(e);
72
136
  }
@@ -104,6 +168,15 @@ export function createShiftLoop(opts) {
104
168
  /** A changed wake whose sweep was skipped or failed must be retried after the
105
169
  * cursor is adopted; otherwise the next unchanged wake hides that work. */
106
170
  let sweepOwed = false;
171
+ /**
172
+ * Has the current at-capacity EPISODE already produced a `capacity` event?
173
+ *
174
+ * Set when the event fires, cleared the moment a slot is free again, so one
175
+ * unbroken stretch at capacity yields exactly one record no matter how many
176
+ * ticks it spans. See the emit site for why the console line next to it is
177
+ * deliberately NOT edge-triggered.
178
+ */
179
+ let atCapReported = false;
107
180
  /** Earliest server-approved time for the next polling iteration. */
108
181
  let backoffUntil = Number.NEGATIVE_INFINITY;
109
182
  /** Bumped whenever a ping starts or is forced due, so a ping that completes
@@ -116,11 +189,25 @@ export function createShiftLoop(opts) {
116
189
  backoffUntil = Math.max(backoffUntil, monotonicNow() + delay);
117
190
  return true;
118
191
  }
119
- function emit(event) {
192
+ /**
193
+ * Fan one observation to every consumer, stamped.
194
+ *
195
+ * THE SINGLE ENVELOPE-STAMPING POINT for everything the loop observes. Call
196
+ * sites build a `ShiftEventBody` — payload only — and this function adds the
197
+ * timestamp and the shift identity, so no construction site can forget them
198
+ * and no consumer can receive a half-stamped record. `shiftName` is read live
199
+ * rather than captured, because `clock_in` can rename a shift mid-run and the
200
+ * record should say what the shift was called when it was written.
201
+ *
202
+ * A THROWING CONSUMER MUST NOT KILL THE SHIFT. The report goes to `opts.err`
203
+ * as free text and stays free text: it is the sink's own failure report, and
204
+ * turning it into an event would write it to the sink that just failed.
205
+ */
206
+ function emit(body) {
120
207
  if (opts.onEvent === undefined)
121
208
  return;
122
209
  try {
123
- opts.onEvent(event);
210
+ opts.onEvent(stampShiftEvent(body, { name: shiftName, id: opts.shiftId ?? '' }, opts.now()));
124
211
  }
125
212
  catch (e) {
126
213
  opts.err(`shift event sink failed: ${errMsg(e)} (continuing)`);
@@ -145,6 +232,30 @@ export function createShiftLoop(opts) {
145
232
  }
146
233
  return result;
147
234
  }
235
+ /**
236
+ * Per-step failure history for the brake. Keyed by `(workflow, step, key)` —
237
+ * see `stepBrakeKey`. `count` is the current consecutive-failure streak for
238
+ * that step, `failedAt` the monotonic instant of the most recent failure (the
239
+ * input to the `STEP_BRAKE_DECAY_MS` streak reset), and `nextAllowedAt` the
240
+ * monotonic instant the next spawn of that step becomes permitted.
241
+ */
242
+ const stepBrake = new Map();
243
+ /**
244
+ * Brake key of the step each dispatched run belongs to.
245
+ *
246
+ * A `WorkerFailure` carries the run id and the step name but NOT the fan-out
247
+ * key, and the brake identity includes the key — so the mapping is recorded
248
+ * here at dispatch, when the whole order is in hand, rather than
249
+ * reconstructed from the failure. Written synchronously in
250
+ * `dispatchCandidate` immediately after the spawn returns, which always
251
+ * precedes the child's `exit` event (that fires on a later tick), so a
252
+ * failure can never arrive before its own entry exists.
253
+ *
254
+ * Entries are removed when the run's failure is noted, when the run ends
255
+ * (`noteRunEnded`), and for any run no longer live at sweep time — so this
256
+ * map tracks live dispatches only and cannot grow without bound.
257
+ */
258
+ const runBrakeKey = new Map();
148
259
  /**
149
260
  * Recheck shared capacity and create the per-run reservation while holding one
150
261
  * state-directory-wide lock. The lock ends before spawn: the durable
@@ -186,6 +297,23 @@ export function createShiftLoop(opts) {
186
297
  }
187
298
  function dispatchCandidate(c) {
188
299
  const childKind = c.kind === 'command' ? 'exec' : 'agent-run';
300
+ // THE BRAKE. This function is the single choke point every spawn passes
301
+ // through — both the sweep's dispatch loop and `drainPending` reach a child
302
+ // only through here — so gating it here cannot be bypassed by either path.
303
+ // Braking BEFORE `reserveCandidate` means no reservation, no state-dir
304
+ // write and no lock contention for an order this shift will not run. The
305
+ // claim already taken by `whats_next` is deliberately left to lapse: the
306
+ // hub reaps a never-contacted claim after 120s, which throttles the
307
+ // re-offer rate on the hub side too.
308
+ const brakeKey = stepBrakeKey(c.workflow, c.order.step, c.order.key);
309
+ const brake = stepBrake.get(brakeKey);
310
+ if (brake !== undefined && monotonicNow() < brake.nextAllowedAt) {
311
+ const waitMs = Math.round(brake.nextAllowedAt - monotonicNow());
312
+ opts.err(`[${c.workflow}/${c.order.run}] step '${c.order.step}' has failed ` +
313
+ `${brake.count} time(s) in a row — braking for ${waitMs}ms and ` +
314
+ `leaving this claim to lapse`);
315
+ return 'braked';
316
+ }
189
317
  let reservation;
190
318
  let cancel;
191
319
  try {
@@ -210,6 +338,12 @@ export function createShiftLoop(opts) {
210
338
  ...(childKind === 'agent-run' ? { step: c.order.step } : {}),
211
339
  });
212
340
  startReservedChild(opts.stateDir, rec);
341
+ // Remember which step this run belongs to, so a later worker failure —
342
+ // which names the run but not the fan-out key — can be charged to the
343
+ // right brake key. NOTHING is counted here: a dispatch is not evidence of
344
+ // trouble, and counting it would meter the legitimately concurrent runs
345
+ // of one step that a single sweep can offer.
346
+ runBrakeKey.set(c.order.run, brakeKey);
213
347
  emit({
214
348
  type: 'dispatched',
215
349
  workflow: c.workflow,
@@ -306,6 +440,28 @@ export function createShiftLoop(opts) {
306
440
  */
307
441
  async function sweep(budget, live, reserved) {
308
442
  let dispatched = 0;
443
+ // Housekeeping for both brake maps, once per sweep rather than on write, so
444
+ // neither can grow for the life of a long-lived shift.
445
+ //
446
+ // `stepBrake`: forget an entry whose window closed STEP_BRAKE_FORGET_MS
447
+ // ago. Behaviour is already governed by STEP_BRAKE_DECAY_MS, which resets a
448
+ // stale streak; this only reclaims the memory.
449
+ //
450
+ // `runBrakeKey`: drop every run that is no longer live. A run whose child
451
+ // exited CLEANLY reports no failure, so nothing else would ever remove its
452
+ // entry. A non-zero exit is charged by `noteWorkerFailure`, which the
453
+ // child's own `exit` event drives on the next event-loop tick — always well
454
+ // before the next sweep, which waits a whole poll interval. If that order
455
+ // ever did invert, the cost is one uncharged failure, never a wrong charge.
456
+ for (const [key, entry] of stepBrake) {
457
+ if (monotonicNow() - entry.nextAllowedAt > STEP_BRAKE_FORGET_MS)
458
+ stepBrake.delete(key);
459
+ }
460
+ const liveOrReserved = new Set([...live.map((r) => r.run), ...reserved.map((r) => r.run)]);
461
+ for (const run of runBrakeKey.keys()) {
462
+ if (!liveOrReserved.has(run))
463
+ runBrakeKey.delete(run);
464
+ }
309
465
  /**
310
466
  * PHASE 4, for the work-dir reaper only. `polled` is the set of workflows
311
467
  * whose `whats_next` SUCCEEDED this sweep — a workflow whose call threw is
@@ -352,12 +508,35 @@ export function createShiftLoop(opts) {
352
508
  catch (e) {
353
509
  noteServerBackoff(e);
354
510
  opts.err(`inbox whats_next failed: ${errMsg(e)}`);
511
+ // THE THIRD HUB-CALL FAILURE PATH, and the most consequential: this one
512
+ // aborts the WHOLE sweep, because the shift never learned which
513
+ // workflows to poll. The targeted `whats_next` below loses one workflow
514
+ // and the `wake` above loses one tick; this loses everything. It is
515
+ // recorded on the same terms as the other two, with `workflow` absent —
516
+ // that omission is what distinguishes the untargeted inbox call from a
517
+ // per-workflow one, since `HubErrorEvent.workflow` is optional.
518
+ emit({ type: 'hub-error', op: 'whats_next', message: errMsg(e) });
355
519
  return { dispatched, polled, openRuns, complete: false };
356
520
  }
357
521
  }
358
522
  // Collect candidates, metering NEW dispatches and deduping in-flight.
359
523
  const claimed = new Set();
360
524
  const candidates = [];
525
+ /**
526
+ * Refuse one order and leave it for the hub pickup window.
527
+ *
528
+ * Every refusal below is a DROPPED UNIT OF WORK, not a debug aside: the
529
+ * shift declines an order the hub already handed it, and until now the only
530
+ * trace was a stderr line a dispatched shift discards. One helper, so the
531
+ * console text and the record cannot drift apart, and so a future refusal
532
+ * cannot be added as text-only by accident. `reason` is the stable machine
533
+ * discriminator; `message` is the same human text as before, unchanged.
534
+ */
535
+ const dropOrder = (workflow, order, reason, message) => {
536
+ opts.err(`[${workflow}/${order.run}] ${message}`);
537
+ emit({ type: 'order-dropped', workflow, run: order.run, step: order.step, reason, message });
538
+ claimed.add(order.run);
539
+ };
361
540
  for (const wf of instances) {
362
541
  let res;
363
542
  const requestStartedAt = monotonicNow();
@@ -377,6 +556,7 @@ export function createShiftLoop(opts) {
377
556
  const rateLimited = noteServerBackoff(e);
378
557
  complete = false;
379
558
  opts.err(`whats_next for ${wf} failed: ${errMsg(e)}`);
559
+ emit({ type: 'hub-error', op: 'whats_next', workflow: wf, message: errMsg(e) });
380
560
  if (rateLimited)
381
561
  break;
382
562
  continue;
@@ -399,6 +579,7 @@ export function createShiftLoop(opts) {
399
579
  }
400
580
  else if (legacyOrders.length > 0 && bundle === null && defName !== undefined) {
401
581
  opts.err(`no cached bundle for def '${defName}' — run \`owenloop work prepare ${defName}\` (legacy orders left for pickup)`);
582
+ emit({ type: 'bundle-miss', workflow: wf, def: defName });
402
583
  }
403
584
  for (const order of orders) {
404
585
  if (claimed.has(order.run))
@@ -423,8 +604,7 @@ export function createShiftLoop(opts) {
423
604
  }
424
605
  else {
425
606
  if (typeof order.defDigest !== 'string' || order.defDigest.trim() === '') {
426
- opts.err(`[${wf}/${order.run}] malformed modern work order: non-empty defDigest is required whenever modern routing metadata is present — leaving for manual pickup`);
427
- claimed.add(order.run);
607
+ dropOrder(wf, order, 'malformed-digest', 'malformed modern work order: non-empty defDigest is required whenever modern routing metadata is present — leaving for manual pickup');
428
608
  continue;
429
609
  }
430
610
  // The deployed projection omits `worker` for the default executor. That
@@ -439,24 +619,20 @@ export function createShiftLoop(opts) {
439
619
  step = await opts.resolveOrderStep?.(order);
440
620
  }
441
621
  catch (error) {
442
- opts.err(`[${wf}/${order.run}] exact command instructions for digest '${order.defDigest}' failed verification: ${errMsg(error)} — leaving for manual pickup`);
443
- claimed.add(order.run);
622
+ dropOrder(wf, order, 'verification-failed', `exact command instructions for digest '${order.defDigest}' failed verification: ${errMsg(error)} — leaving for manual pickup`);
444
623
  continue;
445
624
  }
446
625
  if (step === undefined) {
447
- opts.err(`[${wf}/${order.run}] exact command routing metadata for digest '${order.defDigest}' is unavailable — leaving for manual pickup`);
448
- claimed.add(order.run);
626
+ dropOrder(wf, order, 'metadata-unavailable', `exact command routing metadata for digest '${order.defDigest}' is unavailable — leaving for manual pickup`);
449
627
  continue;
450
628
  }
451
629
  }
452
630
  else if (typeof order.worker !== 'string' || order.worker.trim() === '') {
453
- opts.err(`[${wf}/${order.run}] malformed modern work order: worker must be absent, 'agent', or 'command' — leaving for manual pickup`);
454
- claimed.add(order.run);
631
+ dropOrder(wf, order, 'malformed-worker', "malformed modern work order: worker must be absent, 'agent', or 'command' — leaving for manual pickup");
455
632
  continue;
456
633
  }
457
634
  else {
458
- opts.err(`[${wf}/${order.run}] unsupported worker '${order.worker}' — leaving for manual pickup`);
459
- claimed.add(order.run);
635
+ dropOrder(wf, order, 'unsupported-worker', `unsupported worker '${order.worker}' — leaving for manual pickup`);
460
636
  continue;
461
637
  }
462
638
  }
@@ -534,6 +710,10 @@ export function createShiftLoop(opts) {
534
710
  claimed.add(order.run);
535
711
  }
536
712
  }
713
+ // 'braked' and 'failed' are deliberately NOT queued into pendingCandidates:
714
+ // both mean this shift is not going to run that order now, and re-queuing a
715
+ // braked candidate would just re-dispatch it on the next drain, defeating
716
+ // the brake. The claim lapses through the hub's pickup window instead.
537
717
  for (const candidate of candidates) {
538
718
  if (discardExpiredCandidate(candidate, false))
539
719
  continue;
@@ -645,6 +825,11 @@ export function createShiftLoop(opts) {
645
825
  const reserved = inFlight.reserved;
646
826
  const occupied = live.length + reserved.length;
647
827
  const k = cap - occupied;
828
+ // Re-arm the at-capacity report the instant a slot frees, independently of
829
+ // whether this tick goes on to wake or sweep. The next stretch at capacity
830
+ // is a NEW episode and gets its own record.
831
+ if (k > 0)
832
+ atCapReported = false;
648
833
  // Retry-After suppresses every remaining hub call in the iteration that
649
834
  // received it, and every hub call in later iterations before the deadline.
650
835
  // Local child reconciliation and queued-claim dispatch above still run.
@@ -666,6 +851,7 @@ export function createShiftLoop(opts) {
666
851
  catch (e) {
667
852
  noteServerBackoff(e);
668
853
  opts.err(`wake failed: ${errMsg(e)} (retrying next tick)`);
854
+ emit({ type: 'hub-error', op: 'wake', message: errMsg(e) });
669
855
  }
670
856
  // A changed cursor is not sufficient by itself: if the prior changed tick
671
857
  // had no local capacity, or its sweep failed, the adopted cursor will be
@@ -678,6 +864,30 @@ export function createShiftLoop(opts) {
678
864
  else if (changed && k <= 0) {
679
865
  sweepOwed = true;
680
866
  opts.out(`at capacity (${occupied}/${cap} in flight) — deferring whats_next until capacity is free`);
867
+ // EDGE-TRIGGERED: at most one record per at-capacity EPISODE, not per
868
+ // tick. At MOST, because this branch needs `changed` — an episode during
869
+ // which no wake reports a changed cursor produces no record at all. The
870
+ // absence of one therefore says nothing about occupancy; `dispatched` and
871
+ // `reaped` are the unconditional pair a reader should count.
872
+ //
873
+ // This branch runs on every changed wake for as long as the shift stays
874
+ // full, which on a busy hub is every poll interval. The console line above
875
+ // is a live tail an operator is watching, and repeating it is how it says
876
+ // "still stuck" — so it stays level-triggered and unchanged. The event has
877
+ // one durable consumer that repetition actively harms: `shift.log`, which
878
+ // this change does not rotate. Entering the state is the fact worth
879
+ // recording; staying in it is recoverable from the next record's `ts`.
880
+ //
881
+ // The record's ROUTING is decided elsewhere, and the two are separate
882
+ // concerns. `runtime.ts` withholds `capacity` from the socket entirely
883
+ // (`FILE_ONLY_EVENTS`) because a parked `shift next` must not be woken by
884
+ // news that nothing happened. Edge-triggering keeps the FILE readable;
885
+ // that set is what keeps the SOCKET correct. Neither substitutes for the
886
+ // other — the loop emits, `consumeEvent` decides who hears it.
887
+ if (!atCapReported) {
888
+ atCapReported = true;
889
+ emit({ type: 'capacity', inFlight: occupied, cap });
890
+ }
681
891
  }
682
892
  const liveAfter = reconcile().live;
683
893
  // PHASE 4 — the work-directory reaper.
@@ -745,7 +955,35 @@ export function createShiftLoop(opts) {
745
955
  getAttendedAt: () => attendedAt,
746
956
  noteRunEnded: (run) => {
747
957
  pendingCandidates.delete(run);
958
+ // A run that ENDED is a run that progressed — the shift's MCP `submit`
959
+ // tool calls this only when the hub reports the submit closed the run.
960
+ // That clears the step's failure streak outright, so one green run wipes
961
+ // out an earlier storm's penalty instead of leaving it to decay.
962
+ const brakeKey = runBrakeKey.get(run);
963
+ if (brakeKey !== undefined) {
964
+ runBrakeKey.delete(run);
965
+ stepBrake.delete(brakeKey);
966
+ }
748
967
  removeChildRecord(opts.stateDir, run);
749
968
  },
969
+ noteWorkerFailure: (failure) => {
970
+ const brakeKey = runBrakeKey.get(failure.run);
971
+ if (brakeKey === undefined)
972
+ return; // not ours, or already accounted for
973
+ runBrakeKey.delete(failure.run);
974
+ const at = monotonicNow();
975
+ const prior = stepBrake.get(brakeKey);
976
+ // A failure long after the previous one starts a NEW streak rather than
977
+ // extending a stale one — see `STEP_BRAKE_DECAY_MS`.
978
+ const priorCount = prior !== undefined && at - prior.failedAt < STEP_BRAKE_DECAY_MS
979
+ ? prior.count
980
+ : 0;
981
+ const count = priorCount + 1;
982
+ stepBrake.set(brakeKey, {
983
+ count,
984
+ failedAt: at,
985
+ nextAllowedAt: at + stepBrakeDelayMs(count),
986
+ });
987
+ },
750
988
  };
751
989
  }
@@ -69,7 +69,130 @@ export interface GateEvent {
69
69
  export interface EndedEvent {
70
70
  type: 'ended';
71
71
  }
72
- export type ShiftEvent = DispatchedEvent | ReapedEvent | FailedEvent | GateEvent | EndedEvent;
72
+ /**
73
+ * The shift announced its identity and started serving. The FIRST record a
74
+ * shift writes, so a reader can place every later record on the same machine
75
+ * without external context. `shift` and `shiftId` are not repeated here: they
76
+ * are on the envelope of this and every other record.
77
+ *
78
+ * FILE-ONLY. Written to `shift.log` and never delivered over the socket. An
79
+ * idle `owenloop shift next` must BLOCK until there is work to report, and a
80
+ * startup record sitting in the FIFO would satisfy the first `next` after every
81
+ * start. A socket client needs none of this — it knows which shift it reached
82
+ * because it connected to that shift's socket, and `op: 'status'` answers name,
83
+ * crews, and cap on demand.
84
+ */
85
+ export interface ParkedEvent {
86
+ type: 'parked';
87
+ origin: string;
88
+ cap: number;
89
+ serveCrews: string[];
90
+ hostname: string;
91
+ cwd: string;
92
+ }
93
+ /**
94
+ * No local capacity, so the `whats_next` SWEEP was deferred. Explains a shift
95
+ * that is running nothing new while work remains outstanding.
96
+ *
97
+ * NOT "the hub was not polled" — `opts.hub.wake(cursor)` runs every tick
98
+ * regardless of local capacity. Only the follow-up sweep is skipped, so this
99
+ * record means "there was news and I had nowhere to put it".
100
+ *
101
+ * EDGE-TRIGGERED: AT MOST one record per unbroken stretch at capacity, emitted
102
+ * when the shift ENTERS that state, not once per tick while it stays there. How
103
+ * long the stretch lasted is recoverable from the next record's `ts`.
104
+ *
105
+ * At most, not exactly: the emit sits inside `loop.ts`'s `changed && k <= 0`
106
+ * branch, so it needs a wake that reports a CHANGED cursor. A shift that fills
107
+ * up and stays full while the hub reports nothing new produces no record at all,
108
+ * and the absence of one is not evidence the shift had free slots.
109
+ *
110
+ * FILE-ONLY, for the same reason as `ParkedEvent`. On the socket it is both
111
+ * redundant and harmful: every `ShiftCapacity` response already carries live
112
+ * `cap`, `free`, and `running`, so this record restates on the wire what the
113
+ * response states anyway — and queueing it satisfies a parked `next` with news
114
+ * that nothing happened. In the file, which has no response envelope, it is the
115
+ * only thing distinguishing a saturated shift from an idle one.
116
+ */
117
+ export interface CapacityEvent {
118
+ type: 'capacity';
119
+ inFlight: number;
120
+ cap: number;
121
+ }
122
+ /** A hub call failed. `workflow` is present only for per-workflow calls. */
123
+ export interface HubErrorEvent {
124
+ type: 'hub-error';
125
+ op: 'wake' | 'whats_next';
126
+ workflow?: string;
127
+ message: string;
128
+ }
129
+ /** A legacy order named a def with no cached bundle, so it was left for pickup. */
130
+ export interface BundleMissEvent {
131
+ type: 'bundle-miss';
132
+ workflow: string;
133
+ def: string;
134
+ }
135
+ /**
136
+ * The shift refused one order and left it for the hub pickup window. A dropped
137
+ * unit of work, which is why it is a record and not a debug aside. `reason` is
138
+ * the stable machine discriminator; `message` is the human text.
139
+ */
140
+ export interface OrderDroppedEvent {
141
+ type: 'order-dropped';
142
+ workflow: string;
143
+ run: string;
144
+ step: string;
145
+ reason: 'malformed-digest' | 'malformed-worker' | 'unsupported-worker' | 'verification-failed' | 'metadata-unavailable';
146
+ message: string;
147
+ }
148
+ /**
149
+ * The socket event FIFO overflowed and discarded its oldest event. Written
150
+ * STRAIGHT to the file sink, never through the loop's `emit()` — `emit()` feeds
151
+ * the very queue that overflowed, so routing this through it would recurse
152
+ * under exactly the load that produced it. `dropped` is cumulative for the
153
+ * shift process, so each record states the running total lost so far.
154
+ */
155
+ export interface EventQueueOverflowEvent {
156
+ type: 'event-queue-overflow';
157
+ dropped: number;
158
+ }
159
+ /**
160
+ * What a construction site builds: the event's own payload, with no identity
161
+ * and no timestamp. Every consumer receives the stamped `ShiftEvent` instead.
162
+ */
163
+ export type ShiftEventBody = DispatchedEvent | ReapedEvent | FailedEvent | GateEvent | EndedEvent | ParkedEvent | CapacityEvent | HubErrorEvent | BundleMissEvent | OrderDroppedEvent | EventQueueOverflowEvent;
164
+ /**
165
+ * The identity every event carries, added once on a shared envelope rather than
166
+ * per variant.
167
+ *
168
+ * This is what makes one record SELF-DESCRIBING: a consumer holding a single
169
+ * line from an unknown machine can place it in time (`ts`) and attribute it to
170
+ * one shift process (`shiftId`) with no external context. That property is what
171
+ * lets the on-disk log, the socket, and a future uploader be three consumers of
172
+ * ONE contract instead of three shapes.
173
+ */
174
+ export interface ShiftEventEnvelope {
175
+ /** ISO-8601 UTC with milliseconds, e.g. `2026-08-13T18:04:11.412Z`. */
176
+ ts: string;
177
+ /** The shift's human name — live-mutable via `clock_in`, so it can change between records. */
178
+ shift: string;
179
+ /** The shift process incarnation's id (`shf_<uuid>`); `''` when undeclared. Stable for the process. */
180
+ shiftId: string;
181
+ }
182
+ /** A body plus its envelope: what every consumer actually receives. */
183
+ export type ShiftEvent = ShiftEventBody & ShiftEventEnvelope;
184
+ /** Who emitted an event. `id` is `''` when the emitter declared no shift id. */
185
+ export interface ShiftIdentity {
186
+ name: string;
187
+ id: string;
188
+ }
189
+ /**
190
+ * Stamp a body with its envelope. The ONE place the envelope is constructed —
191
+ * every emitter (the loop's `emit()`, the daemon's `ended`, the runtime's worker
192
+ * failures and startup record) routes through here so no consumer can receive a
193
+ * half-stamped record.
194
+ */
195
+ export declare function stampShiftEvent(body: ShiftEventBody, identity: ShiftIdentity, now: number): ShiftEvent;
73
196
  export interface ShiftCapacity {
74
197
  cap: number;
75
198
  free: number;