owenloop 0.5.27 → 0.5.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/dist/packages/work/src/agent/brief.d.ts +10 -0
  3. package/dist/packages/work/src/agent/brief.js +38 -0
  4. package/dist/packages/work/src/agent/loop.js +4 -0
  5. package/dist/packages/work/src/consumed-verifier.d.ts +22 -2
  6. package/dist/packages/work/src/consumed-verifier.js +131 -12
  7. package/dist/packages/work/src/contained-path.d.ts +18 -0
  8. package/dist/packages/work/src/contained-path.js +42 -0
  9. package/dist/packages/work/src/exec/instructions.js +43 -3
  10. package/dist/packages/work/src/harness/claude.js +21 -0
  11. package/dist/packages/work/src/harness/codex.js +29 -0
  12. package/dist/packages/work/src/harness/contract.d.ts +26 -0
  13. package/dist/packages/work/src/hold/mcp.d.ts +3 -40
  14. package/dist/packages/work/src/hold/mcp.js +121 -2
  15. package/dist/packages/work/src/hub/client.d.ts +10 -1
  16. package/dist/packages/work/src/hub/client.js +23 -0
  17. package/dist/packages/work/src/hub/types.d.ts +45 -0
  18. package/dist/packages/work/src/roles/shift.js +7 -1
  19. package/dist/packages/work/src/settings/settings.d.ts +12 -1
  20. package/dist/packages/work/src/settings/settings.js +7 -0
  21. package/dist/packages/work/src/shift/disk-floor.d.ts +107 -0
  22. package/dist/packages/work/src/shift/disk-floor.js +148 -0
  23. package/dist/packages/work/src/shift/host-preflight.d.ts +85 -0
  24. package/dist/packages/work/src/shift/host-preflight.js +112 -0
  25. package/dist/packages/work/src/shift/loop.d.ts +18 -0
  26. package/dist/packages/work/src/shift/loop.js +179 -7
  27. package/dist/packages/work/src/shift/protocol.d.ts +62 -1
  28. package/dist/packages/work/src/shift/runtime.d.ts +2 -0
  29. package/dist/packages/work/src/shift/runtime.js +17 -0
  30. package/dist/packages/work/src/submit-file.d.ts +3 -2
  31. package/dist/packages/work/src/submit-file.js +6 -22
  32. package/dist/src/crypto/records.d.ts +1 -0
  33. package/dist/src/crypto/records.js +1 -0
  34. package/dist/src/crypto/verify-consumed.d.ts +16 -1
  35. package/dist/src/crypto/verify-consumed.js +21 -6
  36. package/dist/src/schemas/index.d.ts +23 -0
  37. package/dist/src/schemas/order.v1.schema.json +14 -0
  38. package/dist/src/store/index.d.ts +1 -1
  39. package/dist/src/store/instruction-source.d.ts +15 -0
  40. package/dist/src/store/instruction-source.js +64 -4
  41. package/dist/src/store/types.d.ts +6 -2
  42. package/dist/src/store/types.js +5 -1
  43. package/dist/src/types.d.ts +11 -0
  44. package/docs/bundles.md +5 -1
  45. package/docs/cli.md +8 -5
  46. package/docs/crypto.md +46 -0
  47. package/docs/shift-logs.md +64 -7
  48. package/docs/wire-contracts.md +15 -2
  49. package/package.json +1 -1
  50. package/plugins/claude-code/plugin/.claude-plugin/plugin.json +1 -1
  51. package/plugins/claude-code/plugin/.mcp.json +1 -1
  52. package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +1 -1
  53. package/plugins/codex/plugins/owenloop/.mcp.json +1 -1
@@ -0,0 +1,85 @@
1
+ /**
2
+ * Is this machine still the machine the shift started on?
3
+ *
4
+ * A shift daemon is a long-lived process holding paths it resolved once, at
5
+ * start, and one of those paths is not stable for the life of the process:
6
+ * macOS gives each boot a fresh per-user temp directory
7
+ * (`/var/folders/<a>/<b>/T`). A process that survives a reboot — or was started
8
+ * before one and re-parented — keeps the OLD `TMPDIR` in its environment, and
9
+ * that directory is gone.
10
+ *
11
+ * The fault does not announce itself. The shift keeps polling, every hub call
12
+ * and every child spawn fails for a reason the error text attributes to the
13
+ * network ("fetch failed"), and the daemon looks alive to `pgrep` and answers
14
+ * `shift end` while dispatching nothing. It was measured wedged this way for 42
15
+ * minutes with no record naming the cause.
16
+ *
17
+ * WHAT THIS MODULE IS FOR, AND WHAT IT IS NOT FOR. It answers exactly one
18
+ * question: "is a LOCAL, non-self-healing fault the reason this shift cannot
19
+ * work?" It is deliberately not a health check and not a reachability probe. A
20
+ * hub outage, a Cloudflare 1015 ban, an expired credential and a flaky DHCP
21
+ * lease are all REMOTE and all self-heal without operator action, so this
22
+ * module must report them clean — the caller's whole decision rests on that
23
+ * split, because a clean answer means "keep polling" and a dirty one means
24
+ * "stop, a human must fix the host".
25
+ *
26
+ * PURE AND INJECTABLE. Every filesystem call is taken through `FsProbe` so the
27
+ * caller's tests can present a missing temp directory without one, and so a
28
+ * probe that itself throws is reported as a fault rather than crashing the
29
+ * shift.
30
+ */
31
+ /** Which of the shift's own directories a fault was found in. */
32
+ export type HostPathRole = 'temp-dir';
33
+ export interface HostFault {
34
+ /**
35
+ * The stable machine discriminator. The three kinds are separate because
36
+ * they have different operator fixes: a missing temp directory means the host
37
+ * rebooted and the shift must be restarted from a fresh shell, while an
38
+ * unwritable one means a permission change that restarting will not repair.
39
+ */
40
+ kind: 'missing' | 'not-a-directory' | 'unwritable';
41
+ role: HostPathRole;
42
+ path: string;
43
+ /** Operator-facing text: what is wrong, why it happened, and what to do. */
44
+ message: string;
45
+ }
46
+ /** The filesystem calls this check makes, as one injectable seam. */
47
+ export interface FsProbe {
48
+ isDirectory: (path: string) => boolean;
49
+ exists: (path: string) => boolean;
50
+ isWritable: (path: string) => boolean;
51
+ }
52
+ export declare const realFsProbe: FsProbe;
53
+ /**
54
+ * The path checked, and DELIBERATELY only this one.
55
+ *
56
+ * A shift also holds a state directory, a bundle cache directory and a work
57
+ * root, and none of them is here: the shift CREATES all three on demand
58
+ * (`resolveWorkRoot` defaults the work root to `<cacheDir>/work`, and
59
+ * `prepareWorkdir` mkdir -p's it), so their absence is a normal first-run
60
+ * condition rather than a fault. A check that treated it as one would refuse to
61
+ * start every fresh install — which is exactly what an earlier draft of this
62
+ * module did, and what the suite caught. The temp directory is the one path the
63
+ * shift depends on, does not create, and cannot recover on its own.
64
+ *
65
+ * The operator's `--work-root` list is a different thing again: it NARROWS
66
+ * which roots an order may name rather than supplying a directory the shift
67
+ * writes to, and losing one of several does not stop the shift serving orders
68
+ * that name the others. If it is ever worth checking, it is worth checking as
69
+ * its own decision, not folded in here.
70
+ */
71
+ export interface HostPaths {
72
+ /** Absent means "read the live `TMPDIR`", which is what the shift itself uses. */
73
+ tempDir?: string;
74
+ }
75
+ /**
76
+ * Report every local host fault, in a fixed role order so a reader comparing
77
+ * two records sees the same fault in the same position.
78
+ *
79
+ * The return is an array rather than a single fault so that adding a second
80
+ * role later does not change this function's contract, and so a caller never
81
+ * has to ask whether the one fault it was handed was the only one. An empty
82
+ * array means "no local fault" — which the caller must read as "the cause is
83
+ * elsewhere", never as "the shift is healthy".
84
+ */
85
+ export declare function checkHost(paths: HostPaths, fs?: FsProbe): HostFault[];
@@ -0,0 +1,112 @@
1
+ /**
2
+ * Is this machine still the machine the shift started on?
3
+ *
4
+ * A shift daemon is a long-lived process holding paths it resolved once, at
5
+ * start, and one of those paths is not stable for the life of the process:
6
+ * macOS gives each boot a fresh per-user temp directory
7
+ * (`/var/folders/<a>/<b>/T`). A process that survives a reboot — or was started
8
+ * before one and re-parented — keeps the OLD `TMPDIR` in its environment, and
9
+ * that directory is gone.
10
+ *
11
+ * The fault does not announce itself. The shift keeps polling, every hub call
12
+ * and every child spawn fails for a reason the error text attributes to the
13
+ * network ("fetch failed"), and the daemon looks alive to `pgrep` and answers
14
+ * `shift end` while dispatching nothing. It was measured wedged this way for 42
15
+ * minutes with no record naming the cause.
16
+ *
17
+ * WHAT THIS MODULE IS FOR, AND WHAT IT IS NOT FOR. It answers exactly one
18
+ * question: "is a LOCAL, non-self-healing fault the reason this shift cannot
19
+ * work?" It is deliberately not a health check and not a reachability probe. A
20
+ * hub outage, a Cloudflare 1015 ban, an expired credential and a flaky DHCP
21
+ * lease are all REMOTE and all self-heal without operator action, so this
22
+ * module must report them clean — the caller's whole decision rests on that
23
+ * split, because a clean answer means "keep polling" and a dirty one means
24
+ * "stop, a human must fix the host".
25
+ *
26
+ * PURE AND INJECTABLE. Every filesystem call is taken through `FsProbe` so the
27
+ * caller's tests can present a missing temp directory without one, and so a
28
+ * probe that itself throws is reported as a fault rather than crashing the
29
+ * shift.
30
+ */
31
+ import { accessSync, constants, statSync } from 'node:fs';
32
+ import { tmpdir } from 'node:os';
33
+ export const realFsProbe = {
34
+ exists: (path) => {
35
+ try {
36
+ statSync(path);
37
+ return true;
38
+ }
39
+ catch {
40
+ return false;
41
+ }
42
+ },
43
+ isDirectory: (path) => {
44
+ try {
45
+ return statSync(path).isDirectory();
46
+ }
47
+ catch {
48
+ return false;
49
+ }
50
+ },
51
+ isWritable: (path) => {
52
+ try {
53
+ accessSync(path, constants.W_OK);
54
+ return true;
55
+ }
56
+ catch {
57
+ return false;
58
+ }
59
+ },
60
+ };
61
+ const REMEDY = {
62
+ 'temp-dir': 'the host\'s temp directory changed, which a reboot does — stop this shift and start a new one from a fresh shell so it inherits the current TMPDIR',
63
+ };
64
+ const LABEL = {
65
+ 'temp-dir': 'TMPDIR',
66
+ };
67
+ function checkOne(role, path, fs) {
68
+ if (!fs.exists(path)) {
69
+ return { kind: 'missing', role, path, message: `${LABEL[role]} '${path}' no longer exists — ${REMEDY[role]}` };
70
+ }
71
+ if (!fs.isDirectory(path)) {
72
+ return {
73
+ kind: 'not-a-directory',
74
+ role,
75
+ path,
76
+ message: `${LABEL[role]} '${path}' is not a directory — ${REMEDY[role]}`,
77
+ };
78
+ }
79
+ if (!fs.isWritable(path)) {
80
+ return {
81
+ kind: 'unwritable',
82
+ role,
83
+ path,
84
+ message: `${LABEL[role]} '${path}' is not writable by this process — ${REMEDY[role]}`,
85
+ };
86
+ }
87
+ return undefined;
88
+ }
89
+ /**
90
+ * Report every local host fault, in a fixed role order so a reader comparing
91
+ * two records sees the same fault in the same position.
92
+ *
93
+ * The return is an array rather than a single fault so that adding a second
94
+ * role later does not change this function's contract, and so a caller never
95
+ * has to ask whether the one fault it was handed was the only one. An empty
96
+ * array means "no local fault" — which the caller must read as "the cause is
97
+ * elsewhere", never as "the shift is healthy".
98
+ */
99
+ export function checkHost(paths, fs = realFsProbe) {
100
+ const candidates = [
101
+ ['temp-dir', paths.tempDir ?? tmpdir()],
102
+ ];
103
+ const faults = [];
104
+ for (const [role, path] of candidates) {
105
+ if (path === undefined || path === '')
106
+ continue;
107
+ const fault = checkOne(role, path, fs);
108
+ if (fault !== undefined)
109
+ faults.push(fault);
110
+ }
111
+ return faults;
112
+ }
@@ -4,6 +4,8 @@ import { type WorkOrder } from '../hub/types.ts';
4
4
  import type { FetchedStep } from '../bundle/types.ts';
5
5
  import { type ReconcileOptions, type Liveness } from './state.ts';
6
6
  import { sweepWorkDirs as sweepWorkDirsImpl } from '../agent/workdir.ts';
7
+ import { type HostFault } from './host-preflight.ts';
8
+ import { type DiskSpace } from './disk-floor.ts';
7
9
  import type { Spawner, WorkerFailure } from './spawn.ts';
8
10
  import { type ShiftEvent } from './protocol.ts';
9
11
  export interface ShiftLoopOptions {
@@ -104,6 +106,22 @@ export interface ShiftLoopOptions {
104
106
  /** PHASE 4 — injected in tests so the reaper can be exercised without a
105
107
  * filesystem. Defaults to `sweepWorkDirs` from `src/agent/workdir.ts`. */
106
108
  sweepWorkDirs?: typeof sweepWorkDirsImpl;
109
+ /**
110
+ * Does this machine still have the directories the shift needs? Injected in
111
+ * tests so a vanished temp directory can be presented without one, and so the
112
+ * check itself can be made to throw. Defaults to `checkHost`, which is
113
+ * `stat`-only and issues no hub traffic. An empty array means "no LOCAL
114
+ * fault", never "healthy" — see `host-preflight.ts`.
115
+ */
116
+ hostPreflight?: () => HostFault[];
117
+ /**
118
+ * Is there room on this disk to start another order? Injected in tests so a
119
+ * full volume can be presented without filling one. Defaults to a real
120
+ * `statfs` of `workRoot` (or its nearest existing ancestor) against the
121
+ * resolved floor. `unknown` means "could not measure, or switched off" and is
122
+ * treated as permission to dispatch — see `disk-floor.ts`.
123
+ */
124
+ diskSpace?: () => DiskSpace;
107
125
  }
108
126
  export interface LockedRemovalOptions {
109
127
  dispatchLockOptions?: Pick<AcquireFileLockOpts, 'beforeOpen'>;
@@ -63,6 +63,8 @@ import { cancelReservedChild, finalizeChildReservation, reconcileInFlight, reser
63
63
  import { DEFAULT_WORK_DIR_TTL_MS, sweepWorkDirs as sweepWorkDirsImpl } from "../agent/workdir.js";
64
64
  import { withHubRosterSyncTimeout } from "../settings/hub-roster-cache.js";
65
65
  import { sessionsPath } from "../harness/session-store.js";
66
+ import { checkHost } from "./host-preflight.js";
67
+ import { formatBytes } from "./disk-floor.js";
66
68
  import { stampShiftEvent, } from "./protocol.js";
67
69
  /** Run a state mutation while holding the lock that serializes dispatch state. */
68
70
  export function withDispatchLock(stateDir, options = {}, fn) {
@@ -196,12 +198,53 @@ function stepBrakeDelayMs(count) {
196
198
  const index = Math.min(count, STEP_BRAKE_DELAYS_MS.length) - 1;
197
199
  return STEP_BRAKE_DELAYS_MS[index] ?? 0;
198
200
  }
201
+ /** How many links of an error's `cause` chain `errMsg` renders. */
202
+ const MAX_CAUSE_DEPTH = 4;
203
+ /**
204
+ * Consecutive failed hub polls before the shift asks whether the fault is local.
205
+ *
206
+ * NOT a timeout and not a health threshold — nothing is declared broken at this
207
+ * number. Crossing it only makes the shift run `checkHost`, which is a handful
208
+ * of `stat` calls and is free of hub traffic; if the host is fine the shift
209
+ * keeps polling and the streak keeps climbing. So the number only has to be
210
+ * large enough that a single transient failure does not pay for the check, and
211
+ * small enough that an operator is not left guessing for long. At the shipped
212
+ * 30s poll interval three failures is about a minute and a half.
213
+ */
214
+ const HOST_PREFLIGHT_STREAK = 3;
199
215
  /** The identity a storm repeats on. `key` is '' for an unfanned step. */
200
216
  function stepBrakeKey(workflow, step, key) {
201
217
  return `${workflow}\u0000${step}\u0000${key ?? ''}`;
202
218
  }
219
+ /**
220
+ * An error's message, with its cause chain appended.
221
+ *
222
+ * THE CHAIN IS THE POINT. Node's fetch reports every transport failure as the
223
+ * bare string `fetch failed` and puts the real reason — `ENOENT`, `ECONNREFUSED`,
224
+ * `EAI_AGAIN`, a TLS error — on `cause`. Dropping it makes a dead local host, an
225
+ * unreachable hub and a DNS failure produce byte-identical records, which is
226
+ * precisely how a wedged shift stayed unattributed for 42 minutes.
227
+ *
228
+ * Bounded by construction: the walk follows at most `MAX_CAUSE_DEPTH` links and
229
+ * refuses to revisit a cause it has already rendered, so a self-referential or
230
+ * mutually-referential chain terminates instead of hanging the emitter.
231
+ */
203
232
  function errMsg(e) {
204
- return e instanceof Error ? e.message : String(e);
233
+ if (!(e instanceof Error))
234
+ return String(e);
235
+ const parts = [e.message];
236
+ const seen = new Set([e]);
237
+ let cause = e.cause;
238
+ for (let depth = 0; depth < MAX_CAUSE_DEPTH && cause !== undefined && cause !== null; depth += 1) {
239
+ if (seen.has(cause))
240
+ break;
241
+ seen.add(cause);
242
+ const text = cause instanceof Error ? cause.message : String(cause);
243
+ if (text !== '' && !parts.includes(text))
244
+ parts.push(text);
245
+ cause = cause instanceof Error ? cause.cause : undefined;
246
+ }
247
+ return parts.join(': ');
205
248
  }
206
249
  /** The explicit hub refusal produced when an inbox row becomes terminal after
207
250
  * the inbox snapshot but before its targeted serving call. This is a normal
@@ -298,8 +341,34 @@ export function createShiftLoop(opts) {
298
341
  * deliberately NOT edge-triggered.
299
342
  */
300
343
  let atCapReported = false;
344
+ /**
345
+ * EDGE flag for the low-disk refusal, paired with `atCapReported` because the
346
+ * two gates have the same shape: a condition that persists across many ticks
347
+ * but is worth exactly one record per episode. Cleared the moment the disk
348
+ * measures healthy again, so a machine that fills and drains twice writes two
349
+ * records rather than one.
350
+ */
351
+ let lowDiskReported = false;
301
352
  /** Earliest server-approved time for the next polling iteration. */
302
353
  let backoffUntil = Number.NEGATIVE_INFINITY;
354
+ /**
355
+ * Consecutive hub polls that failed for a reason the hub did not sanction.
356
+ *
357
+ * RATE LIMITING IS EXCLUDED, and that exclusion is the whole design. A 429 is
358
+ * the server saying "come back later", and a Cloudflare 1015 ban is the same
359
+ * answer at the edge: both are recoverable degradation that clears on its own,
360
+ * and a shift that killed itself over one would take the whole fleet down for
361
+ * a condition the fleet itself caused. `noteServerBackoff` already returns
362
+ * exactly that discrimination, so the streak counts only the failures nobody
363
+ * promised would end.
364
+ *
365
+ * Reset by any SUCCESSFUL wake, which is the loop's one unconditional hub call
366
+ * per unbackoffed tick and therefore the cheapest honest proof that the hub is
367
+ * reachable and this host can reach it.
368
+ */
369
+ let hubFailureStreak = 0;
370
+ /** Set once the shift has stopped for a named local fault, so `run` exits non-zero. */
371
+ let wedged = false;
303
372
  /** Bumped whenever a ping starts or is forced due, so a ping that completes
304
373
  * after a force does not stamp the cadence over that force. */
305
374
  let presenceGeneration = 0;
@@ -377,6 +446,58 @@ export function createShiftLoop(opts) {
377
446
  backoffUntil = Math.max(backoffUntil, monotonicNow() + delay);
378
447
  return true;
379
448
  }
449
+ /**
450
+ * Ask whether a LOCAL fault is the reason this shift cannot work, and stop if
451
+ * one is. Returns true when the shift has been wedged.
452
+ *
453
+ * STOPPING IS THE POINT, and it is the opposite of what every other failure
454
+ * path here does. Everything else in this loop is built to survive: a failed
455
+ * wake retries next tick, a failed roster sync degrades and continues, a
456
+ * rate-limited poll backs off. All of that is correct for a REMOTE fault,
457
+ * which ends without anyone doing anything. It is exactly wrong for a local
458
+ * one, where continuing means an operator watching a live process that will
459
+ * never dispatch again — the measured shape of this defect was a shift that
460
+ * looked alive to `pgrep`, answered `shift end`, and did nothing for 42
461
+ * minutes while attributing a vanished temp directory to `fetch failed`.
462
+ *
463
+ * A PRE-FLIGHT THAT ITSELF FAILS IS NOT A FAULT. If the probe throws, this
464
+ * reports it and returns false, because "I could not determine whether the
465
+ * host is broken" is not evidence that it is, and killing a working shift over
466
+ * an unreadable `stat` would be a worse failure than the one being detected.
467
+ */
468
+ /**
469
+ * Free space for the volume the shift's children will write into.
470
+ *
471
+ * Defaults to `unknown` — never a refusal — when no probe was injected and no
472
+ * work root was resolved: the floor and the work root are both RUNTIME
473
+ * configuration, and a loop constructed without them has no basis to decline
474
+ * anything. `runtime.ts` supplies the real closure.
475
+ */
476
+ function diskSpace() {
477
+ if (opts.diskSpace !== undefined)
478
+ return opts.diskSpace();
479
+ return { state: 'unknown', path: opts.workRoot ?? '' };
480
+ }
481
+ function runHostPreflight(streak) {
482
+ let faults;
483
+ try {
484
+ faults = opts.hostPreflight !== undefined
485
+ ? opts.hostPreflight()
486
+ : checkHost({});
487
+ }
488
+ catch (e) {
489
+ opts.err(`host pre-flight failed to run: ${errMsg(e)} (continuing)`);
490
+ return false;
491
+ }
492
+ if (faults.length === 0)
493
+ return false;
494
+ for (const fault of faults)
495
+ opts.err(`shift cannot continue: ${fault.message}`);
496
+ wedged = true;
497
+ stopped = true;
498
+ emit({ type: 'wedged', faults, streak });
499
+ return true;
500
+ }
380
501
  /**
381
502
  * Fan one observation to every consumer, stamped.
382
503
  *
@@ -1322,11 +1443,24 @@ export function createShiftLoop(opts) {
1322
1443
  changed = w.changed;
1323
1444
  cursor = w.cursor;
1324
1445
  wakeSucceeded = true;
1446
+ // The loop's one unconditional hub call per unbackoffed tick, so its
1447
+ // success is the cheapest honest proof that this host can reach the hub.
1448
+ hubFailureStreak = 0;
1325
1449
  }
1326
1450
  catch (e) {
1327
- noteServerBackoff(e);
1451
+ const rateLimited = noteServerBackoff(e);
1328
1452
  opts.err(`wake failed: ${errMsg(e)} (retrying next tick)`);
1329
1453
  emit({ type: 'hub-error', op: 'wake', message: errMsg(e) });
1454
+ if (!rateLimited) {
1455
+ hubFailureStreak += 1;
1456
+ // Every HOST_PREFLIGHT_STREAK-th failure, not only the first crossing:
1457
+ // a work root can be unmounted at failure seven, and a shift that
1458
+ // checked once at failure three would never see it. The check is
1459
+ // `stat`-only and adds no hub traffic, so repeating it is nearly free.
1460
+ if (hubFailureStreak % HOST_PREFLIGHT_STREAK === 0 && runHostPreflight(hubFailureStreak)) {
1461
+ return dispatched;
1462
+ }
1463
+ }
1330
1464
  }
1331
1465
  // A changed cursor is not sufficient by itself: if the prior changed tick
1332
1466
  // had no local capacity, or its sweep failed, the adopted cursor will be
@@ -1339,7 +1473,38 @@ export function createShiftLoop(opts) {
1339
1473
  // can actually make the targeted decision.
1340
1474
  const capacityCooldownDue = capacityCooldownDueAt !== undefined && monotonicNow() >= capacityCooldownDueAt;
1341
1475
  const capacityBecameReady = capacityCooldownReady(k, agentRoom);
1342
- if (wakeSucceeded && (changed || sweepOwed || brakeDue || capacityCooldownDue || capacityBecameReady) && k > 0) {
1476
+ const wantSweep = wakeSucceeded && (changed || sweepOwed || brakeDue || capacityCooldownDue || capacityBecameReady) && k > 0;
1477
+ /*
1478
+ * Measured ONLY when a sweep would otherwise happen, so a parked or
1479
+ * saturated shift pays no syscall for a question it is not about to act on.
1480
+ * That placement is also what makes recovery automatic: refusing sets
1481
+ * `sweepOwed`, `sweepOwed` makes the next tick want to sweep, and wanting to
1482
+ * sweep re-measures the disk. No timer and no separate recovery path.
1483
+ */
1484
+ const disk = wantSweep ? diskSpace() : undefined;
1485
+ if (wantSweep && disk?.state === 'low') {
1486
+ /*
1487
+ * REFUSE NEW WORK, DO NOT STOP. Unlike the vanished temp directory that
1488
+ * `runHostPreflight` exits for, a full disk clears on its own, so the
1489
+ * shift stays up and keeps polling. In-flight children are deliberately
1490
+ * left running: killing one frees little and destroys all of its work.
1491
+ */
1492
+ sweepOwed = true;
1493
+ opts.out(`low disk: ${formatBytes(disk.freeBytes)} free at ${disk.path}, below the ${formatBytes(disk.floorBytes)} floor — ` +
1494
+ `deferring whats_next until space is free`);
1495
+ // EDGE-TRIGGERED, exactly as the at-capacity record below: one per
1496
+ // low-disk EPISODE. The console line above stays level-triggered because
1497
+ // a live tail is watched by someone who wants to see it is still stuck.
1498
+ if (!lowDiskReported) {
1499
+ lowDiskReported = true;
1500
+ emit({ type: 'low-disk', path: disk.path, freeBytes: disk.freeBytes, floorBytes: disk.floorBytes });
1501
+ }
1502
+ }
1503
+ else if (wantSweep) {
1504
+ if (lowDiskReported) {
1505
+ lowDiskReported = false;
1506
+ opts.out('disk space recovered — resuming dispatch');
1507
+ }
1343
1508
  // Clear before awaiting: a new brake armed during sweep must survive.
1344
1509
  if (brakeDue)
1345
1510
  brakeSweepDueAt = undefined;
@@ -1394,21 +1559,28 @@ export function createShiftLoop(opts) {
1394
1559
  }
1395
1560
  async function run() {
1396
1561
  ensureStateDir(opts.stateDir);
1562
+ // BEFORE THE FIRST HUB CALL. A shift started into a broken host — the usual
1563
+ // way being a supervisor that restarts it across a reboot with the old
1564
+ // environment — must say so at once. Making it wait for three failed polls
1565
+ // would buy nothing and spend the operator's attention on network theories.
1566
+ // `streak: 0` marks this record as the boot check rather than a poll one.
1567
+ if (runHostPreflight(0))
1568
+ return 1;
1397
1569
  if (opts.once === true) {
1398
1570
  await iteration();
1399
- return 0;
1571
+ return wedged ? 1 : 0;
1400
1572
  }
1401
1573
  for (;;) {
1402
1574
  if (stopped)
1403
- return 0;
1575
+ return wedged ? 1 : 0;
1404
1576
  await iteration();
1405
1577
  if (stopped)
1406
- return 0;
1578
+ return wedged ? 1 : 0;
1407
1579
  const delay = Math.max(opts.pollIntervalMs, backoffUntil - monotonicNow());
1408
1580
  await opts.sleep(delay);
1409
1581
  // Re-check after the park so a stop during sleep makes no further hub call.
1410
1582
  if (stopped)
1411
- return 0;
1583
+ return wedged ? 1 : 0;
1412
1584
  }
1413
1585
  }
1414
1586
  function stop() {
@@ -6,6 +6,7 @@
6
6
  * workflow state; this protocol only carries local dispatch observations and
7
7
  * attendance commands between the CLI and one daemon process.
8
8
  */
9
+ import type { HostFault } from './host-preflight.ts';
9
10
  export declare const SHIFT_SOCKET_NAME = "shift.sock";
10
11
  export declare const DEFAULT_NEXT_WAIT_MS = 90000;
11
12
  export declare const MAX_REQUEST_LINE_BYTES: number;
@@ -119,6 +120,30 @@ export interface CapacityEvent {
119
120
  inFlight: number;
120
121
  cap: number;
121
122
  }
123
+ /**
124
+ * The shift declined to start new work because the volume holding its work root
125
+ * is nearly full.
126
+ *
127
+ * EDGE-TRIGGERED, like `CapacityEvent`: one record per low-disk EPISODE, not one
128
+ * per tick. A shift that sits on a full disk for an hour writes this once, and
129
+ * the recovery is legible from the timestamp of whatever it dispatches next.
130
+ *
131
+ * FILE-ONLY, for the same reason as `CapacityEvent` and `ParkedEvent`: a parked
132
+ * `shift next` must not be woken by news that nothing happened. In the file it
133
+ * is the only thing separating "this shift had no work" from "this shift had
134
+ * work and could not safely take it".
135
+ *
136
+ * `freeBytes` and `floorBytes` are both carried so the record is self-contained.
137
+ * A reader must be able to tell an operator who set an unreachable floor from a
138
+ * genuinely full disk without knowing what the shift was configured with.
139
+ */
140
+ export interface LowDiskEvent {
141
+ type: 'low-disk';
142
+ /** The measured directory: the work root, or its nearest existing ancestor. */
143
+ path: string;
144
+ freeBytes: number;
145
+ floorBytes: number;
146
+ }
122
147
  /** A hub call failed. `workflow` is present for per-workflow whats_next and release calls. */
123
148
  export interface HubErrorEvent {
124
149
  type: 'hub-error';
@@ -146,6 +171,42 @@ export interface OrderDroppedEvent {
146
171
  reason: 'malformed-digest' | 'malformed-worker' | 'unsupported-worker' | 'verification-failed' | 'metadata-unavailable' | 'agent-lane-closed' | 'dispatch-cap-full' | 'agent-cap-full' | 'claim-expired';
147
172
  message: string;
148
173
  }
174
+ /**
175
+ * The shift STOPPED because the machine underneath it is broken, and named the
176
+ * fault. The last record a wedged shift writes; the process exits non-zero
177
+ * immediately after.
178
+ *
179
+ * WHY THIS EXISTS AS ITS OWN RECORD, rather than one more `hub-error`. A shift
180
+ * whose host temp directory vanished — which a macOS reboot does to any process
181
+ * that survives it — keeps polling forever, and every hub call fails with
182
+ * undici's generic `fetch failed`. From the outside that is indistinguishable
183
+ * from a hub outage: `pgrep` shows the process, `shift end` answers, and the
184
+ * log fills with records that attribute a LOCAL fault to the network. This
185
+ * record is the one that says the cause is here, on this machine, and that no
186
+ * amount of waiting will fix it.
187
+ *
188
+ * `faults` is never empty. That invariant is what separates this record from
189
+ * every retryable failure: a hub outage, a rate-limit ban and an expired
190
+ * credential all self-heal, so the shift keeps working through them and emits
191
+ * nothing here. A record exists only when a human must change something on the
192
+ * host.
193
+ *
194
+ * `streak` is how many consecutive hub calls failed before the check ran, and
195
+ * is 0 for the pre-flight the shift runs at start, before any hub call. It is
196
+ * evidence, not a cause: the streak is only what PROMPTED the check, and the
197
+ * `faults` are why the shift stopped.
198
+ *
199
+ * ON THE SOCKET AS WELL AS IN THE FILE, unlike `parked` and `capacity`. A
200
+ * parked `owenloop shift next` must block until there is something to report,
201
+ * and "the shift you are waiting on is dead" is the most report-worthy thing
202
+ * that can happen to it — a client that stays parked on a socket whose server
203
+ * is about to close waits for work that will never come.
204
+ */
205
+ export interface WedgedEvent {
206
+ type: 'wedged';
207
+ faults: HostFault[];
208
+ streak: number;
209
+ }
149
210
  /**
150
211
  * The socket event FIFO overflowed and discarded its oldest event. Written
151
212
  * STRAIGHT to the file sink, never through the loop's `emit()` — `emit()` feeds
@@ -161,7 +222,7 @@ export interface EventQueueOverflowEvent {
161
222
  * What a construction site builds: the event's own payload, with no identity
162
223
  * and no timestamp. Every consumer receives the stamped `ShiftEvent` instead.
163
224
  */
164
- export type ShiftEventBody = DispatchedEvent | ReapedEvent | FailedEvent | GateEvent | EndedEvent | ParkedEvent | CapacityEvent | HubErrorEvent | BundleMissEvent | OrderDroppedEvent | EventQueueOverflowEvent;
225
+ export type ShiftEventBody = DispatchedEvent | ReapedEvent | FailedEvent | GateEvent | EndedEvent | ParkedEvent | CapacityEvent | HubErrorEvent | BundleMissEvent | OrderDroppedEvent | EventQueueOverflowEvent | WedgedEvent | LowDiskEvent;
165
226
  /**
166
227
  * The identity every event carries, added once on a shared envelope rather than
167
228
  * per variant.
@@ -35,6 +35,8 @@ export interface ParsedArgs {
35
35
  once?: boolean;
36
36
  maxAgents?: number;
37
37
  execReserve?: number;
38
+ /** `--disk-floor <bytes>` — free space required to start new work. `0` disables. */
39
+ diskFloorBytes?: number;
38
40
  localQueueHoldMs?: number;
39
41
  cacheDir?: string;
40
42
  stateDir?: string;
@@ -16,6 +16,7 @@ import { loadSettings } from "../settings/settings.js";
16
16
  import { DEFAULT_HUB_ROSTER_SYNC_TIMEOUT_MS, syncHubRosterCache, withHubRosterSyncTimeout } from "../settings/hub-roster-cache.js";
17
17
  import { computeServeCapabilities } from "../settings/serving.js";
18
18
  import { resolveCacheDir } from "../bundle/cache.js";
19
+ import { checkDiskFloor, resolveDiskFloorBytes } from "./disk-floor.js";
19
20
  import { createLockedRemovalCallbacks, createShiftLoop } from "./loop.js";
20
21
  import { createHubBundleRecoveryHandler } from "../bundle/pull.js";
21
22
  import { createShiftLogSink } from "./logsink.js";
@@ -93,6 +94,7 @@ const FILE_ONLY_EVENTS = new Set([
93
94
  'capacity',
94
95
  'hub-error',
95
96
  'event-queue-overflow',
97
+ 'low-disk',
96
98
  ]);
97
99
  /**
98
100
  * Does this event type reach the SOCKET consumer (a parked `owenloop shift
@@ -161,6 +163,7 @@ export function parseArgs(args) {
161
163
  case '--cap':
162
164
  case '--max-agents':
163
165
  case '--exec-reserve':
166
+ case '--disk-floor':
164
167
  case '--local-queue-hold':
165
168
  case '--workflow':
166
169
  case '--poll-interval':
@@ -217,6 +220,12 @@ export function parseArgs(args) {
217
220
  return { error: n.error };
218
221
  parsed.execReserve = n;
219
222
  }
223
+ else if (name === '--disk-floor') {
224
+ const n = intFlag(r.value, '--disk-floor');
225
+ if (typeof n !== 'number')
226
+ return { error: n.error };
227
+ parsed.diskFloorBytes = n;
228
+ }
220
229
  else if (name === '--local-queue-hold') {
221
230
  const n = intFlag(r.value, '--local-queue-hold');
222
231
  if (typeof n !== 'number')
@@ -404,6 +413,7 @@ export async function runShiftRuntime(parsed, options = {}) {
404
413
  const maxConcurrentAgents = resolveMaxConcurrentAgents(parsed.maxAgents, settings.maxConcurrentAgents);
405
414
  const execReserve = resolveExecReserve(parsed.execReserve, settings.execReserve);
406
415
  const localQueueHoldMs = resolveLocalQueueHoldMs(parsed.localQueueHoldMs, settings.localQueueHoldMs);
416
+ const diskFloorBytes = resolveDiskFloorBytes(parsed.diskFloorBytes, settings.diskFloorBytes);
407
417
  const workRoot = resolveWorkRoot(env, settings.workRoot, cacheDir);
408
418
  const workRepo = resolveWorkRepo(env, settings.workRepo);
409
419
  /**
@@ -588,6 +598,13 @@ export async function runShiftRuntime(parsed, options = {}) {
588
598
  execReserve,
589
599
  localQueueHoldMs,
590
600
  workRoot,
601
+ /*
602
+ * Resolved here, not in the loop: `runtime.ts` owns flag/settings/default
603
+ * precedence for every other tunable, and the loop consumes a decision. A
604
+ * loop built without this closure has no disk gate at all, which is the
605
+ * right default for a caller that never told it where work lands.
606
+ */
607
+ diskSpace: () => checkDiskFloor(workRoot, diskFloorBytes),
591
608
  ...(workRepo !== undefined ? { workRepo } : {}),
592
609
  shiftId,
593
610
  startedAt,
@@ -1,7 +1,8 @@
1
1
  /**
2
2
  * Read a submit value from a JSON document contained in a run worktree.
3
3
  *
4
- * The lexical check rejects traversal before probing the filesystem. The
5
- * canonical check rejects symlinks whose target leaves the worktree.
4
+ * Containment is `resolveContainedPath`'s job — lexical traversal check first,
5
+ * then a canonical symlink check. This function owns only what is specific to
6
+ * a submit value: the argument shape and the JSON decode.
6
7
  */
7
8
  export declare function readSubmitValueFile(workdir: string, valueFile: unknown): Promise<unknown>;