owenloop 0.5.26 → 0.5.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/dist/packages/work/src/agent/brief.d.ts +10 -0
  3. package/dist/packages/work/src/agent/brief.js +38 -0
  4. package/dist/packages/work/src/agent/loop.js +4 -0
  5. package/dist/packages/work/src/contained-path.d.ts +18 -0
  6. package/dist/packages/work/src/contained-path.js +42 -0
  7. package/dist/packages/work/src/harness/claude.js +21 -0
  8. package/dist/packages/work/src/harness/codex.js +29 -0
  9. package/dist/packages/work/src/harness/contract.d.ts +26 -0
  10. package/dist/packages/work/src/hold/mcp.d.ts +3 -40
  11. package/dist/packages/work/src/hold/mcp.js +121 -2
  12. package/dist/packages/work/src/hub/client.d.ts +10 -1
  13. package/dist/packages/work/src/hub/client.js +23 -0
  14. package/dist/packages/work/src/hub/types.d.ts +31 -0
  15. package/dist/packages/work/src/roles/shift.js +7 -1
  16. package/dist/packages/work/src/settings/settings.d.ts +12 -1
  17. package/dist/packages/work/src/settings/settings.js +7 -0
  18. package/dist/packages/work/src/shift/disk-floor.d.ts +107 -0
  19. package/dist/packages/work/src/shift/disk-floor.js +148 -0
  20. package/dist/packages/work/src/shift/host-preflight.d.ts +85 -0
  21. package/dist/packages/work/src/shift/host-preflight.js +112 -0
  22. package/dist/packages/work/src/shift/loop.d.ts +18 -0
  23. package/dist/packages/work/src/shift/loop.js +179 -7
  24. package/dist/packages/work/src/shift/protocol.d.ts +62 -1
  25. package/dist/packages/work/src/shift/runtime.d.ts +2 -0
  26. package/dist/packages/work/src/shift/runtime.js +17 -0
  27. package/dist/packages/work/src/submit-file.d.ts +3 -2
  28. package/dist/packages/work/src/submit-file.js +6 -22
  29. package/dist/src/defs.js +59 -8
  30. package/dist/src/types.d.ts +13 -0
  31. package/docs/cli.md +8 -5
  32. package/docs/shift-logs.md +64 -7
  33. package/package.json +1 -1
  34. package/plugins/claude-code/plugin/.claude-plugin/plugin.json +1 -1
  35. package/plugins/claude-code/plugin/.mcp.json +1 -1
  36. package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +1 -1
  37. package/plugins/codex/plugins/owenloop/.mcp.json +1 -1
@@ -63,6 +63,8 @@ import { cancelReservedChild, finalizeChildReservation, reconcileInFlight, reser
63
63
  import { DEFAULT_WORK_DIR_TTL_MS, sweepWorkDirs as sweepWorkDirsImpl } from "../agent/workdir.js";
64
64
  import { withHubRosterSyncTimeout } from "../settings/hub-roster-cache.js";
65
65
  import { sessionsPath } from "../harness/session-store.js";
66
+ import { checkHost } from "./host-preflight.js";
67
+ import { formatBytes } from "./disk-floor.js";
66
68
  import { stampShiftEvent, } from "./protocol.js";
67
69
  /** Run a state mutation while holding the lock that serializes dispatch state. */
68
70
  export function withDispatchLock(stateDir, options = {}, fn) {
@@ -196,12 +198,53 @@ function stepBrakeDelayMs(count) {
196
198
  const index = Math.min(count, STEP_BRAKE_DELAYS_MS.length) - 1;
197
199
  return STEP_BRAKE_DELAYS_MS[index] ?? 0;
198
200
  }
201
+ /** How many links of an error's `cause` chain `errMsg` renders. */
202
+ const MAX_CAUSE_DEPTH = 4;
203
+ /**
204
+ * Consecutive failed hub polls before the shift asks whether the fault is local.
205
+ *
206
+ * NOT a timeout and not a health threshold — nothing is declared broken at this
207
+ * number. Crossing it only makes the shift run `checkHost`, which is a handful
208
+ * of `stat` calls and is free of hub traffic; if the host is fine the shift
209
+ * keeps polling and the streak keeps climbing. So the number only has to be
210
+ * large enough that a single transient failure does not pay for the check, and
211
+ * small enough that an operator is not left guessing for long. At the shipped
212
+ * 30s poll interval three failures is about a minute and a half.
213
+ */
214
+ const HOST_PREFLIGHT_STREAK = 3;
199
215
  /** The identity a storm repeats on. `key` is '' for an unfanned step. */
200
216
  function stepBrakeKey(workflow, step, key) {
201
217
  return `${workflow}\u0000${step}\u0000${key ?? ''}`;
202
218
  }
219
+ /**
220
+ * An error's message, with its cause chain appended.
221
+ *
222
+ * THE CHAIN IS THE POINT. Node's fetch reports every transport failure as the
223
+ * bare string `fetch failed` and puts the real reason — `ENOENT`, `ECONNREFUSED`,
224
+ * `EAI_AGAIN`, a TLS error — on `cause`. Dropping it makes a dead local host, an
225
+ * unreachable hub and a DNS failure produce byte-identical records, which is
226
+ * precisely how a wedged shift stayed unattributed for 42 minutes.
227
+ *
228
+ * Bounded by construction: the walk follows at most `MAX_CAUSE_DEPTH` links and
229
+ * refuses to revisit a cause it has already rendered, so a self-referential or
230
+ * mutually-referential chain terminates instead of hanging the emitter.
231
+ */
203
232
  function errMsg(e) {
204
- return e instanceof Error ? e.message : String(e);
233
+ if (!(e instanceof Error))
234
+ return String(e);
235
+ const parts = [e.message];
236
+ const seen = new Set([e]);
237
+ let cause = e.cause;
238
+ for (let depth = 0; depth < MAX_CAUSE_DEPTH && cause !== undefined && cause !== null; depth += 1) {
239
+ if (seen.has(cause))
240
+ break;
241
+ seen.add(cause);
242
+ const text = cause instanceof Error ? cause.message : String(cause);
243
+ if (text !== '' && !parts.includes(text))
244
+ parts.push(text);
245
+ cause = cause instanceof Error ? cause.cause : undefined;
246
+ }
247
+ return parts.join(': ');
205
248
  }
206
249
  /** The explicit hub refusal produced when an inbox row becomes terminal after
207
250
  * the inbox snapshot but before its targeted serving call. This is a normal
@@ -298,8 +341,34 @@ export function createShiftLoop(opts) {
298
341
  * deliberately NOT edge-triggered.
299
342
  */
300
343
  let atCapReported = false;
344
+ /**
345
+ * EDGE flag for the low-disk refusal, paired with `atCapReported` because the
346
+ * two gates have the same shape: a condition that persists across many ticks
347
+ * but is worth exactly one record per episode. Cleared the moment the disk
348
+ * measures healthy again, so a machine that fills and drains twice writes two
349
+ * records rather than one.
350
+ */
351
+ let lowDiskReported = false;
301
352
  /** Earliest server-approved time for the next polling iteration. */
302
353
  let backoffUntil = Number.NEGATIVE_INFINITY;
354
+ /**
355
+ * Consecutive hub polls that failed for a reason the hub did not sanction.
356
+ *
357
+ * RATE LIMITING IS EXCLUDED, and that exclusion is the whole design. A 429 is
358
+ * the server saying "come back later", and a Cloudflare 1015 ban is the same
359
+ * answer at the edge: both are recoverable degradation that clears on its own,
360
+ * and a shift that killed itself over one would take the whole fleet down for
361
+ * a condition the fleet itself caused. `noteServerBackoff` already returns
362
+ * exactly that discrimination, so the streak counts only the failures nobody
363
+ * promised would end.
364
+ *
365
+ * Reset by any SUCCESSFUL wake, which is the loop's one unconditional hub call
366
+ * per unbackoffed tick and therefore the cheapest honest proof that the hub is
367
+ * reachable and this host can reach it.
368
+ */
369
+ let hubFailureStreak = 0;
370
+ /** Set once the shift has stopped for a named local fault, so `run` exits non-zero. */
371
+ let wedged = false;
303
372
  /** Bumped whenever a ping starts or is forced due, so a ping that completes
304
373
  * after a force does not stamp the cadence over that force. */
305
374
  let presenceGeneration = 0;
@@ -377,6 +446,58 @@ export function createShiftLoop(opts) {
377
446
  backoffUntil = Math.max(backoffUntil, monotonicNow() + delay);
378
447
  return true;
379
448
  }
449
+ /**
450
+ * Ask whether a LOCAL fault is the reason this shift cannot work, and stop if
451
+ * one is. Returns true when the shift has been wedged.
452
+ *
453
+ * STOPPING IS THE POINT, and it is the opposite of what every other failure
454
+ * path here does. Everything else in this loop is built to survive: a failed
455
+ * wake retries next tick, a failed roster sync degrades and continues, a
456
+ * rate-limited poll backs off. All of that is correct for a REMOTE fault,
457
+ * which ends without anyone doing anything. It is exactly wrong for a local
458
+ * one, where continuing means an operator watching a live process that will
459
+ * never dispatch again — the measured shape of this defect was a shift that
460
+ * looked alive to `pgrep`, answered `shift end`, and did nothing for 42
461
+ * minutes while attributing a vanished temp directory to `fetch failed`.
462
+ *
463
+ * A PRE-FLIGHT THAT ITSELF FAILS IS NOT A FAULT. If the probe throws, this
464
+ * reports it and returns false, because "I could not determine whether the
465
+ * host is broken" is not evidence that it is, and killing a working shift over
466
+ * an unreadable `stat` would be a worse failure than the one being detected.
467
+ */
468
+ /**
469
+ * Free space for the volume the shift's children will write into.
470
+ *
471
+ * Defaults to `unknown` — never a refusal — when no probe was injected and no
472
+ * work root was resolved: the floor and the work root are both RUNTIME
473
+ * configuration, and a loop constructed without them has no basis to decline
474
+ * anything. `runtime.ts` supplies the real closure.
475
+ */
476
+ function diskSpace() {
477
+ if (opts.diskSpace !== undefined)
478
+ return opts.diskSpace();
479
+ return { state: 'unknown', path: opts.workRoot ?? '' };
480
+ }
481
+ function runHostPreflight(streak) {
482
+ let faults;
483
+ try {
484
+ faults = opts.hostPreflight !== undefined
485
+ ? opts.hostPreflight()
486
+ : checkHost({});
487
+ }
488
+ catch (e) {
489
+ opts.err(`host pre-flight failed to run: ${errMsg(e)} (continuing)`);
490
+ return false;
491
+ }
492
+ if (faults.length === 0)
493
+ return false;
494
+ for (const fault of faults)
495
+ opts.err(`shift cannot continue: ${fault.message}`);
496
+ wedged = true;
497
+ stopped = true;
498
+ emit({ type: 'wedged', faults, streak });
499
+ return true;
500
+ }
380
501
  /**
381
502
  * Fan one observation to every consumer, stamped.
382
503
  *
@@ -1322,11 +1443,24 @@ export function createShiftLoop(opts) {
1322
1443
  changed = w.changed;
1323
1444
  cursor = w.cursor;
1324
1445
  wakeSucceeded = true;
1446
+ // The loop's one unconditional hub call per unbackoffed tick, so its
1447
+ // success is the cheapest honest proof that this host can reach the hub.
1448
+ hubFailureStreak = 0;
1325
1449
  }
1326
1450
  catch (e) {
1327
- noteServerBackoff(e);
1451
+ const rateLimited = noteServerBackoff(e);
1328
1452
  opts.err(`wake failed: ${errMsg(e)} (retrying next tick)`);
1329
1453
  emit({ type: 'hub-error', op: 'wake', message: errMsg(e) });
1454
+ if (!rateLimited) {
1455
+ hubFailureStreak += 1;
1456
+ // Every HOST_PREFLIGHT_STREAK-th failure, not only the first crossing:
1457
+ // a work root can be unmounted at failure seven, and a shift that
1458
+ // checked once at failure three would never see it. The check is
1459
+ // `stat`-only and adds no hub traffic, so repeating it is nearly free.
1460
+ if (hubFailureStreak % HOST_PREFLIGHT_STREAK === 0 && runHostPreflight(hubFailureStreak)) {
1461
+ return dispatched;
1462
+ }
1463
+ }
1330
1464
  }
1331
1465
  // A changed cursor is not sufficient by itself: if the prior changed tick
1332
1466
  // had no local capacity, or its sweep failed, the adopted cursor will be
@@ -1339,7 +1473,38 @@ export function createShiftLoop(opts) {
1339
1473
  // can actually make the targeted decision.
1340
1474
  const capacityCooldownDue = capacityCooldownDueAt !== undefined && monotonicNow() >= capacityCooldownDueAt;
1341
1475
  const capacityBecameReady = capacityCooldownReady(k, agentRoom);
1342
- if (wakeSucceeded && (changed || sweepOwed || brakeDue || capacityCooldownDue || capacityBecameReady) && k > 0) {
1476
+ const wantSweep = wakeSucceeded && (changed || sweepOwed || brakeDue || capacityCooldownDue || capacityBecameReady) && k > 0;
1477
+ /*
1478
+ * Measured ONLY when a sweep would otherwise happen, so a parked or
1479
+ * saturated shift pays no syscall for a question it is not about to act on.
1480
+ * That placement is also what makes recovery automatic: refusing sets
1481
+ * `sweepOwed`, `sweepOwed` makes the next tick want to sweep, and wanting to
1482
+ * sweep re-measures the disk. No timer and no separate recovery path.
1483
+ */
1484
+ const disk = wantSweep ? diskSpace() : undefined;
1485
+ if (wantSweep && disk?.state === 'low') {
1486
+ /*
1487
+ * REFUSE NEW WORK, DO NOT STOP. Unlike the vanished temp directory that
1488
+ * `runHostPreflight` exits for, a full disk clears on its own, so the
1489
+ * shift stays up and keeps polling. In-flight children are deliberately
1490
+ * left running: killing one frees little and destroys all of its work.
1491
+ */
1492
+ sweepOwed = true;
1493
+ opts.out(`low disk: ${formatBytes(disk.freeBytes)} free at ${disk.path}, below the ${formatBytes(disk.floorBytes)} floor — ` +
1494
+ `deferring whats_next until space is free`);
1495
+ // EDGE-TRIGGERED, exactly as the at-capacity record below: one per
1496
+ // low-disk EPISODE. The console line above stays level-triggered because
1497
+ // a live tail is watched by someone who wants to see it is still stuck.
1498
+ if (!lowDiskReported) {
1499
+ lowDiskReported = true;
1500
+ emit({ type: 'low-disk', path: disk.path, freeBytes: disk.freeBytes, floorBytes: disk.floorBytes });
1501
+ }
1502
+ }
1503
+ else if (wantSweep) {
1504
+ if (lowDiskReported) {
1505
+ lowDiskReported = false;
1506
+ opts.out('disk space recovered — resuming dispatch');
1507
+ }
1343
1508
  // Clear before awaiting: a new brake armed during sweep must survive.
1344
1509
  if (brakeDue)
1345
1510
  brakeSweepDueAt = undefined;
@@ -1394,21 +1559,28 @@ export function createShiftLoop(opts) {
1394
1559
  }
1395
1560
  async function run() {
1396
1561
  ensureStateDir(opts.stateDir);
1562
+ // BEFORE THE FIRST HUB CALL. A shift started into a broken host — the usual
1563
+ // way being a supervisor that restarts it across a reboot with the old
1564
+ // environment — must say so at once. Making it wait for three failed polls
1565
+ // would buy nothing and spend the operator's attention on network theories.
1566
+ // `streak: 0` marks this record as the boot check rather than a poll one.
1567
+ if (runHostPreflight(0))
1568
+ return 1;
1397
1569
  if (opts.once === true) {
1398
1570
  await iteration();
1399
- return 0;
1571
+ return wedged ? 1 : 0;
1400
1572
  }
1401
1573
  for (;;) {
1402
1574
  if (stopped)
1403
- return 0;
1575
+ return wedged ? 1 : 0;
1404
1576
  await iteration();
1405
1577
  if (stopped)
1406
- return 0;
1578
+ return wedged ? 1 : 0;
1407
1579
  const delay = Math.max(opts.pollIntervalMs, backoffUntil - monotonicNow());
1408
1580
  await opts.sleep(delay);
1409
1581
  // Re-check after the park so a stop during sleep makes no further hub call.
1410
1582
  if (stopped)
1411
- return 0;
1583
+ return wedged ? 1 : 0;
1412
1584
  }
1413
1585
  }
1414
1586
  function stop() {
@@ -6,6 +6,7 @@
6
6
  * workflow state; this protocol only carries local dispatch observations and
7
7
  * attendance commands between the CLI and one daemon process.
8
8
  */
9
+ import type { HostFault } from './host-preflight.ts';
9
10
  export declare const SHIFT_SOCKET_NAME = "shift.sock";
10
11
  export declare const DEFAULT_NEXT_WAIT_MS = 90000;
11
12
  export declare const MAX_REQUEST_LINE_BYTES: number;
@@ -119,6 +120,30 @@ export interface CapacityEvent {
119
120
  inFlight: number;
120
121
  cap: number;
121
122
  }
123
+ /**
124
+ * The shift declined to start new work because the volume holding its work root
125
+ * is nearly full.
126
+ *
127
+ * EDGE-TRIGGERED, like `CapacityEvent`: one record per low-disk EPISODE, not one
128
+ * per tick. A shift that sits on a full disk for an hour writes this once, and
129
+ * the recovery is legible from the timestamp of whatever it dispatches next.
130
+ *
131
+ * FILE-ONLY, for the same reason as `CapacityEvent` and `ParkedEvent`: a parked
132
+ * `shift next` must not be woken by news that nothing happened. In the file it
133
+ * is the only thing separating "this shift had no work" from "this shift had
134
+ * work and could not safely take it".
135
+ *
136
+ * `freeBytes` and `floorBytes` are both carried so the record is self-contained.
137
+ * A reader must be able to tell an operator who set an unreachable floor from a
138
+ * genuinely full disk without knowing what the shift was configured with.
139
+ */
140
+ export interface LowDiskEvent {
141
+ type: 'low-disk';
142
+ /** The measured directory: the work root, or its nearest existing ancestor. */
143
+ path: string;
144
+ freeBytes: number;
145
+ floorBytes: number;
146
+ }
122
147
  /** A hub call failed. `workflow` is present for per-workflow whats_next and release calls. */
123
148
  export interface HubErrorEvent {
124
149
  type: 'hub-error';
@@ -146,6 +171,42 @@ export interface OrderDroppedEvent {
146
171
  reason: 'malformed-digest' | 'malformed-worker' | 'unsupported-worker' | 'verification-failed' | 'metadata-unavailable' | 'agent-lane-closed' | 'dispatch-cap-full' | 'agent-cap-full' | 'claim-expired';
147
172
  message: string;
148
173
  }
174
+ /**
175
+ * The shift STOPPED because the machine underneath it is broken, and named the
176
+ * fault. The last record a wedged shift writes; the process exits non-zero
177
+ * immediately after.
178
+ *
179
+ * WHY THIS EXISTS AS ITS OWN RECORD, rather than one more `hub-error`. A shift
180
+ * whose host temp directory vanished — which a macOS reboot does to any process
181
+ * that survives it — keeps polling forever, and every hub call fails with
182
+ * undici's generic `fetch failed`. From the outside that is indistinguishable
183
+ * from a hub outage: `pgrep` shows the process, `shift end` answers, and the
184
+ * log fills with records that attribute a LOCAL fault to the network. This
185
+ * record is the one that says the cause is here, on this machine, and that no
186
+ * amount of waiting will fix it.
187
+ *
188
+ * `faults` is never empty. That invariant is what separates this record from
189
+ * every retryable failure: a hub outage, a rate-limit ban and an expired
190
+ * credential all self-heal, so the shift keeps working through them and emits
191
+ * nothing here. A record exists only when a human must change something on the
192
+ * host.
193
+ *
194
+ * `streak` is how many consecutive hub calls failed before the check ran, and
195
+ * is 0 for the pre-flight the shift runs at start, before any hub call. It is
196
+ * evidence, not a cause: the streak is only what PROMPTED the check, and the
197
+ * `faults` are why the shift stopped.
198
+ *
199
+ * ON THE SOCKET AS WELL AS IN THE FILE, unlike `parked` and `capacity`. A
200
+ * parked `owenloop shift next` must block until there is something to report,
201
+ * and "the shift you are waiting on is dead" is the most report-worthy thing
202
+ * that can happen to it — a client that stays parked on a socket whose server
203
+ * is about to close waits for work that will never come.
204
+ */
205
+ export interface WedgedEvent {
206
+ type: 'wedged';
207
+ faults: HostFault[];
208
+ streak: number;
209
+ }
149
210
  /**
150
211
  * The socket event FIFO overflowed and discarded its oldest event. Written
151
212
  * STRAIGHT to the file sink, never through the loop's `emit()` — `emit()` feeds
@@ -161,7 +222,7 @@ export interface EventQueueOverflowEvent {
161
222
  * What a construction site builds: the event's own payload, with no identity
162
223
  * and no timestamp. Every consumer receives the stamped `ShiftEvent` instead.
163
224
  */
164
- export type ShiftEventBody = DispatchedEvent | ReapedEvent | FailedEvent | GateEvent | EndedEvent | ParkedEvent | CapacityEvent | HubErrorEvent | BundleMissEvent | OrderDroppedEvent | EventQueueOverflowEvent;
225
+ export type ShiftEventBody = DispatchedEvent | ReapedEvent | FailedEvent | GateEvent | EndedEvent | ParkedEvent | CapacityEvent | HubErrorEvent | BundleMissEvent | OrderDroppedEvent | EventQueueOverflowEvent | WedgedEvent | LowDiskEvent;
165
226
  /**
166
227
  * The identity every event carries, added once on a shared envelope rather than
167
228
  * per variant.
@@ -35,6 +35,8 @@ export interface ParsedArgs {
35
35
  once?: boolean;
36
36
  maxAgents?: number;
37
37
  execReserve?: number;
38
+ /** `--disk-floor <bytes>` — free space required to start new work. `0` disables. */
39
+ diskFloorBytes?: number;
38
40
  localQueueHoldMs?: number;
39
41
  cacheDir?: string;
40
42
  stateDir?: string;
@@ -16,6 +16,7 @@ import { loadSettings } from "../settings/settings.js";
16
16
  import { DEFAULT_HUB_ROSTER_SYNC_TIMEOUT_MS, syncHubRosterCache, withHubRosterSyncTimeout } from "../settings/hub-roster-cache.js";
17
17
  import { computeServeCapabilities } from "../settings/serving.js";
18
18
  import { resolveCacheDir } from "../bundle/cache.js";
19
+ import { checkDiskFloor, resolveDiskFloorBytes } from "./disk-floor.js";
19
20
  import { createLockedRemovalCallbacks, createShiftLoop } from "./loop.js";
20
21
  import { createHubBundleRecoveryHandler } from "../bundle/pull.js";
21
22
  import { createShiftLogSink } from "./logsink.js";
@@ -93,6 +94,7 @@ const FILE_ONLY_EVENTS = new Set([
93
94
  'capacity',
94
95
  'hub-error',
95
96
  'event-queue-overflow',
97
+ 'low-disk',
96
98
  ]);
97
99
  /**
98
100
  * Does this event type reach the SOCKET consumer (a parked `owenloop shift
@@ -161,6 +163,7 @@ export function parseArgs(args) {
161
163
  case '--cap':
162
164
  case '--max-agents':
163
165
  case '--exec-reserve':
166
+ case '--disk-floor':
164
167
  case '--local-queue-hold':
165
168
  case '--workflow':
166
169
  case '--poll-interval':
@@ -217,6 +220,12 @@ export function parseArgs(args) {
217
220
  return { error: n.error };
218
221
  parsed.execReserve = n;
219
222
  }
223
+ else if (name === '--disk-floor') {
224
+ const n = intFlag(r.value, '--disk-floor');
225
+ if (typeof n !== 'number')
226
+ return { error: n.error };
227
+ parsed.diskFloorBytes = n;
228
+ }
220
229
  else if (name === '--local-queue-hold') {
221
230
  const n = intFlag(r.value, '--local-queue-hold');
222
231
  if (typeof n !== 'number')
@@ -404,6 +413,7 @@ export async function runShiftRuntime(parsed, options = {}) {
404
413
  const maxConcurrentAgents = resolveMaxConcurrentAgents(parsed.maxAgents, settings.maxConcurrentAgents);
405
414
  const execReserve = resolveExecReserve(parsed.execReserve, settings.execReserve);
406
415
  const localQueueHoldMs = resolveLocalQueueHoldMs(parsed.localQueueHoldMs, settings.localQueueHoldMs);
416
+ const diskFloorBytes = resolveDiskFloorBytes(parsed.diskFloorBytes, settings.diskFloorBytes);
407
417
  const workRoot = resolveWorkRoot(env, settings.workRoot, cacheDir);
408
418
  const workRepo = resolveWorkRepo(env, settings.workRepo);
409
419
  /**
@@ -588,6 +598,13 @@ export async function runShiftRuntime(parsed, options = {}) {
588
598
  execReserve,
589
599
  localQueueHoldMs,
590
600
  workRoot,
601
+ /*
602
+ * Resolved here, not in the loop: `runtime.ts` owns flag/settings/default
603
+ * precedence for every other tunable, and the loop consumes a decision. A
604
+ * loop built without this closure has no disk gate at all, which is the
605
+ * right default for a caller that never told it where work lands.
606
+ */
607
+ diskSpace: () => checkDiskFloor(workRoot, diskFloorBytes),
591
608
  ...(workRepo !== undefined ? { workRepo } : {}),
592
609
  shiftId,
593
610
  startedAt,
@@ -1,7 +1,8 @@
1
1
  /**
2
2
  * Read a submit value from a JSON document contained in a run worktree.
3
3
  *
4
- * The lexical check rejects traversal before probing the filesystem. The
5
- * canonical check rejects symlinks whose target leaves the worktree.
4
+ * Containment is `resolveContainedPath`'s job — lexical traversal check first,
5
+ * then a canonical symlink check. This function owns only what is specific to
6
+ * a submit value: the argument shape and the JSON decode.
6
7
  */
7
8
  export declare function readSubmitValueFile(workdir: string, valueFile: unknown): Promise<unknown>;
@@ -1,36 +1,20 @@
1
- import { readFile, realpath } from 'node:fs/promises';
2
- import { resolve } from 'node:path';
3
- import { isInside } from "./harness/gatekeeper.js";
1
+ import { readFile } from 'node:fs/promises';
2
+ import { resolveContainedPath } from "./contained-path.js";
4
3
  function errorMessage(error) {
5
4
  return error instanceof Error ? error.message : String(error);
6
5
  }
7
6
  /**
8
7
  * Read a submit value from a JSON document contained in a run worktree.
9
8
  *
10
- * The lexical check rejects traversal before probing the filesystem. The
11
- * canonical check rejects symlinks whose target leaves the worktree.
9
+ * Containment is `resolveContainedPath`'s job — lexical traversal check first,
10
+ * then a canonical symlink check. This function owns only what is specific to
11
+ * a submit value: the argument shape and the JSON decode.
12
12
  */
13
13
  export async function readSubmitValueFile(workdir, valueFile) {
14
14
  if (typeof valueFile !== 'string' || valueFile.trim() === '') {
15
15
  throw new Error(`submit-value-file-invalid: valueFile must be a non-empty string: ${String(valueFile)}`);
16
16
  }
17
- const candidate = resolve(workdir, valueFile);
18
- if (!isInside(workdir, candidate)) {
19
- throw new Error(`submit-value-file-outside-workdir: ${valueFile} is outside the run workdir`);
20
- }
21
- let canonicalCandidate;
22
- try {
23
- const canonicalWorkdir = await realpath(workdir);
24
- canonicalCandidate = await realpath(candidate);
25
- if (!isInside(canonicalWorkdir, canonicalCandidate)) {
26
- throw new Error(`submit-value-file-outside-workdir: ${valueFile} is outside the run workdir`);
27
- }
28
- }
29
- catch (error) {
30
- if (error instanceof Error && error.message.startsWith('submit-value-file-outside-workdir:'))
31
- throw error;
32
- throw new Error(`submit-value-file-read-failed: could not read ${valueFile}: ${errorMessage(error)}`);
33
- }
17
+ const canonicalCandidate = await resolveContainedPath(workdir, valueFile, 'submit-value-file');
34
18
  let text;
35
19
  try {
36
20
  text = await readFile(canonicalCandidate, 'utf8');
package/dist/src/defs.js CHANGED
@@ -39,7 +39,7 @@ const RAW_PRODUCE_KEYS = ['name', 'schema', 'judges', 'maxAttempts', 'maxSchemaF
39
39
  * fields, so it gets its own allowlist rather than folding into
40
40
  * RAW_PRODUCE_KEYS (§27 unknown-key rejection). */
41
41
  const RAW_GROUP_KEYS = ['group', 'mode', 'of'];
42
- const RAW_JUDGE_KEYS = ['name', 'body', 'bodyFile', 'model', 'inputs', 'cadence', 'maxRunsPerDay', 'executor', 'command', 'spec', 'capabilities', 'modifiers'];
42
+ const RAW_JUDGE_KEYS = ['name', 'body', 'bodyFile', 'model', 'inputs', 'cadence', 'maxRunsPerDay', 'executor', 'command', 'spec', 'workdir', 'workdirFrom', 'capabilities', 'modifiers'];
43
43
  /** Keys valid on a normal (non-calls:, non-include:) step entry. */
44
44
  const RAW_STEP_KEYS = [
45
45
  'name', 'consumes', 'produces', 'generates', 'invalidates', 'cadence',
@@ -362,6 +362,14 @@ function parseJudges(v, ctx, baseDir) {
362
362
  judge.command = asString(raw.command, `judge '${name}'.command`);
363
363
  if (raw.spec !== undefined)
364
364
  judge.spec = asExtension(raw.spec, `judge '${name}'.spec`);
365
+ if (raw.workdir !== undefined)
366
+ judge.workdir = asString(raw.workdir, `judge '${name}'.workdir`);
367
+ if (raw.workdirFrom !== undefined) {
368
+ judge.workdirFrom = asString(raw.workdirFrom, `judge '${name}'.workdirFrom`);
369
+ }
370
+ if (raw.workdir !== undefined && raw.workdirFrom !== undefined) {
371
+ throw new DefError(`judge '${name}' may not declare both workdir and workdirFrom`);
372
+ }
365
373
  if (raw.capabilities !== undefined) {
366
374
  const caps = asStringArray(raw.capabilities, `judge '${name}'.capabilities`);
367
375
  if (caps.length === 0) {
@@ -997,24 +1005,63 @@ export function parseDef(raw, source, baseDir) {
997
1005
  }
998
1006
  return def;
999
1007
  }
1008
+ /**
1009
+ * The stem half of a `workdirFrom` expression, resolved against a list of
1010
+ * consumed stems. Mirrors `parseWorkdirFrom`'s boundary walk (longest stem
1011
+ * first, because a stem may itself contain dots) but takes bare names, since a
1012
+ * synthesized judge has not built its consume patterns yet. Returns null when
1013
+ * no consumed stem matches — the expression then names a definition input,
1014
+ * which validateDef resolves without a consume edge.
1015
+ */
1016
+ function workdirFromStem(raw, stems) {
1017
+ const r = raw.trim();
1018
+ for (let boundary = r.lastIndexOf('.'); boundary > 0; boundary = r.lastIndexOf('.', boundary - 1)) {
1019
+ if (r.length - boundary - 1 === 0)
1020
+ continue;
1021
+ const stem = r.slice(0, boundary);
1022
+ if (stems.includes(stem))
1023
+ return stem;
1024
+ }
1025
+ return null;
1026
+ }
1000
1027
  /**
1001
1028
  * Synthesize one full StepDef per declared `judges:` entry on a produce
1002
1029
  * pattern (§24 §3.2, §7.2). Shape mirrors the `calls:` template above, with
1003
1030
  * exactly three deltas from a hand-written step: the `judges: <stem>` marker
1004
1031
  * (eligibility trigger, replacing inputsGreen — see model.ts), `produces: []`
1005
1032
  * (a judge emits a verdict against the judged stem, not a new artifact), and
1006
- * `consumes: [stem, ...(inputs ? producerConsumeStems : [])]` so authority
1007
- * flows from the existing consume-edge check (`assertAuthority`) with no
1008
- * special-casing. Everything else (cadence, maxRunsPerDay, model, body,
1009
- * maxAttempts/maxSchemaFailures defaults) is inherited exactly like
1010
- * an ordinary step, because judge orders flow through the normal
1033
+ * `consumes: [stem, ...(inputs ? producerConsumeStems : []), ...workdirFromStem]`
1034
+ * so authority flows from the existing consume-edge check (`assertAuthority`)
1035
+ * with no special-casing. Everything else (cadence, maxRunsPerDay, model, body,
1036
+ * workdir/workdirFrom, maxAttempts/maxSchemaFailures defaults) is inherited
1037
+ * exactly like an ordinary step, because judge orders flow through the normal
1011
1038
  * eligibleFirings → applySchedule → claim → buildOrder pipeline (§7.1).
1012
1039
  */
1013
- function synthesizeJudgeSteps(producerStepName, pat, producerConsumeStems, producerX, producerCapabilities) {
1040
+ function synthesizeJudgeSteps(producerStepName, pat, producerConsumeStems, producerX, producerCapabilities, producerWorkdir, producerWorkdirFrom) {
1014
1041
  if (!pat.judges || pat.judges.length === 0)
1015
1042
  return [];
1016
1043
  return pat.judges.map((j) => {
1044
+ // A judge with no workdir of its own INHERITS the producer's, so it reads
1045
+ // the same tree the work it judges was produced in. Declaring either key
1046
+ // opts out of both halves of the inheritance: producer and judge each carry
1047
+ // at most one, and silently pairing an authored `workdir` with an inherited
1048
+ // `workdirFrom` would trip validateDef's mutual-exclusion check on a
1049
+ // definition the author never wrote that way.
1050
+ const authoredWorkdir = j.workdir !== undefined || j.workdirFrom !== undefined;
1051
+ const workdir = authoredWorkdir ? j.workdir : producerWorkdir;
1052
+ const workdirFrom = authoredWorkdir ? j.workdirFrom : producerWorkdirFrom;
1017
1053
  const consumeStems = j.inputs ? [pat.stem, ...producerConsumeStems] : [pat.stem];
1054
+ // A workdirFrom value is resolved from a CONSUMED artifact, so the stem it
1055
+ // names must be on this step's consume list or validateDef rejects the whole
1056
+ // definition. `inputs: false` is the case that needs this: the judge sees
1057
+ // only the judged stem, which is never the stem the producer resolved its
1058
+ // workdir from. Deduplicated, so `inputs: true` (where the stem is already
1059
+ // carried) stays byte-identical to the pre-inheritance shape.
1060
+ if (workdirFrom !== undefined) {
1061
+ const stem = workdirFromStem(workdirFrom, producerConsumeStems);
1062
+ if (stem !== null && !consumeStems.includes(stem))
1063
+ consumeStems.push(stem);
1064
+ }
1018
1065
  const consumes = consumeStems.map((stem) => parseConsume(stem));
1019
1066
  const cadence = j.cadence ?? DEFAULTS.cadence;
1020
1067
  const step = {
@@ -1055,6 +1102,10 @@ function synthesizeJudgeSteps(producerStepName, pat, producerConsumeStems, produ
1055
1102
  step.command = j.command;
1056
1103
  if (j.spec !== undefined)
1057
1104
  step.spec = j.spec;
1105
+ if (workdir !== undefined)
1106
+ step.workdir = workdir;
1107
+ if (workdirFrom !== undefined)
1108
+ step.workdirFrom = workdirFrom;
1058
1109
  if (j.modifiers !== undefined)
1059
1110
  step.judgeModifiers = [...j.modifiers];
1060
1111
  return step;
@@ -1243,7 +1294,7 @@ function buildStep(rl, i, baseDir) {
1243
1294
  }
1244
1295
  }
1245
1296
  const producerConsumeStems = consumes.map((c) => c.stem);
1246
- const judgeSteps = producesPatterns.flatMap((p) => synthesizeJudgeSteps(name, p, producerConsumeStems, step.x, step.capabilities));
1297
+ const judgeSteps = producesPatterns.flatMap((p) => synthesizeJudgeSteps(name, p, producerConsumeStems, step.x, step.capabilities, step.workdir, step.workdirFrom));
1247
1298
  return [step, ...judgeSteps];
1248
1299
  }
1249
1300
  // ---- validation --------------------------------------------------------------
@@ -485,6 +485,19 @@ export interface ProducePattern {
485
485
  * (or additionally alongside `command`). Shape-checked as a plain map
486
486
  * only (mirrors `x:`'s asExtension contract) — contents never read. */
487
487
  spec?: Record<string, unknown>;
488
+ /** opaque location hint passed through on the judge's synthesized order.
489
+ * Omitted = INHERIT the producing step's `workdir`/`workdirFrom`, so a
490
+ * judge reads the same tree the work it judges was produced in. Mutually
491
+ * exclusive with workdirFrom, same as StepDef. */
492
+ workdir?: string;
493
+ /**
494
+ * Resolve the judge order's workdir from `<consumedStem>.<dotted.path>` in a
495
+ * consumed artifact value. Mutually exclusive with workdir. Omitted =
496
+ * INHERIT the producer's. The named stem is added to the synthesized step's
497
+ * consumes when it is not already there, so the value passes the engine's
498
+ * consume-side verification gate.
499
+ */
500
+ workdirFrom?: string;
488
501
  /** Routing capabilities for this judge's synthesized step. Omitted =
489
502
  * INHERIT the producing step's `capabilities`, so a judge routes to the
490
503
  * same grade of crew as the work it judges instead of being claimable by
package/docs/cli.md CHANGED
@@ -305,6 +305,7 @@ holds `credentials.json`, `allowed_signers`, `org-root.pub`, `grants/`, and
305
305
  ```text
306
306
  owenloop shift start <crew...> [--all] [--origin <url>] [--as <account>] [--name <n>]
307
307
  [--cap <n>] [--max-agents <n>] [--exec-reserve <n>] [--local-queue-hold <ms>] [--poll-interval <ms>] [--once]
308
+ [--disk-floor <bytes>]
308
309
  [--cache-dir <p>] [--state-dir <p>] [--log-dir <p>] [--log-max-age <ms>]
309
310
  [--work-root <dir>]...
310
311
  ```
@@ -323,6 +324,7 @@ one loop sweep and exits instead of keeping the foreground daemon running.
323
324
  | `--cap <n>` | dispatch capacity; precedence is flag, then `settings.dispatchCap`, then `3` |
324
325
  | `--max-agents <n>` | concurrent agent limit; precedence is flag, then `settings.maxConcurrentAgents`, then `4` |
325
326
  | `--exec-reserve <n>` | slots inside `--cap` that `agent-run` children may never occupy, so an exec/command order always has room; precedence is flag, then `settings.execReserve`, then `1`. Clamped to `cap - 1`, so a `--cap 1` shift gets no reserve. `0` disables the reserve and lets agents fill the whole cap. This does not raise the total child ceiling — `--cap` still bounds every child. |
327
+ | `--disk-floor <bytes>` | free space the shift requires on the volume holding its work root before it will START another order; precedence is flag, then `settings.diskFloorBytes`, then `1073741824` (1 GiB). A shift below the floor keeps polling and declines new work, writing one `low-disk` record per episode, and resumes by itself when space returns; children already running are left alone. `0` disables the check. This is a floor for starting work, not a promise that a particular step will fit. |
326
328
  | `--local-queue-hold <ms>` | how long to retain a claim this shift cannot dispatch before returning it to the hub; precedence is flag, then `settings.localQueueHoldMs`, then `0`. The value is clamped to 90 seconds. `0` means the shift never holds an undispatchable claim locally, so another daemon can be offered it immediately. |
327
329
  | `--poll-interval <ms>` | loop polling interval; defaults to `5000` milliseconds |
328
330
  | `--once` | run one loop sweep and exit; without it, keep the daemon in the foreground |
@@ -587,16 +589,17 @@ either `exec` or `agent-run`:
587
589
  - `bundle-miss`: `{ "type": "bundle-miss", "workflow": "...", "def": "..." }` — a legacy order named a def with no cached bundle, so it was left for hub pickup
588
590
  - `order-dropped`: `{ "type": "order-dropped", "workflow": "...", "run": "...", "step": "...", "reason": "unsupported-worker", "message": "..." }` — the shift refused one order. Match on `reason` (`malformed-digest`, `malformed-worker`, `unsupported-worker`, `verification-failed`, `metadata-unavailable`, `agent-lane-closed`, `dispatch-cap-full`, `agent-cap-full`, `claim-expired`); display `message`. The capacity, expiry, and `agent-lane-closed` reasons return the claim to the hub; malformed and unsupported reasons leave it for the pickup window.
589
591
  - `ended`: `{ "type": "ended" }`, delivered to a parked `next` when `shift end` explicitly shuts down the daemon
592
+ - `wedged`: `{ "type": "wedged", "faults": [{ "kind": "missing", "role": "temp-dir", "path": "...", "message": "..." }], "streak": 3 }` — the shift stopped because the MACHINE it runs on is broken, and the process exits non-zero straight after. `faults` is never empty; display each `message`, which names the path and the operator fix. `streak` is how many consecutive hub calls failed before the check ran, and is `0` for the check a shift makes at start. A shift that cannot reach the hub does NOT produce this — a hub outage or a rate-limit ban recovers on its own, and the shift keeps polling through it. This record means a human must change something on the host.
590
593
 
591
594
  Every event above is also appended to `<log-dir>/shift.log` as JSON Lines, which
592
- is how they survive the daemon. **Four further record types exist in
595
+ is how they survive the daemon. **Five further record types exist in
593
596
  `shift.log` and are never delivered over the socket** — `parked`, `capacity`,
594
- `hub-error`, and `event-queue-overflow`.
597
+ `hub-error`, `event-queue-overflow`, and `low-disk`.
595
598
 
596
599
  An idle `shift next` must block until there is work to report, and each of those
597
- four would satisfy it with news that no work moved: a startup record, a report
598
- that the shift is full, a failed call to the hub, or a record about the socket
599
- queue itself. `parked` and `capacity` are also redundant on the wire — the
600
+ five would satisfy it with news that no work moved: a startup record, a report
601
+ that the shift is full, a failed call to the hub, a record about the socket
602
+ queue itself, or a report that the disk is too full to start anything. `parked` and `capacity` are also redundant on the wire — the
600
603
  response above already carries live `cap`, `free`, and `running`, which is
601
604
  exactly what a `capacity` record restates. `hub-error` is level-triggered at the
602
605
  poll interval, so an unreachable hub would otherwise fill the 1000-slot socket