owenloop 0.5.26 → 0.5.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/packages/work/src/agent/brief.d.ts +10 -0
- package/dist/packages/work/src/agent/brief.js +38 -0
- package/dist/packages/work/src/agent/loop.js +4 -0
- package/dist/packages/work/src/contained-path.d.ts +18 -0
- package/dist/packages/work/src/contained-path.js +42 -0
- package/dist/packages/work/src/harness/claude.js +21 -0
- package/dist/packages/work/src/harness/codex.js +29 -0
- package/dist/packages/work/src/harness/contract.d.ts +26 -0
- package/dist/packages/work/src/hold/mcp.d.ts +3 -40
- package/dist/packages/work/src/hold/mcp.js +121 -2
- package/dist/packages/work/src/hub/client.d.ts +10 -1
- package/dist/packages/work/src/hub/client.js +23 -0
- package/dist/packages/work/src/hub/types.d.ts +31 -0
- package/dist/packages/work/src/roles/shift.js +7 -1
- package/dist/packages/work/src/settings/settings.d.ts +12 -1
- package/dist/packages/work/src/settings/settings.js +7 -0
- package/dist/packages/work/src/shift/disk-floor.d.ts +107 -0
- package/dist/packages/work/src/shift/disk-floor.js +148 -0
- package/dist/packages/work/src/shift/host-preflight.d.ts +85 -0
- package/dist/packages/work/src/shift/host-preflight.js +112 -0
- package/dist/packages/work/src/shift/loop.d.ts +18 -0
- package/dist/packages/work/src/shift/loop.js +179 -7
- package/dist/packages/work/src/shift/protocol.d.ts +62 -1
- package/dist/packages/work/src/shift/runtime.d.ts +2 -0
- package/dist/packages/work/src/shift/runtime.js +17 -0
- package/dist/packages/work/src/submit-file.d.ts +3 -2
- package/dist/packages/work/src/submit-file.js +6 -22
- package/dist/src/defs.js +59 -8
- package/dist/src/types.d.ts +13 -0
- package/docs/cli.md +8 -5
- package/docs/shift-logs.md +64 -7
- package/package.json +1 -1
- package/plugins/claude-code/plugin/.claude-plugin/plugin.json +1 -1
- package/plugins/claude-code/plugin/.mcp.json +1 -1
- package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +1 -1
- package/plugins/codex/plugins/owenloop/.mcp.json +1 -1
|
@@ -63,6 +63,8 @@ import { cancelReservedChild, finalizeChildReservation, reconcileInFlight, reser
|
|
|
63
63
|
import { DEFAULT_WORK_DIR_TTL_MS, sweepWorkDirs as sweepWorkDirsImpl } from "../agent/workdir.js";
|
|
64
64
|
import { withHubRosterSyncTimeout } from "../settings/hub-roster-cache.js";
|
|
65
65
|
import { sessionsPath } from "../harness/session-store.js";
|
|
66
|
+
import { checkHost } from "./host-preflight.js";
|
|
67
|
+
import { formatBytes } from "./disk-floor.js";
|
|
66
68
|
import { stampShiftEvent, } from "./protocol.js";
|
|
67
69
|
/** Run a state mutation while holding the lock that serializes dispatch state. */
|
|
68
70
|
export function withDispatchLock(stateDir, options = {}, fn) {
|
|
@@ -196,12 +198,53 @@ function stepBrakeDelayMs(count) {
|
|
|
196
198
|
const index = Math.min(count, STEP_BRAKE_DELAYS_MS.length) - 1;
|
|
197
199
|
return STEP_BRAKE_DELAYS_MS[index] ?? 0;
|
|
198
200
|
}
|
|
201
|
+
/** How many links of an error's `cause` chain `errMsg` renders. */
|
|
202
|
+
const MAX_CAUSE_DEPTH = 4;
|
|
203
|
+
/**
|
|
204
|
+
* Consecutive failed hub polls before the shift asks whether the fault is local.
|
|
205
|
+
*
|
|
206
|
+
* NOT a timeout and not a health threshold — nothing is declared broken at this
|
|
207
|
+
* number. Crossing it only makes the shift run `checkHost`, which is a handful
|
|
208
|
+
* of `stat` calls and is free of hub traffic; if the host is fine the shift
|
|
209
|
+
* keeps polling and the streak keeps climbing. So the number only has to be
|
|
210
|
+
* large enough that a single transient failure does not pay for the check, and
|
|
211
|
+
* small enough that an operator is not left guessing for long. At the shipped
|
|
212
|
+
* 30s poll interval three failures is about a minute and a half.
|
|
213
|
+
*/
|
|
214
|
+
const HOST_PREFLIGHT_STREAK = 3;
|
|
199
215
|
/** The identity a storm repeats on. `key` is '' for an unfanned step. */
|
|
200
216
|
function stepBrakeKey(workflow, step, key) {
|
|
201
217
|
return `${workflow}\u0000${step}\u0000${key ?? ''}`;
|
|
202
218
|
}
|
|
219
|
+
/**
|
|
220
|
+
* An error's message, with its cause chain appended.
|
|
221
|
+
*
|
|
222
|
+
* THE CHAIN IS THE POINT. Node's fetch reports every transport failure as the
|
|
223
|
+
* bare string `fetch failed` and puts the real reason — `ENOENT`, `ECONNREFUSED`,
|
|
224
|
+
* `EAI_AGAIN`, a TLS error — on `cause`. Dropping it makes a dead local host, an
|
|
225
|
+
* unreachable hub and a DNS failure produce byte-identical records, which is
|
|
226
|
+
* precisely how a wedged shift stayed unattributed for 42 minutes.
|
|
227
|
+
*
|
|
228
|
+
* Bounded by construction: the walk follows at most `MAX_CAUSE_DEPTH` links and
|
|
229
|
+
* refuses to revisit a cause it has already rendered, so a self-referential or
|
|
230
|
+
* mutually-referential chain terminates instead of hanging the emitter.
|
|
231
|
+
*/
|
|
203
232
|
function errMsg(e) {
|
|
204
|
-
|
|
233
|
+
if (!(e instanceof Error))
|
|
234
|
+
return String(e);
|
|
235
|
+
const parts = [e.message];
|
|
236
|
+
const seen = new Set([e]);
|
|
237
|
+
let cause = e.cause;
|
|
238
|
+
for (let depth = 0; depth < MAX_CAUSE_DEPTH && cause !== undefined && cause !== null; depth += 1) {
|
|
239
|
+
if (seen.has(cause))
|
|
240
|
+
break;
|
|
241
|
+
seen.add(cause);
|
|
242
|
+
const text = cause instanceof Error ? cause.message : String(cause);
|
|
243
|
+
if (text !== '' && !parts.includes(text))
|
|
244
|
+
parts.push(text);
|
|
245
|
+
cause = cause instanceof Error ? cause.cause : undefined;
|
|
246
|
+
}
|
|
247
|
+
return parts.join(': ');
|
|
205
248
|
}
|
|
206
249
|
/** The explicit hub refusal produced when an inbox row becomes terminal after
|
|
207
250
|
* the inbox snapshot but before its targeted serving call. This is a normal
|
|
@@ -298,8 +341,34 @@ export function createShiftLoop(opts) {
|
|
|
298
341
|
* deliberately NOT edge-triggered.
|
|
299
342
|
*/
|
|
300
343
|
let atCapReported = false;
|
|
344
|
+
/**
|
|
345
|
+
* EDGE flag for the low-disk refusal, paired with `atCapReported` because the
|
|
346
|
+
* two gates have the same shape: a condition that persists across many ticks
|
|
347
|
+
* but is worth exactly one record per episode. Cleared the moment the disk
|
|
348
|
+
* measures healthy again, so a machine that fills and drains twice writes two
|
|
349
|
+
* records rather than one.
|
|
350
|
+
*/
|
|
351
|
+
let lowDiskReported = false;
|
|
301
352
|
/** Earliest server-approved time for the next polling iteration. */
|
|
302
353
|
let backoffUntil = Number.NEGATIVE_INFINITY;
|
|
354
|
+
/**
|
|
355
|
+
* Consecutive hub polls that failed for a reason the hub did not sanction.
|
|
356
|
+
*
|
|
357
|
+
* RATE LIMITING IS EXCLUDED, and that exclusion is the whole design. A 429 is
|
|
358
|
+
* the server saying "come back later", and a Cloudflare 1015 ban is the same
|
|
359
|
+
* answer at the edge: both are recoverable degradation that clears on its own,
|
|
360
|
+
* and a shift that killed itself over one would take the whole fleet down for
|
|
361
|
+
* a condition the fleet itself caused. `noteServerBackoff` already returns
|
|
362
|
+
* exactly that discrimination, so the streak counts only the failures nobody
|
|
363
|
+
* promised would end.
|
|
364
|
+
*
|
|
365
|
+
* Reset by any SUCCESSFUL wake, which is the loop's one unconditional hub call
|
|
366
|
+
* per unbackoffed tick and therefore the cheapest honest proof that the hub is
|
|
367
|
+
* reachable and this host can reach it.
|
|
368
|
+
*/
|
|
369
|
+
let hubFailureStreak = 0;
|
|
370
|
+
/** Set once the shift has stopped for a named local fault, so `run` exits non-zero. */
|
|
371
|
+
let wedged = false;
|
|
303
372
|
/** Bumped whenever a ping starts or is forced due, so a ping that completes
|
|
304
373
|
* after a force does not stamp the cadence over that force. */
|
|
305
374
|
let presenceGeneration = 0;
|
|
@@ -377,6 +446,58 @@ export function createShiftLoop(opts) {
|
|
|
377
446
|
backoffUntil = Math.max(backoffUntil, monotonicNow() + delay);
|
|
378
447
|
return true;
|
|
379
448
|
}
|
|
449
|
+
/**
|
|
450
|
+
* Ask whether a LOCAL fault is the reason this shift cannot work, and stop if
|
|
451
|
+
* one is. Returns true when the shift has been wedged.
|
|
452
|
+
*
|
|
453
|
+
* STOPPING IS THE POINT, and it is the opposite of what every other failure
|
|
454
|
+
* path here does. Everything else in this loop is built to survive: a failed
|
|
455
|
+
* wake retries next tick, a failed roster sync degrades and continues, a
|
|
456
|
+
* rate-limited poll backs off. All of that is correct for a REMOTE fault,
|
|
457
|
+
* which ends without anyone doing anything. It is exactly wrong for a local
|
|
458
|
+
* one, where continuing means an operator watching a live process that will
|
|
459
|
+
* never dispatch again — the measured shape of this defect was a shift that
|
|
460
|
+
* looked alive to `pgrep`, answered `shift end`, and did nothing for 42
|
|
461
|
+
* minutes while attributing a vanished temp directory to `fetch failed`.
|
|
462
|
+
*
|
|
463
|
+
* A PRE-FLIGHT THAT ITSELF FAILS IS NOT A FAULT. If the probe throws, this
|
|
464
|
+
* reports it and returns false, because "I could not determine whether the
|
|
465
|
+
* host is broken" is not evidence that it is, and killing a working shift over
|
|
466
|
+
* an unreadable `stat` would be a worse failure than the one being detected.
|
|
467
|
+
*/
|
|
468
|
+
/**
|
|
469
|
+
* Free space for the volume the shift's children will write into.
|
|
470
|
+
*
|
|
471
|
+
* Defaults to `unknown` — never a refusal — when no probe was injected and no
|
|
472
|
+
* work root was resolved: the floor and the work root are both RUNTIME
|
|
473
|
+
* configuration, and a loop constructed without them has no basis to decline
|
|
474
|
+
* anything. `runtime.ts` supplies the real closure.
|
|
475
|
+
*/
|
|
476
|
+
function diskSpace() {
|
|
477
|
+
if (opts.diskSpace !== undefined)
|
|
478
|
+
return opts.diskSpace();
|
|
479
|
+
return { state: 'unknown', path: opts.workRoot ?? '' };
|
|
480
|
+
}
|
|
481
|
+
function runHostPreflight(streak) {
|
|
482
|
+
let faults;
|
|
483
|
+
try {
|
|
484
|
+
faults = opts.hostPreflight !== undefined
|
|
485
|
+
? opts.hostPreflight()
|
|
486
|
+
: checkHost({});
|
|
487
|
+
}
|
|
488
|
+
catch (e) {
|
|
489
|
+
opts.err(`host pre-flight failed to run: ${errMsg(e)} (continuing)`);
|
|
490
|
+
return false;
|
|
491
|
+
}
|
|
492
|
+
if (faults.length === 0)
|
|
493
|
+
return false;
|
|
494
|
+
for (const fault of faults)
|
|
495
|
+
opts.err(`shift cannot continue: ${fault.message}`);
|
|
496
|
+
wedged = true;
|
|
497
|
+
stopped = true;
|
|
498
|
+
emit({ type: 'wedged', faults, streak });
|
|
499
|
+
return true;
|
|
500
|
+
}
|
|
380
501
|
/**
|
|
381
502
|
* Fan one observation to every consumer, stamped.
|
|
382
503
|
*
|
|
@@ -1322,11 +1443,24 @@ export function createShiftLoop(opts) {
|
|
|
1322
1443
|
changed = w.changed;
|
|
1323
1444
|
cursor = w.cursor;
|
|
1324
1445
|
wakeSucceeded = true;
|
|
1446
|
+
// The loop's one unconditional hub call per unbackoffed tick, so its
|
|
1447
|
+
// success is the cheapest honest proof that this host can reach the hub.
|
|
1448
|
+
hubFailureStreak = 0;
|
|
1325
1449
|
}
|
|
1326
1450
|
catch (e) {
|
|
1327
|
-
noteServerBackoff(e);
|
|
1451
|
+
const rateLimited = noteServerBackoff(e);
|
|
1328
1452
|
opts.err(`wake failed: ${errMsg(e)} (retrying next tick)`);
|
|
1329
1453
|
emit({ type: 'hub-error', op: 'wake', message: errMsg(e) });
|
|
1454
|
+
if (!rateLimited) {
|
|
1455
|
+
hubFailureStreak += 1;
|
|
1456
|
+
// Every HOST_PREFLIGHT_STREAK-th failure, not only the first crossing:
|
|
1457
|
+
// a work root can be unmounted at failure seven, and a shift that
|
|
1458
|
+
// checked once at failure three would never see it. The check is
|
|
1459
|
+
// `stat`-only and adds no hub traffic, so repeating it is nearly free.
|
|
1460
|
+
if (hubFailureStreak % HOST_PREFLIGHT_STREAK === 0 && runHostPreflight(hubFailureStreak)) {
|
|
1461
|
+
return dispatched;
|
|
1462
|
+
}
|
|
1463
|
+
}
|
|
1330
1464
|
}
|
|
1331
1465
|
// A changed cursor is not sufficient by itself: if the prior changed tick
|
|
1332
1466
|
// had no local capacity, or its sweep failed, the adopted cursor will be
|
|
@@ -1339,7 +1473,38 @@ export function createShiftLoop(opts) {
|
|
|
1339
1473
|
// can actually make the targeted decision.
|
|
1340
1474
|
const capacityCooldownDue = capacityCooldownDueAt !== undefined && monotonicNow() >= capacityCooldownDueAt;
|
|
1341
1475
|
const capacityBecameReady = capacityCooldownReady(k, agentRoom);
|
|
1342
|
-
|
|
1476
|
+
const wantSweep = wakeSucceeded && (changed || sweepOwed || brakeDue || capacityCooldownDue || capacityBecameReady) && k > 0;
|
|
1477
|
+
/*
|
|
1478
|
+
* Measured ONLY when a sweep would otherwise happen, so a parked or
|
|
1479
|
+
* saturated shift pays no syscall for a question it is not about to act on.
|
|
1480
|
+
* That placement is also what makes recovery automatic: refusing sets
|
|
1481
|
+
* `sweepOwed`, `sweepOwed` makes the next tick want to sweep, and wanting to
|
|
1482
|
+
* sweep re-measures the disk. No timer and no separate recovery path.
|
|
1483
|
+
*/
|
|
1484
|
+
const disk = wantSweep ? diskSpace() : undefined;
|
|
1485
|
+
if (wantSweep && disk?.state === 'low') {
|
|
1486
|
+
/*
|
|
1487
|
+
* REFUSE NEW WORK, DO NOT STOP. Unlike the vanished temp directory that
|
|
1488
|
+
* `runHostPreflight` exits for, a full disk clears on its own, so the
|
|
1489
|
+
* shift stays up and keeps polling. In-flight children are deliberately
|
|
1490
|
+
* left running: killing one frees little and destroys all of its work.
|
|
1491
|
+
*/
|
|
1492
|
+
sweepOwed = true;
|
|
1493
|
+
opts.out(`low disk: ${formatBytes(disk.freeBytes)} free at ${disk.path}, below the ${formatBytes(disk.floorBytes)} floor — ` +
|
|
1494
|
+
`deferring whats_next until space is free`);
|
|
1495
|
+
// EDGE-TRIGGERED, exactly as the at-capacity record below: one per
|
|
1496
|
+
// low-disk EPISODE. The console line above stays level-triggered because
|
|
1497
|
+
// a live tail is watched by someone who wants to see it is still stuck.
|
|
1498
|
+
if (!lowDiskReported) {
|
|
1499
|
+
lowDiskReported = true;
|
|
1500
|
+
emit({ type: 'low-disk', path: disk.path, freeBytes: disk.freeBytes, floorBytes: disk.floorBytes });
|
|
1501
|
+
}
|
|
1502
|
+
}
|
|
1503
|
+
else if (wantSweep) {
|
|
1504
|
+
if (lowDiskReported) {
|
|
1505
|
+
lowDiskReported = false;
|
|
1506
|
+
opts.out('disk space recovered — resuming dispatch');
|
|
1507
|
+
}
|
|
1343
1508
|
// Clear before awaiting: a new brake armed during sweep must survive.
|
|
1344
1509
|
if (brakeDue)
|
|
1345
1510
|
brakeSweepDueAt = undefined;
|
|
@@ -1394,21 +1559,28 @@ export function createShiftLoop(opts) {
|
|
|
1394
1559
|
}
|
|
1395
1560
|
async function run() {
|
|
1396
1561
|
ensureStateDir(opts.stateDir);
|
|
1562
|
+
// BEFORE THE FIRST HUB CALL. A shift started into a broken host — the usual
|
|
1563
|
+
// way being a supervisor that restarts it across a reboot with the old
|
|
1564
|
+
// environment — must say so at once. Making it wait for three failed polls
|
|
1565
|
+
// would buy nothing and spend the operator's attention on network theories.
|
|
1566
|
+
// `streak: 0` marks this record as the boot check rather than a poll one.
|
|
1567
|
+
if (runHostPreflight(0))
|
|
1568
|
+
return 1;
|
|
1397
1569
|
if (opts.once === true) {
|
|
1398
1570
|
await iteration();
|
|
1399
|
-
return 0;
|
|
1571
|
+
return wedged ? 1 : 0;
|
|
1400
1572
|
}
|
|
1401
1573
|
for (;;) {
|
|
1402
1574
|
if (stopped)
|
|
1403
|
-
return 0;
|
|
1575
|
+
return wedged ? 1 : 0;
|
|
1404
1576
|
await iteration();
|
|
1405
1577
|
if (stopped)
|
|
1406
|
-
return 0;
|
|
1578
|
+
return wedged ? 1 : 0;
|
|
1407
1579
|
const delay = Math.max(opts.pollIntervalMs, backoffUntil - monotonicNow());
|
|
1408
1580
|
await opts.sleep(delay);
|
|
1409
1581
|
// Re-check after the park so a stop during sleep makes no further hub call.
|
|
1410
1582
|
if (stopped)
|
|
1411
|
-
return 0;
|
|
1583
|
+
return wedged ? 1 : 0;
|
|
1412
1584
|
}
|
|
1413
1585
|
}
|
|
1414
1586
|
function stop() {
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
* workflow state; this protocol only carries local dispatch observations and
|
|
7
7
|
* attendance commands between the CLI and one daemon process.
|
|
8
8
|
*/
|
|
9
|
+
import type { HostFault } from './host-preflight.ts';
|
|
9
10
|
export declare const SHIFT_SOCKET_NAME = "shift.sock";
|
|
10
11
|
export declare const DEFAULT_NEXT_WAIT_MS = 90000;
|
|
11
12
|
export declare const MAX_REQUEST_LINE_BYTES: number;
|
|
@@ -119,6 +120,30 @@ export interface CapacityEvent {
|
|
|
119
120
|
inFlight: number;
|
|
120
121
|
cap: number;
|
|
121
122
|
}
|
|
123
|
+
/**
|
|
124
|
+
* The shift declined to start new work because the volume holding its work root
|
|
125
|
+
* is nearly full.
|
|
126
|
+
*
|
|
127
|
+
* EDGE-TRIGGERED, like `CapacityEvent`: one record per low-disk EPISODE, not one
|
|
128
|
+
* per tick. A shift that sits on a full disk for an hour writes this once, and
|
|
129
|
+
* the recovery is legible from the timestamp of whatever it dispatches next.
|
|
130
|
+
*
|
|
131
|
+
* FILE-ONLY, for the same reason as `CapacityEvent` and `ParkedEvent`: a parked
|
|
132
|
+
* `shift next` must not be woken by news that nothing happened. In the file it
|
|
133
|
+
* is the only thing separating "this shift had no work" from "this shift had
|
|
134
|
+
* work and could not safely take it".
|
|
135
|
+
*
|
|
136
|
+
* `freeBytes` and `floorBytes` are both carried so the record is self-contained.
|
|
137
|
+
* A reader must be able to tell an operator who set an unreachable floor from a
|
|
138
|
+
* genuinely full disk without knowing what the shift was configured with.
|
|
139
|
+
*/
|
|
140
|
+
export interface LowDiskEvent {
|
|
141
|
+
type: 'low-disk';
|
|
142
|
+
/** The measured directory: the work root, or its nearest existing ancestor. */
|
|
143
|
+
path: string;
|
|
144
|
+
freeBytes: number;
|
|
145
|
+
floorBytes: number;
|
|
146
|
+
}
|
|
122
147
|
/** A hub call failed. `workflow` is present for per-workflow whats_next and release calls. */
|
|
123
148
|
export interface HubErrorEvent {
|
|
124
149
|
type: 'hub-error';
|
|
@@ -146,6 +171,42 @@ export interface OrderDroppedEvent {
|
|
|
146
171
|
reason: 'malformed-digest' | 'malformed-worker' | 'unsupported-worker' | 'verification-failed' | 'metadata-unavailable' | 'agent-lane-closed' | 'dispatch-cap-full' | 'agent-cap-full' | 'claim-expired';
|
|
147
172
|
message: string;
|
|
148
173
|
}
|
|
174
|
+
/**
|
|
175
|
+
* The shift STOPPED because the machine underneath it is broken, and named the
|
|
176
|
+
* fault. The last record a wedged shift writes; the process exits non-zero
|
|
177
|
+
* immediately after.
|
|
178
|
+
*
|
|
179
|
+
* WHY THIS EXISTS AS ITS OWN RECORD, rather than one more `hub-error`. A shift
|
|
180
|
+
* whose host temp directory vanished — which a macOS reboot does to any process
|
|
181
|
+
* that survives it — keeps polling forever, and every hub call fails with
|
|
182
|
+
* undici's generic `fetch failed`. From the outside that is indistinguishable
|
|
183
|
+
* from a hub outage: `pgrep` shows the process, `shift end` answers, and the
|
|
184
|
+
* log fills with records that attribute a LOCAL fault to the network. This
|
|
185
|
+
* record is the one that says the cause is here, on this machine, and that no
|
|
186
|
+
* amount of waiting will fix it.
|
|
187
|
+
*
|
|
188
|
+
* `faults` is never empty. That invariant is what separates this record from
|
|
189
|
+
* every retryable failure: a hub outage, a rate-limit ban and an expired
|
|
190
|
+
* credential all self-heal, so the shift keeps working through them and emits
|
|
191
|
+
* nothing here. A record exists only when a human must change something on the
|
|
192
|
+
* host.
|
|
193
|
+
*
|
|
194
|
+
* `streak` is how many consecutive hub calls failed before the check ran, and
|
|
195
|
+
* is 0 for the pre-flight the shift runs at start, before any hub call. It is
|
|
196
|
+
* evidence, not a cause: the streak is only what PROMPTED the check, and the
|
|
197
|
+
* `faults` are why the shift stopped.
|
|
198
|
+
*
|
|
199
|
+
* ON THE SOCKET AS WELL AS IN THE FILE, unlike `parked` and `capacity`. A
|
|
200
|
+
* parked `owenloop shift next` must block until there is something to report,
|
|
201
|
+
* and "the shift you are waiting on is dead" is the most report-worthy thing
|
|
202
|
+
* that can happen to it — a client that stays parked on a socket whose server
|
|
203
|
+
* is about to close waits for work that will never come.
|
|
204
|
+
*/
|
|
205
|
+
export interface WedgedEvent {
|
|
206
|
+
type: 'wedged';
|
|
207
|
+
faults: HostFault[];
|
|
208
|
+
streak: number;
|
|
209
|
+
}
|
|
149
210
|
/**
|
|
150
211
|
* The socket event FIFO overflowed and discarded its oldest event. Written
|
|
151
212
|
* STRAIGHT to the file sink, never through the loop's `emit()` — `emit()` feeds
|
|
@@ -161,7 +222,7 @@ export interface EventQueueOverflowEvent {
|
|
|
161
222
|
* What a construction site builds: the event's own payload, with no identity
|
|
162
223
|
* and no timestamp. Every consumer receives the stamped `ShiftEvent` instead.
|
|
163
224
|
*/
|
|
164
|
-
export type ShiftEventBody = DispatchedEvent | ReapedEvent | FailedEvent | GateEvent | EndedEvent | ParkedEvent | CapacityEvent | HubErrorEvent | BundleMissEvent | OrderDroppedEvent | EventQueueOverflowEvent;
|
|
225
|
+
export type ShiftEventBody = DispatchedEvent | ReapedEvent | FailedEvent | GateEvent | EndedEvent | ParkedEvent | CapacityEvent | HubErrorEvent | BundleMissEvent | OrderDroppedEvent | EventQueueOverflowEvent | WedgedEvent | LowDiskEvent;
|
|
165
226
|
/**
|
|
166
227
|
* The identity every event carries, added once on a shared envelope rather than
|
|
167
228
|
* per variant.
|
|
@@ -35,6 +35,8 @@ export interface ParsedArgs {
|
|
|
35
35
|
once?: boolean;
|
|
36
36
|
maxAgents?: number;
|
|
37
37
|
execReserve?: number;
|
|
38
|
+
/** `--disk-floor <bytes>` — free space required to start new work. `0` disables. */
|
|
39
|
+
diskFloorBytes?: number;
|
|
38
40
|
localQueueHoldMs?: number;
|
|
39
41
|
cacheDir?: string;
|
|
40
42
|
stateDir?: string;
|
|
@@ -16,6 +16,7 @@ import { loadSettings } from "../settings/settings.js";
|
|
|
16
16
|
import { DEFAULT_HUB_ROSTER_SYNC_TIMEOUT_MS, syncHubRosterCache, withHubRosterSyncTimeout } from "../settings/hub-roster-cache.js";
|
|
17
17
|
import { computeServeCapabilities } from "../settings/serving.js";
|
|
18
18
|
import { resolveCacheDir } from "../bundle/cache.js";
|
|
19
|
+
import { checkDiskFloor, resolveDiskFloorBytes } from "./disk-floor.js";
|
|
19
20
|
import { createLockedRemovalCallbacks, createShiftLoop } from "./loop.js";
|
|
20
21
|
import { createHubBundleRecoveryHandler } from "../bundle/pull.js";
|
|
21
22
|
import { createShiftLogSink } from "./logsink.js";
|
|
@@ -93,6 +94,7 @@ const FILE_ONLY_EVENTS = new Set([
|
|
|
93
94
|
'capacity',
|
|
94
95
|
'hub-error',
|
|
95
96
|
'event-queue-overflow',
|
|
97
|
+
'low-disk',
|
|
96
98
|
]);
|
|
97
99
|
/**
|
|
98
100
|
* Does this event type reach the SOCKET consumer (a parked `owenloop shift
|
|
@@ -161,6 +163,7 @@ export function parseArgs(args) {
|
|
|
161
163
|
case '--cap':
|
|
162
164
|
case '--max-agents':
|
|
163
165
|
case '--exec-reserve':
|
|
166
|
+
case '--disk-floor':
|
|
164
167
|
case '--local-queue-hold':
|
|
165
168
|
case '--workflow':
|
|
166
169
|
case '--poll-interval':
|
|
@@ -217,6 +220,12 @@ export function parseArgs(args) {
|
|
|
217
220
|
return { error: n.error };
|
|
218
221
|
parsed.execReserve = n;
|
|
219
222
|
}
|
|
223
|
+
else if (name === '--disk-floor') {
|
|
224
|
+
const n = intFlag(r.value, '--disk-floor');
|
|
225
|
+
if (typeof n !== 'number')
|
|
226
|
+
return { error: n.error };
|
|
227
|
+
parsed.diskFloorBytes = n;
|
|
228
|
+
}
|
|
220
229
|
else if (name === '--local-queue-hold') {
|
|
221
230
|
const n = intFlag(r.value, '--local-queue-hold');
|
|
222
231
|
if (typeof n !== 'number')
|
|
@@ -404,6 +413,7 @@ export async function runShiftRuntime(parsed, options = {}) {
|
|
|
404
413
|
const maxConcurrentAgents = resolveMaxConcurrentAgents(parsed.maxAgents, settings.maxConcurrentAgents);
|
|
405
414
|
const execReserve = resolveExecReserve(parsed.execReserve, settings.execReserve);
|
|
406
415
|
const localQueueHoldMs = resolveLocalQueueHoldMs(parsed.localQueueHoldMs, settings.localQueueHoldMs);
|
|
416
|
+
const diskFloorBytes = resolveDiskFloorBytes(parsed.diskFloorBytes, settings.diskFloorBytes);
|
|
407
417
|
const workRoot = resolveWorkRoot(env, settings.workRoot, cacheDir);
|
|
408
418
|
const workRepo = resolveWorkRepo(env, settings.workRepo);
|
|
409
419
|
/**
|
|
@@ -588,6 +598,13 @@ export async function runShiftRuntime(parsed, options = {}) {
|
|
|
588
598
|
execReserve,
|
|
589
599
|
localQueueHoldMs,
|
|
590
600
|
workRoot,
|
|
601
|
+
/*
|
|
602
|
+
* Resolved here, not in the loop: `runtime.ts` owns flag/settings/default
|
|
603
|
+
* precedence for every other tunable, and the loop consumes a decision. A
|
|
604
|
+
* loop built without this closure has no disk gate at all, which is the
|
|
605
|
+
* right default for a caller that never told it where work lands.
|
|
606
|
+
*/
|
|
607
|
+
diskSpace: () => checkDiskFloor(workRoot, diskFloorBytes),
|
|
591
608
|
...(workRepo !== undefined ? { workRepo } : {}),
|
|
592
609
|
shiftId,
|
|
593
610
|
startedAt,
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Read a submit value from a JSON document contained in a run worktree.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* canonical check
|
|
4
|
+
* Containment is `resolveContainedPath`'s job — lexical traversal check first,
|
|
5
|
+
* then a canonical symlink check. This function owns only what is specific to
|
|
6
|
+
* a submit value: the argument shape and the JSON decode.
|
|
6
7
|
*/
|
|
7
8
|
export declare function readSubmitValueFile(workdir: string, valueFile: unknown): Promise<unknown>;
|
|
@@ -1,36 +1,20 @@
|
|
|
1
|
-
import { readFile
|
|
2
|
-
import {
|
|
3
|
-
import { isInside } from "./harness/gatekeeper.js";
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { resolveContainedPath } from "./contained-path.js";
|
|
4
3
|
function errorMessage(error) {
|
|
5
4
|
return error instanceof Error ? error.message : String(error);
|
|
6
5
|
}
|
|
7
6
|
/**
|
|
8
7
|
* Read a submit value from a JSON document contained in a run worktree.
|
|
9
8
|
*
|
|
10
|
-
*
|
|
11
|
-
* canonical check
|
|
9
|
+
* Containment is `resolveContainedPath`'s job — lexical traversal check first,
|
|
10
|
+
* then a canonical symlink check. This function owns only what is specific to
|
|
11
|
+
* a submit value: the argument shape and the JSON decode.
|
|
12
12
|
*/
|
|
13
13
|
export async function readSubmitValueFile(workdir, valueFile) {
|
|
14
14
|
if (typeof valueFile !== 'string' || valueFile.trim() === '') {
|
|
15
15
|
throw new Error(`submit-value-file-invalid: valueFile must be a non-empty string: ${String(valueFile)}`);
|
|
16
16
|
}
|
|
17
|
-
const
|
|
18
|
-
if (!isInside(workdir, candidate)) {
|
|
19
|
-
throw new Error(`submit-value-file-outside-workdir: ${valueFile} is outside the run workdir`);
|
|
20
|
-
}
|
|
21
|
-
let canonicalCandidate;
|
|
22
|
-
try {
|
|
23
|
-
const canonicalWorkdir = await realpath(workdir);
|
|
24
|
-
canonicalCandidate = await realpath(candidate);
|
|
25
|
-
if (!isInside(canonicalWorkdir, canonicalCandidate)) {
|
|
26
|
-
throw new Error(`submit-value-file-outside-workdir: ${valueFile} is outside the run workdir`);
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
catch (error) {
|
|
30
|
-
if (error instanceof Error && error.message.startsWith('submit-value-file-outside-workdir:'))
|
|
31
|
-
throw error;
|
|
32
|
-
throw new Error(`submit-value-file-read-failed: could not read ${valueFile}: ${errorMessage(error)}`);
|
|
33
|
-
}
|
|
17
|
+
const canonicalCandidate = await resolveContainedPath(workdir, valueFile, 'submit-value-file');
|
|
34
18
|
let text;
|
|
35
19
|
try {
|
|
36
20
|
text = await readFile(canonicalCandidate, 'utf8');
|
package/dist/src/defs.js
CHANGED
|
@@ -39,7 +39,7 @@ const RAW_PRODUCE_KEYS = ['name', 'schema', 'judges', 'maxAttempts', 'maxSchemaF
|
|
|
39
39
|
* fields, so it gets its own allowlist rather than folding into
|
|
40
40
|
* RAW_PRODUCE_KEYS (§27 unknown-key rejection). */
|
|
41
41
|
const RAW_GROUP_KEYS = ['group', 'mode', 'of'];
|
|
42
|
-
const RAW_JUDGE_KEYS = ['name', 'body', 'bodyFile', 'model', 'inputs', 'cadence', 'maxRunsPerDay', 'executor', 'command', 'spec', 'capabilities', 'modifiers'];
|
|
42
|
+
const RAW_JUDGE_KEYS = ['name', 'body', 'bodyFile', 'model', 'inputs', 'cadence', 'maxRunsPerDay', 'executor', 'command', 'spec', 'workdir', 'workdirFrom', 'capabilities', 'modifiers'];
|
|
43
43
|
/** Keys valid on a normal (non-calls:, non-include:) step entry. */
|
|
44
44
|
const RAW_STEP_KEYS = [
|
|
45
45
|
'name', 'consumes', 'produces', 'generates', 'invalidates', 'cadence',
|
|
@@ -362,6 +362,14 @@ function parseJudges(v, ctx, baseDir) {
|
|
|
362
362
|
judge.command = asString(raw.command, `judge '${name}'.command`);
|
|
363
363
|
if (raw.spec !== undefined)
|
|
364
364
|
judge.spec = asExtension(raw.spec, `judge '${name}'.spec`);
|
|
365
|
+
if (raw.workdir !== undefined)
|
|
366
|
+
judge.workdir = asString(raw.workdir, `judge '${name}'.workdir`);
|
|
367
|
+
if (raw.workdirFrom !== undefined) {
|
|
368
|
+
judge.workdirFrom = asString(raw.workdirFrom, `judge '${name}'.workdirFrom`);
|
|
369
|
+
}
|
|
370
|
+
if (raw.workdir !== undefined && raw.workdirFrom !== undefined) {
|
|
371
|
+
throw new DefError(`judge '${name}' may not declare both workdir and workdirFrom`);
|
|
372
|
+
}
|
|
365
373
|
if (raw.capabilities !== undefined) {
|
|
366
374
|
const caps = asStringArray(raw.capabilities, `judge '${name}'.capabilities`);
|
|
367
375
|
if (caps.length === 0) {
|
|
@@ -997,24 +1005,63 @@ export function parseDef(raw, source, baseDir) {
|
|
|
997
1005
|
}
|
|
998
1006
|
return def;
|
|
999
1007
|
}
|
|
1008
|
+
/**
|
|
1009
|
+
* The stem half of a `workdirFrom` expression, resolved against a list of
|
|
1010
|
+
* consumed stems. Mirrors `parseWorkdirFrom`'s boundary walk (longest stem
|
|
1011
|
+
* first, because a stem may itself contain dots) but takes bare names, since a
|
|
1012
|
+
* synthesized judge has not built its consume patterns yet. Returns null when
|
|
1013
|
+
* no consumed stem matches — the expression then names a definition input,
|
|
1014
|
+
* which validateDef resolves without a consume edge.
|
|
1015
|
+
*/
|
|
1016
|
+
function workdirFromStem(raw, stems) {
|
|
1017
|
+
const r = raw.trim();
|
|
1018
|
+
for (let boundary = r.lastIndexOf('.'); boundary > 0; boundary = r.lastIndexOf('.', boundary - 1)) {
|
|
1019
|
+
if (r.length - boundary - 1 === 0)
|
|
1020
|
+
continue;
|
|
1021
|
+
const stem = r.slice(0, boundary);
|
|
1022
|
+
if (stems.includes(stem))
|
|
1023
|
+
return stem;
|
|
1024
|
+
}
|
|
1025
|
+
return null;
|
|
1026
|
+
}
|
|
1000
1027
|
/**
|
|
1001
1028
|
* Synthesize one full StepDef per declared `judges:` entry on a produce
|
|
1002
1029
|
* pattern (§24 §3.2, §7.2). Shape mirrors the `calls:` template above, with
|
|
1003
1030
|
* exactly three deltas from a hand-written step: the `judges: <stem>` marker
|
|
1004
1031
|
* (eligibility trigger, replacing inputsGreen — see model.ts), `produces: []`
|
|
1005
1032
|
* (a judge emits a verdict against the judged stem, not a new artifact), and
|
|
1006
|
-
* `consumes: [stem, ...(inputs ? producerConsumeStems : [])]`
|
|
1007
|
-
* flows from the existing consume-edge check (`assertAuthority`)
|
|
1008
|
-
* special-casing. Everything else (cadence, maxRunsPerDay, model, body,
|
|
1009
|
-
* maxAttempts/maxSchemaFailures defaults) is inherited
|
|
1010
|
-
* an ordinary step, because judge orders flow through the normal
|
|
1033
|
+
* `consumes: [stem, ...(inputs ? producerConsumeStems : []), ...workdirFromStem]`
|
|
1034
|
+
* so authority flows from the existing consume-edge check (`assertAuthority`)
|
|
1035
|
+
* with no special-casing. Everything else (cadence, maxRunsPerDay, model, body,
|
|
1036
|
+
* workdir/workdirFrom, maxAttempts/maxSchemaFailures defaults) is inherited
|
|
1037
|
+
* exactly like an ordinary step, because judge orders flow through the normal
|
|
1011
1038
|
* eligibleFirings → applySchedule → claim → buildOrder pipeline (§7.1).
|
|
1012
1039
|
*/
|
|
1013
|
-
function synthesizeJudgeSteps(producerStepName, pat, producerConsumeStems, producerX, producerCapabilities) {
|
|
1040
|
+
function synthesizeJudgeSteps(producerStepName, pat, producerConsumeStems, producerX, producerCapabilities, producerWorkdir, producerWorkdirFrom) {
|
|
1014
1041
|
if (!pat.judges || pat.judges.length === 0)
|
|
1015
1042
|
return [];
|
|
1016
1043
|
return pat.judges.map((j) => {
|
|
1044
|
+
// A judge with no workdir of its own INHERITS the producer's, so it reads
|
|
1045
|
+
// the same tree the work it judges was produced in. Declaring either key
|
|
1046
|
+
// opts out of both halves of the inheritance: producer and judge each carry
|
|
1047
|
+
// at most one, and silently pairing an authored `workdir` with an inherited
|
|
1048
|
+
// `workdirFrom` would trip validateDef's mutual-exclusion check on a
|
|
1049
|
+
// definition the author never wrote that way.
|
|
1050
|
+
const authoredWorkdir = j.workdir !== undefined || j.workdirFrom !== undefined;
|
|
1051
|
+
const workdir = authoredWorkdir ? j.workdir : producerWorkdir;
|
|
1052
|
+
const workdirFrom = authoredWorkdir ? j.workdirFrom : producerWorkdirFrom;
|
|
1017
1053
|
const consumeStems = j.inputs ? [pat.stem, ...producerConsumeStems] : [pat.stem];
|
|
1054
|
+
// A workdirFrom value is resolved from a CONSUMED artifact, so the stem it
|
|
1055
|
+
// names must be on this step's consume list or validateDef rejects the whole
|
|
1056
|
+
// definition. `inputs: false` is the case that needs this: the judge sees
|
|
1057
|
+
// only the judged stem, which is never the stem the producer resolved its
|
|
1058
|
+
// workdir from. Deduplicated, so `inputs: true` (where the stem is already
|
|
1059
|
+
// carried) stays byte-identical to the pre-inheritance shape.
|
|
1060
|
+
if (workdirFrom !== undefined) {
|
|
1061
|
+
const stem = workdirFromStem(workdirFrom, producerConsumeStems);
|
|
1062
|
+
if (stem !== null && !consumeStems.includes(stem))
|
|
1063
|
+
consumeStems.push(stem);
|
|
1064
|
+
}
|
|
1018
1065
|
const consumes = consumeStems.map((stem) => parseConsume(stem));
|
|
1019
1066
|
const cadence = j.cadence ?? DEFAULTS.cadence;
|
|
1020
1067
|
const step = {
|
|
@@ -1055,6 +1102,10 @@ function synthesizeJudgeSteps(producerStepName, pat, producerConsumeStems, produ
|
|
|
1055
1102
|
step.command = j.command;
|
|
1056
1103
|
if (j.spec !== undefined)
|
|
1057
1104
|
step.spec = j.spec;
|
|
1105
|
+
if (workdir !== undefined)
|
|
1106
|
+
step.workdir = workdir;
|
|
1107
|
+
if (workdirFrom !== undefined)
|
|
1108
|
+
step.workdirFrom = workdirFrom;
|
|
1058
1109
|
if (j.modifiers !== undefined)
|
|
1059
1110
|
step.judgeModifiers = [...j.modifiers];
|
|
1060
1111
|
return step;
|
|
@@ -1243,7 +1294,7 @@ function buildStep(rl, i, baseDir) {
|
|
|
1243
1294
|
}
|
|
1244
1295
|
}
|
|
1245
1296
|
const producerConsumeStems = consumes.map((c) => c.stem);
|
|
1246
|
-
const judgeSteps = producesPatterns.flatMap((p) => synthesizeJudgeSteps(name, p, producerConsumeStems, step.x, step.capabilities));
|
|
1297
|
+
const judgeSteps = producesPatterns.flatMap((p) => synthesizeJudgeSteps(name, p, producerConsumeStems, step.x, step.capabilities, step.workdir, step.workdirFrom));
|
|
1247
1298
|
return [step, ...judgeSteps];
|
|
1248
1299
|
}
|
|
1249
1300
|
// ---- validation --------------------------------------------------------------
|
package/dist/src/types.d.ts
CHANGED
|
@@ -485,6 +485,19 @@ export interface ProducePattern {
|
|
|
485
485
|
* (or additionally alongside `command`). Shape-checked as a plain map
|
|
486
486
|
* only (mirrors `x:`'s asExtension contract) — contents never read. */
|
|
487
487
|
spec?: Record<string, unknown>;
|
|
488
|
+
/** opaque location hint passed through on the judge's synthesized order.
|
|
489
|
+
* Omitted = INHERIT the producing step's `workdir`/`workdirFrom`, so a
|
|
490
|
+
* judge reads the same tree the work it judges was produced in. Mutually
|
|
491
|
+
* exclusive with workdirFrom, same as StepDef. */
|
|
492
|
+
workdir?: string;
|
|
493
|
+
/**
|
|
494
|
+
* Resolve the judge order's workdir from `<consumedStem>.<dotted.path>` in a
|
|
495
|
+
* consumed artifact value. Mutually exclusive with workdir. Omitted =
|
|
496
|
+
* INHERIT the producer's. The named stem is added to the synthesized step's
|
|
497
|
+
* consumes when it is not already there, so the value passes the engine's
|
|
498
|
+
* consume-side verification gate.
|
|
499
|
+
*/
|
|
500
|
+
workdirFrom?: string;
|
|
488
501
|
/** Routing capabilities for this judge's synthesized step. Omitted =
|
|
489
502
|
* INHERIT the producing step's `capabilities`, so a judge routes to the
|
|
490
503
|
* same grade of crew as the work it judges instead of being claimable by
|
package/docs/cli.md
CHANGED
|
@@ -305,6 +305,7 @@ holds `credentials.json`, `allowed_signers`, `org-root.pub`, `grants/`, and
|
|
|
305
305
|
```text
|
|
306
306
|
owenloop shift start <crew...> [--all] [--origin <url>] [--as <account>] [--name <n>]
|
|
307
307
|
[--cap <n>] [--max-agents <n>] [--exec-reserve <n>] [--local-queue-hold <ms>] [--poll-interval <ms>] [--once]
|
|
308
|
+
[--disk-floor <bytes>]
|
|
308
309
|
[--cache-dir <p>] [--state-dir <p>] [--log-dir <p>] [--log-max-age <ms>]
|
|
309
310
|
[--work-root <dir>]...
|
|
310
311
|
```
|
|
@@ -323,6 +324,7 @@ one loop sweep and exits instead of keeping the foreground daemon running.
|
|
|
323
324
|
| `--cap <n>` | dispatch capacity; precedence is flag, then `settings.dispatchCap`, then `3` |
|
|
324
325
|
| `--max-agents <n>` | concurrent agent limit; precedence is flag, then `settings.maxConcurrentAgents`, then `4` |
|
|
325
326
|
| `--exec-reserve <n>` | slots inside `--cap` that `agent-run` children may never occupy, so an exec/command order always has room; precedence is flag, then `settings.execReserve`, then `1`. Clamped to `cap - 1`, so a `--cap 1` shift gets no reserve. `0` disables the reserve and lets agents fill the whole cap. This does not raise the total child ceiling — `--cap` still bounds every child. |
|
|
327
|
+
| `--disk-floor <bytes>` | free space the shift requires on the volume holding its work root before it will START another order; precedence is flag, then `settings.diskFloorBytes`, then `1073741824` (1 GiB). A shift below the floor keeps polling and declines new work, writing one `low-disk` record per episode, and resumes by itself when space returns; children already running are left alone. `0` disables the check. This is a floor for starting work, not a promise that a particular step will fit. |
|
|
326
328
|
| `--local-queue-hold <ms>` | how long to retain a claim this shift cannot dispatch before returning it to the hub; precedence is flag, then `settings.localQueueHoldMs`, then `0`. The value is clamped to 90 seconds. `0` means the shift never holds an undispatchable claim locally, so another daemon can be offered it immediately. |
|
|
327
329
|
| `--poll-interval <ms>` | loop polling interval; defaults to `5000` milliseconds |
|
|
328
330
|
| `--once` | run one loop sweep and exit; without it, keep the daemon in the foreground |
|
|
@@ -587,16 +589,17 @@ either `exec` or `agent-run`:
|
|
|
587
589
|
- `bundle-miss`: `{ "type": "bundle-miss", "workflow": "...", "def": "..." }` — a legacy order named a def with no cached bundle, so it was left for hub pickup
|
|
588
590
|
- `order-dropped`: `{ "type": "order-dropped", "workflow": "...", "run": "...", "step": "...", "reason": "unsupported-worker", "message": "..." }` — the shift refused one order. Match on `reason` (`malformed-digest`, `malformed-worker`, `unsupported-worker`, `verification-failed`, `metadata-unavailable`, `agent-lane-closed`, `dispatch-cap-full`, `agent-cap-full`, `claim-expired`); display `message`. The capacity, expiry, and `agent-lane-closed` reasons return the claim to the hub; malformed and unsupported reasons leave it for the pickup window.
|
|
589
591
|
- `ended`: `{ "type": "ended" }`, delivered to a parked `next` when `shift end` explicitly shuts down the daemon
|
|
592
|
+
- `wedged`: `{ "type": "wedged", "faults": [{ "kind": "missing", "role": "temp-dir", "path": "...", "message": "..." }], "streak": 3 }` — the shift stopped because the MACHINE it runs on is broken, and the process exits non-zero straight after. `faults` is never empty; display each `message`, which names the path and the operator fix. `streak` is how many consecutive hub calls failed before the check ran, and is `0` for the check a shift makes at start. A shift that cannot reach the hub does NOT produce this — a hub outage or a rate-limit ban recovers on its own, and the shift keeps polling through it. This record means a human must change something on the host.
|
|
590
593
|
|
|
591
594
|
Every event above is also appended to `<log-dir>/shift.log` as JSON Lines, which
|
|
592
|
-
is how they survive the daemon. **
|
|
595
|
+
is how they survive the daemon. **Five further record types exist in
|
|
593
596
|
`shift.log` and are never delivered over the socket** — `parked`, `capacity`,
|
|
594
|
-
`hub-error`,
|
|
597
|
+
`hub-error`, `event-queue-overflow`, and `low-disk`.
|
|
595
598
|
|
|
596
599
|
An idle `shift next` must block until there is work to report, and each of those
|
|
597
|
-
|
|
598
|
-
that the shift is full, a failed call to the hub,
|
|
599
|
-
queue itself. `parked` and `capacity` are also redundant on the wire — the
|
|
600
|
+
five would satisfy it with news that no work moved: a startup record, a report
|
|
601
|
+
that the shift is full, a failed call to the hub, a record about the socket
|
|
602
|
+
queue itself, or a report that the disk is too full to start anything. `parked` and `capacity` are also redundant on the wire — the
|
|
600
603
|
response above already carries live `cap`, `free`, and `running`, which is
|
|
601
604
|
exactly what a `capacity` record restates. `hub-error` is level-triggered at the
|
|
602
605
|
poll interval, so an unreachable hub would otherwise fill the 1000-slot socket
|