owenloop 0.5.27 → 0.5.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/packages/work/src/agent/brief.d.ts +10 -0
- package/dist/packages/work/src/agent/brief.js +38 -0
- package/dist/packages/work/src/agent/loop.js +4 -0
- package/dist/packages/work/src/consumed-verifier.d.ts +22 -2
- package/dist/packages/work/src/consumed-verifier.js +131 -12
- package/dist/packages/work/src/contained-path.d.ts +18 -0
- package/dist/packages/work/src/contained-path.js +42 -0
- package/dist/packages/work/src/exec/instructions.js +43 -3
- package/dist/packages/work/src/harness/claude.js +21 -0
- package/dist/packages/work/src/harness/codex.js +29 -0
- package/dist/packages/work/src/harness/contract.d.ts +26 -0
- package/dist/packages/work/src/hold/mcp.d.ts +3 -40
- package/dist/packages/work/src/hold/mcp.js +121 -2
- package/dist/packages/work/src/hub/client.d.ts +10 -1
- package/dist/packages/work/src/hub/client.js +23 -0
- package/dist/packages/work/src/hub/types.d.ts +45 -0
- package/dist/packages/work/src/roles/shift.js +7 -1
- package/dist/packages/work/src/settings/settings.d.ts +12 -1
- package/dist/packages/work/src/settings/settings.js +7 -0
- package/dist/packages/work/src/shift/disk-floor.d.ts +107 -0
- package/dist/packages/work/src/shift/disk-floor.js +148 -0
- package/dist/packages/work/src/shift/host-preflight.d.ts +85 -0
- package/dist/packages/work/src/shift/host-preflight.js +112 -0
- package/dist/packages/work/src/shift/loop.d.ts +18 -0
- package/dist/packages/work/src/shift/loop.js +179 -7
- package/dist/packages/work/src/shift/protocol.d.ts +62 -1
- package/dist/packages/work/src/shift/runtime.d.ts +2 -0
- package/dist/packages/work/src/shift/runtime.js +17 -0
- package/dist/packages/work/src/submit-file.d.ts +3 -2
- package/dist/packages/work/src/submit-file.js +6 -22
- package/dist/src/crypto/records.d.ts +1 -0
- package/dist/src/crypto/records.js +1 -0
- package/dist/src/crypto/verify-consumed.d.ts +16 -1
- package/dist/src/crypto/verify-consumed.js +21 -6
- package/dist/src/schemas/index.d.ts +23 -0
- package/dist/src/schemas/order.v1.schema.json +14 -0
- package/dist/src/store/index.d.ts +1 -1
- package/dist/src/store/instruction-source.d.ts +15 -0
- package/dist/src/store/instruction-source.js +64 -4
- package/dist/src/store/types.d.ts +6 -2
- package/dist/src/store/types.js +5 -1
- package/dist/src/types.d.ts +11 -0
- package/docs/bundles.md +5 -1
- package/docs/cli.md +8 -5
- package/docs/crypto.md +46 -0
- package/docs/shift-logs.md +64 -7
- package/docs/wire-contracts.md +15 -2
- package/package.json +1 -1
- package/plugins/claude-code/plugin/.claude-plugin/plugin.json +1 -1
- package/plugins/claude-code/plugin/.mcp.json +1 -1
- package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +1 -1
- package/plugins/codex/plugins/owenloop/.mcp.json +1 -1
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Is this machine still the machine the shift started on?
|
|
3
|
+
*
|
|
4
|
+
* A shift daemon is a long-lived process holding paths it resolved once, at
|
|
5
|
+
* start, and one of those paths is not stable for the life of the process:
|
|
6
|
+
* macOS gives each boot a fresh per-user temp directory
|
|
7
|
+
* (`/var/folders/<a>/<b>/T`). A process that survives a reboot — or was started
|
|
8
|
+
* before one and re-parented — keeps the OLD `TMPDIR` in its environment, and
|
|
9
|
+
* that directory is gone.
|
|
10
|
+
*
|
|
11
|
+
* The fault does not announce itself. The shift keeps polling, every hub call
|
|
12
|
+
* and every child spawn fails for a reason the error text attributes to the
|
|
13
|
+
* network ("fetch failed"), and the daemon looks alive to `pgrep` and answers
|
|
14
|
+
* `shift end` while dispatching nothing. It was measured wedged this way for 42
|
|
15
|
+
* minutes with no record naming the cause.
|
|
16
|
+
*
|
|
17
|
+
* WHAT THIS MODULE IS FOR, AND WHAT IT IS NOT FOR. It answers exactly one
|
|
18
|
+
* question: "is a LOCAL, non-self-healing fault the reason this shift cannot
|
|
19
|
+
* work?" It is deliberately not a health check and not a reachability probe. A
|
|
20
|
+
* hub outage, a Cloudflare 1015 ban, an expired credential and a flaky DHCP
|
|
21
|
+
* lease are all REMOTE and all self-heal without operator action, so this
|
|
22
|
+
* module must report them clean — the caller's whole decision rests on that
|
|
23
|
+
* split, because a clean answer means "keep polling" and a dirty one means
|
|
24
|
+
* "stop, a human must fix the host".
|
|
25
|
+
*
|
|
26
|
+
* PURE AND INJECTABLE. Every filesystem call is taken through `FsProbe` so the
|
|
27
|
+
* caller's tests can present a missing temp directory without one, and so a
|
|
28
|
+
* probe that itself throws is reported as a fault rather than crashing the
|
|
29
|
+
* shift.
|
|
30
|
+
*/
|
|
31
|
+
/** Which of the shift's own directories a fault was found in. */
|
|
32
|
+
export type HostPathRole = 'temp-dir';
|
|
33
|
+
export interface HostFault {
|
|
34
|
+
/**
|
|
35
|
+
* The stable machine discriminator. The three kinds are separate because
|
|
36
|
+
* they have different operator fixes: a missing temp directory means the host
|
|
37
|
+
* rebooted and the shift must be restarted from a fresh shell, while an
|
|
38
|
+
* unwritable one means a permission change that restarting will not repair.
|
|
39
|
+
*/
|
|
40
|
+
kind: 'missing' | 'not-a-directory' | 'unwritable';
|
|
41
|
+
role: HostPathRole;
|
|
42
|
+
path: string;
|
|
43
|
+
/** Operator-facing text: what is wrong, why it happened, and what to do. */
|
|
44
|
+
message: string;
|
|
45
|
+
}
|
|
46
|
+
/** The filesystem calls this check makes, as one injectable seam. */
|
|
47
|
+
export interface FsProbe {
|
|
48
|
+
isDirectory: (path: string) => boolean;
|
|
49
|
+
exists: (path: string) => boolean;
|
|
50
|
+
isWritable: (path: string) => boolean;
|
|
51
|
+
}
|
|
52
|
+
export declare const realFsProbe: FsProbe;
|
|
53
|
+
/**
|
|
54
|
+
* The path checked, and DELIBERATELY only this one.
|
|
55
|
+
*
|
|
56
|
+
* A shift also holds a state directory, a bundle cache directory and a work
|
|
57
|
+
* root, and none of them is here: the shift CREATES all three on demand
|
|
58
|
+
* (`resolveWorkRoot` defaults the work root to `<cacheDir>/work`, and
|
|
59
|
+
* `prepareWorkdir` mkdir -p's it), so their absence is a normal first-run
|
|
60
|
+
* condition rather than a fault. A check that treated it as one would refuse to
|
|
61
|
+
* start every fresh install — which is exactly what an earlier draft of this
|
|
62
|
+
* module did, and what the suite caught. The temp directory is the one path the
|
|
63
|
+
* shift depends on, does not create, and cannot recover on its own.
|
|
64
|
+
*
|
|
65
|
+
* The operator's `--work-root` list is a different thing again: it NARROWS
|
|
66
|
+
* which roots an order may name rather than supplying a directory the shift
|
|
67
|
+
* writes to, and losing one of several does not stop the shift serving orders
|
|
68
|
+
* that name the others. If it is ever worth checking, it is worth checking as
|
|
69
|
+
* its own decision, not folded in here.
|
|
70
|
+
*/
|
|
71
|
+
export interface HostPaths {
|
|
72
|
+
/** Absent means "read the live `TMPDIR`", which is what the shift itself uses. */
|
|
73
|
+
tempDir?: string;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Report every local host fault, in a fixed role order so a reader comparing
|
|
77
|
+
* two records sees the same fault in the same position.
|
|
78
|
+
*
|
|
79
|
+
* The return is an array rather than a single fault so that adding a second
|
|
80
|
+
* role later does not change this function's contract, and so a caller never
|
|
81
|
+
* has to ask whether the one fault it was handed was the only one. An empty
|
|
82
|
+
* array means "no local fault" — which the caller must read as "the cause is
|
|
83
|
+
* elsewhere", never as "the shift is healthy".
|
|
84
|
+
*/
|
|
85
|
+
export declare function checkHost(paths: HostPaths, fs?: FsProbe): HostFault[];
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Is this machine still the machine the shift started on?
|
|
3
|
+
*
|
|
4
|
+
* A shift daemon is a long-lived process holding paths it resolved once, at
|
|
5
|
+
* start, and one of those paths is not stable for the life of the process:
|
|
6
|
+
* macOS gives each boot a fresh per-user temp directory
|
|
7
|
+
* (`/var/folders/<a>/<b>/T`). A process that survives a reboot — or was started
|
|
8
|
+
* before one and re-parented — keeps the OLD `TMPDIR` in its environment, and
|
|
9
|
+
* that directory is gone.
|
|
10
|
+
*
|
|
11
|
+
* The fault does not announce itself. The shift keeps polling, every hub call
|
|
12
|
+
* and every child spawn fails for a reason the error text attributes to the
|
|
13
|
+
* network ("fetch failed"), and the daemon looks alive to `pgrep` and answers
|
|
14
|
+
* `shift end` while dispatching nothing. It was measured wedged this way for 42
|
|
15
|
+
* minutes with no record naming the cause.
|
|
16
|
+
*
|
|
17
|
+
* WHAT THIS MODULE IS FOR, AND WHAT IT IS NOT FOR. It answers exactly one
|
|
18
|
+
* question: "is a LOCAL, non-self-healing fault the reason this shift cannot
|
|
19
|
+
* work?" It is deliberately not a health check and not a reachability probe. A
|
|
20
|
+
* hub outage, a Cloudflare 1015 ban, an expired credential and a flaky DHCP
|
|
21
|
+
* lease are all REMOTE and all self-heal without operator action, so this
|
|
22
|
+
* module must report them clean — the caller's whole decision rests on that
|
|
23
|
+
* split, because a clean answer means "keep polling" and a dirty one means
|
|
24
|
+
* "stop, a human must fix the host".
|
|
25
|
+
*
|
|
26
|
+
* PURE AND INJECTABLE. Every filesystem call is taken through `FsProbe` so the
|
|
27
|
+
* caller's tests can present a missing temp directory without one, and so a
|
|
28
|
+
* probe that itself throws is reported as a fault rather than crashing the
|
|
29
|
+
* shift.
|
|
30
|
+
*/
|
|
31
|
+
import { accessSync, constants, statSync } from 'node:fs';
|
|
32
|
+
import { tmpdir } from 'node:os';
|
|
33
|
+
export const realFsProbe = {
|
|
34
|
+
exists: (path) => {
|
|
35
|
+
try {
|
|
36
|
+
statSync(path);
|
|
37
|
+
return true;
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
return false;
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
isDirectory: (path) => {
|
|
44
|
+
try {
|
|
45
|
+
return statSync(path).isDirectory();
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
},
|
|
51
|
+
isWritable: (path) => {
|
|
52
|
+
try {
|
|
53
|
+
accessSync(path, constants.W_OK);
|
|
54
|
+
return true;
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
},
|
|
60
|
+
};
|
|
61
|
+
const REMEDY = {
|
|
62
|
+
'temp-dir': 'the host\'s temp directory changed, which a reboot does — stop this shift and start a new one from a fresh shell so it inherits the current TMPDIR',
|
|
63
|
+
};
|
|
64
|
+
const LABEL = {
|
|
65
|
+
'temp-dir': 'TMPDIR',
|
|
66
|
+
};
|
|
67
|
+
function checkOne(role, path, fs) {
|
|
68
|
+
if (!fs.exists(path)) {
|
|
69
|
+
return { kind: 'missing', role, path, message: `${LABEL[role]} '${path}' no longer exists — ${REMEDY[role]}` };
|
|
70
|
+
}
|
|
71
|
+
if (!fs.isDirectory(path)) {
|
|
72
|
+
return {
|
|
73
|
+
kind: 'not-a-directory',
|
|
74
|
+
role,
|
|
75
|
+
path,
|
|
76
|
+
message: `${LABEL[role]} '${path}' is not a directory — ${REMEDY[role]}`,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
if (!fs.isWritable(path)) {
|
|
80
|
+
return {
|
|
81
|
+
kind: 'unwritable',
|
|
82
|
+
role,
|
|
83
|
+
path,
|
|
84
|
+
message: `${LABEL[role]} '${path}' is not writable by this process — ${REMEDY[role]}`,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
return undefined;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Report every local host fault, in a fixed role order so a reader comparing
|
|
91
|
+
* two records sees the same fault in the same position.
|
|
92
|
+
*
|
|
93
|
+
* The return is an array rather than a single fault so that adding a second
|
|
94
|
+
* role later does not change this function's contract, and so a caller never
|
|
95
|
+
* has to ask whether the one fault it was handed was the only one. An empty
|
|
96
|
+
* array means "no local fault" — which the caller must read as "the cause is
|
|
97
|
+
* elsewhere", never as "the shift is healthy".
|
|
98
|
+
*/
|
|
99
|
+
export function checkHost(paths, fs = realFsProbe) {
|
|
100
|
+
const candidates = [
|
|
101
|
+
['temp-dir', paths.tempDir ?? tmpdir()],
|
|
102
|
+
];
|
|
103
|
+
const faults = [];
|
|
104
|
+
for (const [role, path] of candidates) {
|
|
105
|
+
if (path === undefined || path === '')
|
|
106
|
+
continue;
|
|
107
|
+
const fault = checkOne(role, path, fs);
|
|
108
|
+
if (fault !== undefined)
|
|
109
|
+
faults.push(fault);
|
|
110
|
+
}
|
|
111
|
+
return faults;
|
|
112
|
+
}
|
|
@@ -4,6 +4,8 @@ import { type WorkOrder } from '../hub/types.ts';
|
|
|
4
4
|
import type { FetchedStep } from '../bundle/types.ts';
|
|
5
5
|
import { type ReconcileOptions, type Liveness } from './state.ts';
|
|
6
6
|
import { sweepWorkDirs as sweepWorkDirsImpl } from '../agent/workdir.ts';
|
|
7
|
+
import { type HostFault } from './host-preflight.ts';
|
|
8
|
+
import { type DiskSpace } from './disk-floor.ts';
|
|
7
9
|
import type { Spawner, WorkerFailure } from './spawn.ts';
|
|
8
10
|
import { type ShiftEvent } from './protocol.ts';
|
|
9
11
|
export interface ShiftLoopOptions {
|
|
@@ -104,6 +106,22 @@ export interface ShiftLoopOptions {
|
|
|
104
106
|
/** PHASE 4 — injected in tests so the reaper can be exercised without a
|
|
105
107
|
* filesystem. Defaults to `sweepWorkDirs` from `src/agent/workdir.ts`. */
|
|
106
108
|
sweepWorkDirs?: typeof sweepWorkDirsImpl;
|
|
109
|
+
/**
|
|
110
|
+
* Does this machine still have the directories the shift needs? Injected in
|
|
111
|
+
* tests so a vanished temp directory can be presented without one, and so the
|
|
112
|
+
* check itself can be made to throw. Defaults to `checkHost`, which is
|
|
113
|
+
* `stat`-only and issues no hub traffic. An empty array means "no LOCAL
|
|
114
|
+
* fault", never "healthy" — see `host-preflight.ts`.
|
|
115
|
+
*/
|
|
116
|
+
hostPreflight?: () => HostFault[];
|
|
117
|
+
/**
|
|
118
|
+
* Is there room on this disk to start another order? Injected in tests so a
|
|
119
|
+
* full volume can be presented without filling one. Defaults to a real
|
|
120
|
+
* `statfs` of `workRoot` (or its nearest existing ancestor) against the
|
|
121
|
+
* resolved floor. `unknown` means "could not measure, or switched off" and is
|
|
122
|
+
* treated as permission to dispatch — see `disk-floor.ts`.
|
|
123
|
+
*/
|
|
124
|
+
diskSpace?: () => DiskSpace;
|
|
107
125
|
}
|
|
108
126
|
export interface LockedRemovalOptions {
|
|
109
127
|
dispatchLockOptions?: Pick<AcquireFileLockOpts, 'beforeOpen'>;
|
|
@@ -63,6 +63,8 @@ import { cancelReservedChild, finalizeChildReservation, reconcileInFlight, reser
|
|
|
63
63
|
import { DEFAULT_WORK_DIR_TTL_MS, sweepWorkDirs as sweepWorkDirsImpl } from "../agent/workdir.js";
|
|
64
64
|
import { withHubRosterSyncTimeout } from "../settings/hub-roster-cache.js";
|
|
65
65
|
import { sessionsPath } from "../harness/session-store.js";
|
|
66
|
+
import { checkHost } from "./host-preflight.js";
|
|
67
|
+
import { formatBytes } from "./disk-floor.js";
|
|
66
68
|
import { stampShiftEvent, } from "./protocol.js";
|
|
67
69
|
/** Run a state mutation while holding the lock that serializes dispatch state. */
|
|
68
70
|
export function withDispatchLock(stateDir, options = {}, fn) {
|
|
@@ -196,12 +198,53 @@ function stepBrakeDelayMs(count) {
|
|
|
196
198
|
const index = Math.min(count, STEP_BRAKE_DELAYS_MS.length) - 1;
|
|
197
199
|
return STEP_BRAKE_DELAYS_MS[index] ?? 0;
|
|
198
200
|
}
|
|
201
|
+
/** How many links of an error's `cause` chain `errMsg` renders. */
|
|
202
|
+
const MAX_CAUSE_DEPTH = 4;
|
|
203
|
+
/**
|
|
204
|
+
* Consecutive failed hub polls before the shift asks whether the fault is local.
|
|
205
|
+
*
|
|
206
|
+
* NOT a timeout and not a health threshold — nothing is declared broken at this
|
|
207
|
+
* number. Crossing it only makes the shift run `checkHost`, which is a handful
|
|
208
|
+
* of `stat` calls and is free of hub traffic; if the host is fine the shift
|
|
209
|
+
* keeps polling and the streak keeps climbing. So the number only has to be
|
|
210
|
+
* large enough that a single transient failure does not pay for the check, and
|
|
211
|
+
* small enough that an operator is not left guessing for long. At the shipped
|
|
212
|
+
* 30s poll interval three failures is about a minute and a half.
|
|
213
|
+
*/
|
|
214
|
+
const HOST_PREFLIGHT_STREAK = 3;
|
|
199
215
|
/** The identity a storm repeats on. `key` is '' for an unfanned step. */
|
|
200
216
|
function stepBrakeKey(workflow, step, key) {
|
|
201
217
|
return `${workflow}\u0000${step}\u0000${key ?? ''}`;
|
|
202
218
|
}
|
|
219
|
+
/**
|
|
220
|
+
* An error's message, with its cause chain appended.
|
|
221
|
+
*
|
|
222
|
+
* THE CHAIN IS THE POINT. Node's fetch reports every transport failure as the
|
|
223
|
+
* bare string `fetch failed` and puts the real reason — `ENOENT`, `ECONNREFUSED`,
|
|
224
|
+
* `EAI_AGAIN`, a TLS error — on `cause`. Dropping it makes a dead local host, an
|
|
225
|
+
* unreachable hub and a DNS failure produce byte-identical records, which is
|
|
226
|
+
* precisely how a wedged shift stayed unattributed for 42 minutes.
|
|
227
|
+
*
|
|
228
|
+
* Bounded by construction: the walk follows at most `MAX_CAUSE_DEPTH` links and
|
|
229
|
+
* refuses to revisit a cause it has already rendered, so a self-referential or
|
|
230
|
+
* mutually-referential chain terminates instead of hanging the emitter.
|
|
231
|
+
*/
|
|
203
232
|
function errMsg(e) {
|
|
204
|
-
|
|
233
|
+
if (!(e instanceof Error))
|
|
234
|
+
return String(e);
|
|
235
|
+
const parts = [e.message];
|
|
236
|
+
const seen = new Set([e]);
|
|
237
|
+
let cause = e.cause;
|
|
238
|
+
for (let depth = 0; depth < MAX_CAUSE_DEPTH && cause !== undefined && cause !== null; depth += 1) {
|
|
239
|
+
if (seen.has(cause))
|
|
240
|
+
break;
|
|
241
|
+
seen.add(cause);
|
|
242
|
+
const text = cause instanceof Error ? cause.message : String(cause);
|
|
243
|
+
if (text !== '' && !parts.includes(text))
|
|
244
|
+
parts.push(text);
|
|
245
|
+
cause = cause instanceof Error ? cause.cause : undefined;
|
|
246
|
+
}
|
|
247
|
+
return parts.join(': ');
|
|
205
248
|
}
|
|
206
249
|
/** The explicit hub refusal produced when an inbox row becomes terminal after
|
|
207
250
|
* the inbox snapshot but before its targeted serving call. This is a normal
|
|
@@ -298,8 +341,34 @@ export function createShiftLoop(opts) {
|
|
|
298
341
|
* deliberately NOT edge-triggered.
|
|
299
342
|
*/
|
|
300
343
|
let atCapReported = false;
|
|
344
|
+
/**
|
|
345
|
+
* EDGE flag for the low-disk refusal, paired with `atCapReported` because the
|
|
346
|
+
* two gates have the same shape: a condition that persists across many ticks
|
|
347
|
+
* but is worth exactly one record per episode. Cleared the moment the disk
|
|
348
|
+
* measures healthy again, so a machine that fills and drains twice writes two
|
|
349
|
+
* records rather than one.
|
|
350
|
+
*/
|
|
351
|
+
let lowDiskReported = false;
|
|
301
352
|
/** Earliest server-approved time for the next polling iteration. */
|
|
302
353
|
let backoffUntil = Number.NEGATIVE_INFINITY;
|
|
354
|
+
/**
|
|
355
|
+
* Consecutive hub polls that failed for a reason the hub did not sanction.
|
|
356
|
+
*
|
|
357
|
+
* RATE LIMITING IS EXCLUDED, and that exclusion is the whole design. A 429 is
|
|
358
|
+
* the server saying "come back later", and a Cloudflare 1015 ban is the same
|
|
359
|
+
* answer at the edge: both are recoverable degradation that clears on its own,
|
|
360
|
+
* and a shift that killed itself over one would take the whole fleet down for
|
|
361
|
+
* a condition the fleet itself caused. `noteServerBackoff` already returns
|
|
362
|
+
* exactly that discrimination, so the streak counts only the failures nobody
|
|
363
|
+
* promised would end.
|
|
364
|
+
*
|
|
365
|
+
* Reset by any SUCCESSFUL wake, which is the loop's one unconditional hub call
|
|
366
|
+
* per unbackoffed tick and therefore the cheapest honest proof that the hub is
|
|
367
|
+
* reachable and this host can reach it.
|
|
368
|
+
*/
|
|
369
|
+
let hubFailureStreak = 0;
|
|
370
|
+
/** Set once the shift has stopped for a named local fault, so `run` exits non-zero. */
|
|
371
|
+
let wedged = false;
|
|
303
372
|
/** Bumped whenever a ping starts or is forced due, so a ping that completes
|
|
304
373
|
* after a force does not stamp the cadence over that force. */
|
|
305
374
|
let presenceGeneration = 0;
|
|
@@ -377,6 +446,58 @@ export function createShiftLoop(opts) {
|
|
|
377
446
|
backoffUntil = Math.max(backoffUntil, monotonicNow() + delay);
|
|
378
447
|
return true;
|
|
379
448
|
}
|
|
449
|
+
/**
|
|
450
|
+
* Ask whether a LOCAL fault is the reason this shift cannot work, and stop if
|
|
451
|
+
* one is. Returns true when the shift has been wedged.
|
|
452
|
+
*
|
|
453
|
+
* STOPPING IS THE POINT, and it is the opposite of what every other failure
|
|
454
|
+
* path here does. Everything else in this loop is built to survive: a failed
|
|
455
|
+
* wake retries next tick, a failed roster sync degrades and continues, a
|
|
456
|
+
* rate-limited poll backs off. All of that is correct for a REMOTE fault,
|
|
457
|
+
* which ends without anyone doing anything. It is exactly wrong for a local
|
|
458
|
+
* one, where continuing means an operator watching a live process that will
|
|
459
|
+
* never dispatch again — the measured shape of this defect was a shift that
|
|
460
|
+
* looked alive to `pgrep`, answered `shift end`, and did nothing for 42
|
|
461
|
+
* minutes while attributing a vanished temp directory to `fetch failed`.
|
|
462
|
+
*
|
|
463
|
+
* A PRE-FLIGHT THAT ITSELF FAILS IS NOT A FAULT. If the probe throws, this
|
|
464
|
+
* reports it and returns false, because "I could not determine whether the
|
|
465
|
+
* host is broken" is not evidence that it is, and killing a working shift over
|
|
466
|
+
* an unreadable `stat` would be a worse failure than the one being detected.
|
|
467
|
+
*/
|
|
468
|
+
/**
|
|
469
|
+
* Free space for the volume the shift's children will write into.
|
|
470
|
+
*
|
|
471
|
+
* Defaults to `unknown` — never a refusal — when no probe was injected and no
|
|
472
|
+
* work root was resolved: the floor and the work root are both RUNTIME
|
|
473
|
+
* configuration, and a loop constructed without them has no basis to decline
|
|
474
|
+
* anything. `runtime.ts` supplies the real closure.
|
|
475
|
+
*/
|
|
476
|
+
function diskSpace() {
|
|
477
|
+
if (opts.diskSpace !== undefined)
|
|
478
|
+
return opts.diskSpace();
|
|
479
|
+
return { state: 'unknown', path: opts.workRoot ?? '' };
|
|
480
|
+
}
|
|
481
|
+
function runHostPreflight(streak) {
|
|
482
|
+
let faults;
|
|
483
|
+
try {
|
|
484
|
+
faults = opts.hostPreflight !== undefined
|
|
485
|
+
? opts.hostPreflight()
|
|
486
|
+
: checkHost({});
|
|
487
|
+
}
|
|
488
|
+
catch (e) {
|
|
489
|
+
opts.err(`host pre-flight failed to run: ${errMsg(e)} (continuing)`);
|
|
490
|
+
return false;
|
|
491
|
+
}
|
|
492
|
+
if (faults.length === 0)
|
|
493
|
+
return false;
|
|
494
|
+
for (const fault of faults)
|
|
495
|
+
opts.err(`shift cannot continue: ${fault.message}`);
|
|
496
|
+
wedged = true;
|
|
497
|
+
stopped = true;
|
|
498
|
+
emit({ type: 'wedged', faults, streak });
|
|
499
|
+
return true;
|
|
500
|
+
}
|
|
380
501
|
/**
|
|
381
502
|
* Fan one observation to every consumer, stamped.
|
|
382
503
|
*
|
|
@@ -1322,11 +1443,24 @@ export function createShiftLoop(opts) {
|
|
|
1322
1443
|
changed = w.changed;
|
|
1323
1444
|
cursor = w.cursor;
|
|
1324
1445
|
wakeSucceeded = true;
|
|
1446
|
+
// The loop's one unconditional hub call per unbackoffed tick, so its
|
|
1447
|
+
// success is the cheapest honest proof that this host can reach the hub.
|
|
1448
|
+
hubFailureStreak = 0;
|
|
1325
1449
|
}
|
|
1326
1450
|
catch (e) {
|
|
1327
|
-
noteServerBackoff(e);
|
|
1451
|
+
const rateLimited = noteServerBackoff(e);
|
|
1328
1452
|
opts.err(`wake failed: ${errMsg(e)} (retrying next tick)`);
|
|
1329
1453
|
emit({ type: 'hub-error', op: 'wake', message: errMsg(e) });
|
|
1454
|
+
if (!rateLimited) {
|
|
1455
|
+
hubFailureStreak += 1;
|
|
1456
|
+
// Every HOST_PREFLIGHT_STREAK-th failure, not only the first crossing:
|
|
1457
|
+
// a work root can be unmounted at failure seven, and a shift that
|
|
1458
|
+
// checked once at failure three would never see it. The check is
|
|
1459
|
+
// `stat`-only and adds no hub traffic, so repeating it is nearly free.
|
|
1460
|
+
if (hubFailureStreak % HOST_PREFLIGHT_STREAK === 0 && runHostPreflight(hubFailureStreak)) {
|
|
1461
|
+
return dispatched;
|
|
1462
|
+
}
|
|
1463
|
+
}
|
|
1330
1464
|
}
|
|
1331
1465
|
// A changed cursor is not sufficient by itself: if the prior changed tick
|
|
1332
1466
|
// had no local capacity, or its sweep failed, the adopted cursor will be
|
|
@@ -1339,7 +1473,38 @@ export function createShiftLoop(opts) {
|
|
|
1339
1473
|
// can actually make the targeted decision.
|
|
1340
1474
|
const capacityCooldownDue = capacityCooldownDueAt !== undefined && monotonicNow() >= capacityCooldownDueAt;
|
|
1341
1475
|
const capacityBecameReady = capacityCooldownReady(k, agentRoom);
|
|
1342
|
-
|
|
1476
|
+
const wantSweep = wakeSucceeded && (changed || sweepOwed || brakeDue || capacityCooldownDue || capacityBecameReady) && k > 0;
|
|
1477
|
+
/*
|
|
1478
|
+
* Measured ONLY when a sweep would otherwise happen, so a parked or
|
|
1479
|
+
* saturated shift pays no syscall for a question it is not about to act on.
|
|
1480
|
+
* That placement is also what makes recovery automatic: refusing sets
|
|
1481
|
+
* `sweepOwed`, `sweepOwed` makes the next tick want to sweep, and wanting to
|
|
1482
|
+
* sweep re-measures the disk. No timer and no separate recovery path.
|
|
1483
|
+
*/
|
|
1484
|
+
const disk = wantSweep ? diskSpace() : undefined;
|
|
1485
|
+
if (wantSweep && disk?.state === 'low') {
|
|
1486
|
+
/*
|
|
1487
|
+
* REFUSE NEW WORK, DO NOT STOP. Unlike the vanished temp directory that
|
|
1488
|
+
* `runHostPreflight` exits for, a full disk clears on its own, so the
|
|
1489
|
+
* shift stays up and keeps polling. In-flight children are deliberately
|
|
1490
|
+
* left running: killing one frees little and destroys all of its work.
|
|
1491
|
+
*/
|
|
1492
|
+
sweepOwed = true;
|
|
1493
|
+
opts.out(`low disk: ${formatBytes(disk.freeBytes)} free at ${disk.path}, below the ${formatBytes(disk.floorBytes)} floor — ` +
|
|
1494
|
+
`deferring whats_next until space is free`);
|
|
1495
|
+
// EDGE-TRIGGERED, exactly as the at-capacity record below: one per
|
|
1496
|
+
// low-disk EPISODE. The console line above stays level-triggered because
|
|
1497
|
+
// a live tail is watched by someone who wants to see it is still stuck.
|
|
1498
|
+
if (!lowDiskReported) {
|
|
1499
|
+
lowDiskReported = true;
|
|
1500
|
+
emit({ type: 'low-disk', path: disk.path, freeBytes: disk.freeBytes, floorBytes: disk.floorBytes });
|
|
1501
|
+
}
|
|
1502
|
+
}
|
|
1503
|
+
else if (wantSweep) {
|
|
1504
|
+
if (lowDiskReported) {
|
|
1505
|
+
lowDiskReported = false;
|
|
1506
|
+
opts.out('disk space recovered — resuming dispatch');
|
|
1507
|
+
}
|
|
1343
1508
|
// Clear before awaiting: a new brake armed during sweep must survive.
|
|
1344
1509
|
if (brakeDue)
|
|
1345
1510
|
brakeSweepDueAt = undefined;
|
|
@@ -1394,21 +1559,28 @@ export function createShiftLoop(opts) {
|
|
|
1394
1559
|
}
|
|
1395
1560
|
async function run() {
|
|
1396
1561
|
ensureStateDir(opts.stateDir);
|
|
1562
|
+
// BEFORE THE FIRST HUB CALL. A shift started into a broken host — the usual
|
|
1563
|
+
// way being a supervisor that restarts it across a reboot with the old
|
|
1564
|
+
// environment — must say so at once. Making it wait for three failed polls
|
|
1565
|
+
// would buy nothing and spend the operator's attention on network theories.
|
|
1566
|
+
// `streak: 0` marks this record as the boot check rather than a poll one.
|
|
1567
|
+
if (runHostPreflight(0))
|
|
1568
|
+
return 1;
|
|
1397
1569
|
if (opts.once === true) {
|
|
1398
1570
|
await iteration();
|
|
1399
|
-
return 0;
|
|
1571
|
+
return wedged ? 1 : 0;
|
|
1400
1572
|
}
|
|
1401
1573
|
for (;;) {
|
|
1402
1574
|
if (stopped)
|
|
1403
|
-
return 0;
|
|
1575
|
+
return wedged ? 1 : 0;
|
|
1404
1576
|
await iteration();
|
|
1405
1577
|
if (stopped)
|
|
1406
|
-
return 0;
|
|
1578
|
+
return wedged ? 1 : 0;
|
|
1407
1579
|
const delay = Math.max(opts.pollIntervalMs, backoffUntil - monotonicNow());
|
|
1408
1580
|
await opts.sleep(delay);
|
|
1409
1581
|
// Re-check after the park so a stop during sleep makes no further hub call.
|
|
1410
1582
|
if (stopped)
|
|
1411
|
-
return 0;
|
|
1583
|
+
return wedged ? 1 : 0;
|
|
1412
1584
|
}
|
|
1413
1585
|
}
|
|
1414
1586
|
function stop() {
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
* workflow state; this protocol only carries local dispatch observations and
|
|
7
7
|
* attendance commands between the CLI and one daemon process.
|
|
8
8
|
*/
|
|
9
|
+
import type { HostFault } from './host-preflight.ts';
|
|
9
10
|
export declare const SHIFT_SOCKET_NAME = "shift.sock";
|
|
10
11
|
export declare const DEFAULT_NEXT_WAIT_MS = 90000;
|
|
11
12
|
export declare const MAX_REQUEST_LINE_BYTES: number;
|
|
@@ -119,6 +120,30 @@ export interface CapacityEvent {
|
|
|
119
120
|
inFlight: number;
|
|
120
121
|
cap: number;
|
|
121
122
|
}
|
|
123
|
+
/**
|
|
124
|
+
* The shift declined to start new work because the volume holding its work root
|
|
125
|
+
* is nearly full.
|
|
126
|
+
*
|
|
127
|
+
* EDGE-TRIGGERED, like `CapacityEvent`: one record per low-disk EPISODE, not one
|
|
128
|
+
* per tick. A shift that sits on a full disk for an hour writes this once, and
|
|
129
|
+
* the recovery is legible from the timestamp of whatever it dispatches next.
|
|
130
|
+
*
|
|
131
|
+
* FILE-ONLY, for the same reason as `CapacityEvent` and `ParkedEvent`: a parked
|
|
132
|
+
* `shift next` must not be woken by news that nothing happened. In the file it
|
|
133
|
+
* is the only thing separating "this shift had no work" from "this shift had
|
|
134
|
+
* work and could not safely take it".
|
|
135
|
+
*
|
|
136
|
+
* `freeBytes` and `floorBytes` are both carried so the record is self-contained.
|
|
137
|
+
* A reader must be able to tell an operator who set an unreachable floor from a
|
|
138
|
+
* genuinely full disk without knowing what the shift was configured with.
|
|
139
|
+
*/
|
|
140
|
+
export interface LowDiskEvent {
|
|
141
|
+
type: 'low-disk';
|
|
142
|
+
/** The measured directory: the work root, or its nearest existing ancestor. */
|
|
143
|
+
path: string;
|
|
144
|
+
freeBytes: number;
|
|
145
|
+
floorBytes: number;
|
|
146
|
+
}
|
|
122
147
|
/** A hub call failed. `workflow` is present for per-workflow whats_next and release calls. */
|
|
123
148
|
export interface HubErrorEvent {
|
|
124
149
|
type: 'hub-error';
|
|
@@ -146,6 +171,42 @@ export interface OrderDroppedEvent {
|
|
|
146
171
|
reason: 'malformed-digest' | 'malformed-worker' | 'unsupported-worker' | 'verification-failed' | 'metadata-unavailable' | 'agent-lane-closed' | 'dispatch-cap-full' | 'agent-cap-full' | 'claim-expired';
|
|
147
172
|
message: string;
|
|
148
173
|
}
|
|
174
|
+
/**
|
|
175
|
+
* The shift STOPPED because the machine underneath it is broken, and named the
|
|
176
|
+
* fault. The last record a wedged shift writes; the process exits non-zero
|
|
177
|
+
* immediately after.
|
|
178
|
+
*
|
|
179
|
+
* WHY THIS EXISTS AS ITS OWN RECORD, rather than one more `hub-error`. A shift
|
|
180
|
+
* whose host temp directory vanished — which a macOS reboot does to any process
|
|
181
|
+
* that survives it — keeps polling forever, and every hub call fails with
|
|
182
|
+
* undici's generic `fetch failed`. From the outside that is indistinguishable
|
|
183
|
+
* from a hub outage: `pgrep` shows the process, `shift end` answers, and the
|
|
184
|
+
* log fills with records that attribute a LOCAL fault to the network. This
|
|
185
|
+
* record is the one that says the cause is here, on this machine, and that no
|
|
186
|
+
* amount of waiting will fix it.
|
|
187
|
+
*
|
|
188
|
+
* `faults` is never empty. That invariant is what separates this record from
|
|
189
|
+
* every retryable failure: a hub outage, a rate-limit ban and an expired
|
|
190
|
+
* credential all self-heal, so the shift keeps working through them and emits
|
|
191
|
+
* nothing here. A record exists only when a human must change something on the
|
|
192
|
+
* host.
|
|
193
|
+
*
|
|
194
|
+
* `streak` is how many consecutive hub calls failed before the check ran, and
|
|
195
|
+
* is 0 for the pre-flight the shift runs at start, before any hub call. It is
|
|
196
|
+
* evidence, not a cause: the streak is only what PROMPTED the check, and the
|
|
197
|
+
* `faults` are why the shift stopped.
|
|
198
|
+
*
|
|
199
|
+
* ON THE SOCKET AS WELL AS IN THE FILE, unlike `parked` and `capacity`. A
|
|
200
|
+
* parked `owenloop shift next` must block until there is something to report,
|
|
201
|
+
* and "the shift you are waiting on is dead" is the most report-worthy thing
|
|
202
|
+
* that can happen to it — a client that stays parked on a socket whose server
|
|
203
|
+
* is about to close waits for work that will never come.
|
|
204
|
+
*/
|
|
205
|
+
export interface WedgedEvent {
|
|
206
|
+
type: 'wedged';
|
|
207
|
+
faults: HostFault[];
|
|
208
|
+
streak: number;
|
|
209
|
+
}
|
|
149
210
|
/**
|
|
150
211
|
* The socket event FIFO overflowed and discarded its oldest event. Written
|
|
151
212
|
* STRAIGHT to the file sink, never through the loop's `emit()` — `emit()` feeds
|
|
@@ -161,7 +222,7 @@ export interface EventQueueOverflowEvent {
|
|
|
161
222
|
* What a construction site builds: the event's own payload, with no identity
|
|
162
223
|
* and no timestamp. Every consumer receives the stamped `ShiftEvent` instead.
|
|
163
224
|
*/
|
|
164
|
-
export type ShiftEventBody = DispatchedEvent | ReapedEvent | FailedEvent | GateEvent | EndedEvent | ParkedEvent | CapacityEvent | HubErrorEvent | BundleMissEvent | OrderDroppedEvent | EventQueueOverflowEvent;
|
|
225
|
+
export type ShiftEventBody = DispatchedEvent | ReapedEvent | FailedEvent | GateEvent | EndedEvent | ParkedEvent | CapacityEvent | HubErrorEvent | BundleMissEvent | OrderDroppedEvent | EventQueueOverflowEvent | WedgedEvent | LowDiskEvent;
|
|
165
226
|
/**
|
|
166
227
|
* The identity every event carries, added once on a shared envelope rather than
|
|
167
228
|
* per variant.
|
|
@@ -35,6 +35,8 @@ export interface ParsedArgs {
|
|
|
35
35
|
once?: boolean;
|
|
36
36
|
maxAgents?: number;
|
|
37
37
|
execReserve?: number;
|
|
38
|
+
/** `--disk-floor <bytes>` — free space required to start new work. `0` disables. */
|
|
39
|
+
diskFloorBytes?: number;
|
|
38
40
|
localQueueHoldMs?: number;
|
|
39
41
|
cacheDir?: string;
|
|
40
42
|
stateDir?: string;
|
|
@@ -16,6 +16,7 @@ import { loadSettings } from "../settings/settings.js";
|
|
|
16
16
|
import { DEFAULT_HUB_ROSTER_SYNC_TIMEOUT_MS, syncHubRosterCache, withHubRosterSyncTimeout } from "../settings/hub-roster-cache.js";
|
|
17
17
|
import { computeServeCapabilities } from "../settings/serving.js";
|
|
18
18
|
import { resolveCacheDir } from "../bundle/cache.js";
|
|
19
|
+
import { checkDiskFloor, resolveDiskFloorBytes } from "./disk-floor.js";
|
|
19
20
|
import { createLockedRemovalCallbacks, createShiftLoop } from "./loop.js";
|
|
20
21
|
import { createHubBundleRecoveryHandler } from "../bundle/pull.js";
|
|
21
22
|
import { createShiftLogSink } from "./logsink.js";
|
|
@@ -93,6 +94,7 @@ const FILE_ONLY_EVENTS = new Set([
|
|
|
93
94
|
'capacity',
|
|
94
95
|
'hub-error',
|
|
95
96
|
'event-queue-overflow',
|
|
97
|
+
'low-disk',
|
|
96
98
|
]);
|
|
97
99
|
/**
|
|
98
100
|
* Does this event type reach the SOCKET consumer (a parked `owenloop shift
|
|
@@ -161,6 +163,7 @@ export function parseArgs(args) {
|
|
|
161
163
|
case '--cap':
|
|
162
164
|
case '--max-agents':
|
|
163
165
|
case '--exec-reserve':
|
|
166
|
+
case '--disk-floor':
|
|
164
167
|
case '--local-queue-hold':
|
|
165
168
|
case '--workflow':
|
|
166
169
|
case '--poll-interval':
|
|
@@ -217,6 +220,12 @@ export function parseArgs(args) {
|
|
|
217
220
|
return { error: n.error };
|
|
218
221
|
parsed.execReserve = n;
|
|
219
222
|
}
|
|
223
|
+
else if (name === '--disk-floor') {
|
|
224
|
+
const n = intFlag(r.value, '--disk-floor');
|
|
225
|
+
if (typeof n !== 'number')
|
|
226
|
+
return { error: n.error };
|
|
227
|
+
parsed.diskFloorBytes = n;
|
|
228
|
+
}
|
|
220
229
|
else if (name === '--local-queue-hold') {
|
|
221
230
|
const n = intFlag(r.value, '--local-queue-hold');
|
|
222
231
|
if (typeof n !== 'number')
|
|
@@ -404,6 +413,7 @@ export async function runShiftRuntime(parsed, options = {}) {
|
|
|
404
413
|
const maxConcurrentAgents = resolveMaxConcurrentAgents(parsed.maxAgents, settings.maxConcurrentAgents);
|
|
405
414
|
const execReserve = resolveExecReserve(parsed.execReserve, settings.execReserve);
|
|
406
415
|
const localQueueHoldMs = resolveLocalQueueHoldMs(parsed.localQueueHoldMs, settings.localQueueHoldMs);
|
|
416
|
+
const diskFloorBytes = resolveDiskFloorBytes(parsed.diskFloorBytes, settings.diskFloorBytes);
|
|
407
417
|
const workRoot = resolveWorkRoot(env, settings.workRoot, cacheDir);
|
|
408
418
|
const workRepo = resolveWorkRepo(env, settings.workRepo);
|
|
409
419
|
/**
|
|
@@ -588,6 +598,13 @@ export async function runShiftRuntime(parsed, options = {}) {
|
|
|
588
598
|
execReserve,
|
|
589
599
|
localQueueHoldMs,
|
|
590
600
|
workRoot,
|
|
601
|
+
/*
|
|
602
|
+
* Resolved here, not in the loop: `runtime.ts` owns flag/settings/default
|
|
603
|
+
* precedence for every other tunable, and the loop consumes a decision. A
|
|
604
|
+
* loop built without this closure has no disk gate at all, which is the
|
|
605
|
+
* right default for a caller that never told it where work lands.
|
|
606
|
+
*/
|
|
607
|
+
diskSpace: () => checkDiskFloor(workRoot, diskFloorBytes),
|
|
591
608
|
...(workRepo !== undefined ? { workRepo } : {}),
|
|
592
609
|
shiftId,
|
|
593
610
|
startedAt,
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Read a submit value from a JSON document contained in a run worktree.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* canonical check
|
|
4
|
+
* Containment is `resolveContainedPath`'s job — lexical traversal check first,
|
|
5
|
+
* then a canonical symlink check. This function owns only what is specific to
|
|
6
|
+
* a submit value: the argument shape and the JSON decode.
|
|
6
7
|
*/
|
|
7
8
|
export declare function readSubmitValueFile(workdir: string, valueFile: unknown): Promise<unknown>;
|