owenloop 0.5.26 → 0.5.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/packages/work/src/agent/brief.d.ts +10 -0
- package/dist/packages/work/src/agent/brief.js +38 -0
- package/dist/packages/work/src/agent/loop.js +4 -0
- package/dist/packages/work/src/contained-path.d.ts +18 -0
- package/dist/packages/work/src/contained-path.js +42 -0
- package/dist/packages/work/src/harness/claude.js +21 -0
- package/dist/packages/work/src/harness/codex.js +29 -0
- package/dist/packages/work/src/harness/contract.d.ts +26 -0
- package/dist/packages/work/src/hold/mcp.d.ts +3 -40
- package/dist/packages/work/src/hold/mcp.js +121 -2
- package/dist/packages/work/src/hub/client.d.ts +10 -1
- package/dist/packages/work/src/hub/client.js +23 -0
- package/dist/packages/work/src/hub/types.d.ts +31 -0
- package/dist/packages/work/src/roles/shift.js +7 -1
- package/dist/packages/work/src/settings/settings.d.ts +12 -1
- package/dist/packages/work/src/settings/settings.js +7 -0
- package/dist/packages/work/src/shift/disk-floor.d.ts +107 -0
- package/dist/packages/work/src/shift/disk-floor.js +148 -0
- package/dist/packages/work/src/shift/host-preflight.d.ts +85 -0
- package/dist/packages/work/src/shift/host-preflight.js +112 -0
- package/dist/packages/work/src/shift/loop.d.ts +18 -0
- package/dist/packages/work/src/shift/loop.js +179 -7
- package/dist/packages/work/src/shift/protocol.d.ts +62 -1
- package/dist/packages/work/src/shift/runtime.d.ts +2 -0
- package/dist/packages/work/src/shift/runtime.js +17 -0
- package/dist/packages/work/src/submit-file.d.ts +3 -2
- package/dist/packages/work/src/submit-file.js +6 -22
- package/dist/src/defs.js +59 -8
- package/dist/src/types.d.ts +13 -0
- package/docs/cli.md +8 -5
- package/docs/shift-logs.md +64 -7
- package/package.json +1 -1
- package/plugins/claude-code/plugin/.claude-plugin/plugin.json +1 -1
- package/plugins/claude-code/plugin/.mcp.json +1 -1
- package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +1 -1
- package/plugins/codex/plugins/owenloop/.mcp.json +1 -1
|
@@ -17,6 +17,7 @@ import { type Roster } from '../agent/capability-model.ts';
|
|
|
17
17
|
* and `OWENLOOP_SHIFT_LOG_MAX_AGE_MS`, above the 14-day default.
|
|
18
18
|
* - `commandRouting` — who runs `executor: 'command'` steps this machine sees.
|
|
19
19
|
* - `execReserve` — slots within dispatch cap reserved from agent work.
|
|
20
|
+
* - `diskFloorBytes` — free bytes required before starting new work.
|
|
20
21
|
* - `defPolicy` — local publication trust policy (`warn` by default).
|
|
21
22
|
*/
|
|
22
23
|
export interface Settings {
|
|
@@ -78,6 +79,16 @@ export interface Settings {
|
|
|
78
79
|
* `--exec-reserve`; default 1.
|
|
79
80
|
*/
|
|
80
81
|
execReserve?: number;
|
|
82
|
+
/**
|
|
83
|
+
* Free bytes the shift insists on, on the volume holding `workRoot`, before
|
|
84
|
+
* it will START another order. A full disk kills children mid-step with a
|
|
85
|
+
* bare ENOSPC, so the shift declines new work rather than feeding orders to a
|
|
86
|
+
* disk that cannot hold them; it keeps polling and resumes by itself when
|
|
87
|
+
* space returns. `0` disables the check. Fallback below `--disk-floor`;
|
|
88
|
+
* default 1 GiB. This is a floor for STARTING work, not a promise that any
|
|
89
|
+
* particular step will fit.
|
|
90
|
+
*/
|
|
91
|
+
diskFloorBytes?: number;
|
|
81
92
|
/**
|
|
82
93
|
* How long a shift may retain an undispatchable claim locally before
|
|
83
94
|
* returning it to the hub. Default 0: never retain it locally.
|
|
@@ -139,7 +150,7 @@ export interface Settings {
|
|
|
139
150
|
roster?: Roster;
|
|
140
151
|
}
|
|
141
152
|
/** The known settings keys, in the order `owenloop work settings` prints them. */
|
|
142
|
-
export declare const KNOWN_SETTINGS_KEYS: readonly ["hubOrigin", "cacheDir", "stateDir", "dispatchCap", "shiftLogDir", "shiftLogMaxAgeMs", "commandRouting", "defPolicy", "artifactPolicy", "maxConcurrentAgents", "execReserve", "localQueueHoldMs", "workRoot", "workRepo", "allowedWorkdirRoots", "roster"];
|
|
153
|
+
export declare const KNOWN_SETTINGS_KEYS: readonly ["hubOrigin", "cacheDir", "stateDir", "dispatchCap", "shiftLogDir", "shiftLogMaxAgeMs", "commandRouting", "defPolicy", "artifactPolicy", "maxConcurrentAgents", "execReserve", "diskFloorBytes", "localQueueHoldMs", "workRoot", "workRepo", "allowedWorkdirRoots", "roster"];
|
|
143
154
|
/**
|
|
144
155
|
* Keys owenloop still RECOGNIZES but no longer acts on. Each one produces a
|
|
145
156
|
* warning naming what replaced it, and is excluded from `unrecognized` so the
|
|
@@ -29,6 +29,7 @@ export const KNOWN_SETTINGS_KEYS = [
|
|
|
29
29
|
'artifactPolicy',
|
|
30
30
|
'maxConcurrentAgents',
|
|
31
31
|
'execReserve',
|
|
32
|
+
'diskFloorBytes',
|
|
32
33
|
'localQueueHoldMs',
|
|
33
34
|
'workRoot',
|
|
34
35
|
'workRepo',
|
|
@@ -104,6 +105,12 @@ export function validateSettings(raw, path) {
|
|
|
104
105
|
throw bad('execReserve', 'a non-negative integer', v);
|
|
105
106
|
}
|
|
106
107
|
}
|
|
108
|
+
if ('diskFloorBytes' in obj) {
|
|
109
|
+
const v = obj['diskFloorBytes'];
|
|
110
|
+
if (typeof v !== 'number' || !Number.isInteger(v) || v < 0) {
|
|
111
|
+
throw bad('diskFloorBytes', 'a non-negative integer', v);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
107
114
|
if ('localQueueHoldMs' in obj) {
|
|
108
115
|
const v = obj['localQueueHoldMs'];
|
|
109
116
|
if (typeof v !== 'number' || !Number.isInteger(v) || v < 0) {
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Is there enough room on this disk to START another piece of work?
|
|
3
|
+
*
|
|
4
|
+
* A shift's children are the heaviest writers on the machine: a clone, a
|
|
5
|
+
* dependency install, a build tree, and a transcript per agent turn. When the
|
|
6
|
+
* volume holding the work root fills, those children do not fail politely. The
|
|
7
|
+
* incident this exists for is recorded in `loop.ts`'s `noteWorkerFailure`
|
|
8
|
+
* docstring: a worker "exited 1 with no signal, killed by ENOSPC inside its own
|
|
9
|
+
* error logging" — the process died while trying to say why it was dying, so
|
|
10
|
+
* the shift saw a bare non-zero exit and re-offered the order onto the same
|
|
11
|
+
* full disk, forever.
|
|
12
|
+
*
|
|
13
|
+
* WHY THIS REFUSES WORK INSTEAD OF STOPPING THE SHIFT. A full disk is a LOCAL
|
|
14
|
+
* fault, like the vanished `TMPDIR` that `host-preflight.ts` answers for, but
|
|
15
|
+
* unlike it a full disk SELF-HEALS: a large file gets deleted, a cache is
|
|
16
|
+
* pruned, another process finishes and releases its scratch space. Exiting the
|
|
17
|
+
* shift would turn a condition that commonly clears in minutes into an outage
|
|
18
|
+
* that needs a human to notice and restart a daemon. So this gate does the one
|
|
19
|
+
* thing that is both safe and reversible — it declines to take on NEW work,
|
|
20
|
+
* keeps polling, and starts dispatching again by itself the moment space
|
|
21
|
+
* returns. Work already in flight is left alone: killing a nearly-finished
|
|
22
|
+
* child frees little and destroys everything it had done.
|
|
23
|
+
*
|
|
24
|
+
* WHAT IT DOES NOT PROMISE. This is a floor for STARTING work, not a guarantee
|
|
25
|
+
* that a step will finish. Nothing here knows how much space the next order
|
|
26
|
+
* needs, and a step that wants 40 GiB will still die on a disk holding 2. The
|
|
27
|
+
* value of the gate is not that it prevents every ENOSPC; it is that when the
|
|
28
|
+
* disk is the reason nothing is moving, a record says so by name instead of the
|
|
29
|
+
* operator reading a wall of unexplained exit-1s.
|
|
30
|
+
*
|
|
31
|
+
* FAIL-OPEN, DELIBERATELY. If the free-space probe itself fails — an
|
|
32
|
+
* unsupported filesystem, a platform without `statfs`, a path that races away
|
|
33
|
+
* between the walk and the call — this reports `unknown` and the caller
|
|
34
|
+
* dispatches. A disk check that refused work whenever it could not measure the
|
|
35
|
+
* disk would convert its own blind spot into the total outage it exists to
|
|
36
|
+
* prevent, and it would do so on exactly the exotic hosts least likely to be
|
|
37
|
+
* debugged quickly. The measured failure mode is a full disk, not an
|
|
38
|
+
* unmeasurable one.
|
|
39
|
+
*/
|
|
40
|
+
/**
|
|
41
|
+
* How much room the shift insists on before starting anything new.
|
|
42
|
+
*
|
|
43
|
+
* Chosen as a DISTRESS threshold rather than a sufficiency threshold. Below
|
|
44
|
+
* roughly a gibibyte a general-purpose developer machine cannot complete a
|
|
45
|
+
* dependency install or a build of any size, so nearly every order a shift
|
|
46
|
+
* takes will die; above it, whether an order succeeds depends on the order, and
|
|
47
|
+
* this module has no business guessing. The error to avoid is a floor set high
|
|
48
|
+
* enough to refuse work that would have succeeded, because a shift that will
|
|
49
|
+
* not take work is worse than a shift that occasionally loses one to ENOSPC —
|
|
50
|
+
* which is why the default is low and the operator can lower it further.
|
|
51
|
+
*/
|
|
52
|
+
export declare const DEFAULT_DISK_FLOOR_BYTES: number;
|
|
53
|
+
/**
|
|
54
|
+
* `--disk-floor` > `settings.diskFloorBytes` > the default, matching every
|
|
55
|
+
* other shift tunable. `0` disables the gate: an operator who is deliberately
|
|
56
|
+
* running a machine close to full, or who has hit a floor this module got
|
|
57
|
+
* wrong, must be able to say so without downgrading the CLI.
|
|
58
|
+
*/
|
|
59
|
+
export declare function resolveDiskFloorBytes(flagFloor: number | undefined, settingsFloor: number | undefined): number;
|
|
60
|
+
/** The one filesystem call this module makes, as an injectable seam. */
|
|
61
|
+
export interface StatfsProbe {
|
|
62
|
+
/** Bytes available to an unprivileged writer, or `undefined` if unmeasurable. */
|
|
63
|
+
freeBytes: (path: string) => number | undefined;
|
|
64
|
+
exists: (path: string) => boolean;
|
|
65
|
+
}
|
|
66
|
+
export declare const realStatfsProbe: StatfsProbe;
|
|
67
|
+
/**
|
|
68
|
+
* The nearest ancestor of `path` that exists, so free space can be measured for
|
|
69
|
+
* a directory the shift has not created yet.
|
|
70
|
+
*
|
|
71
|
+
* A work root is normally absent on a first run — `resolveWorkRoot` defaults it
|
|
72
|
+
* to `<cacheDir>/work` and `prepareWorkdir` mkdir -p's it on demand — so a
|
|
73
|
+
* check that required the path to exist would report `unknown` on every clean
|
|
74
|
+
* install and never fire when it mattered. This is the same premise that an
|
|
75
|
+
* earlier draft of `host-preflight.ts` got wrong; the difference is that a
|
|
76
|
+
* missing directory is not a fault here, merely a reason to ask its parent.
|
|
77
|
+
*
|
|
78
|
+
* Bounded by the walk reaching the filesystem root, where `dirname` becomes a
|
|
79
|
+
* fixed point, so a path that exists nowhere terminates rather than looping.
|
|
80
|
+
*/
|
|
81
|
+
export declare function nearestExistingAncestor(path: string, fs: StatfsProbe): string | undefined;
|
|
82
|
+
export type DiskSpace = {
|
|
83
|
+
state: 'ok';
|
|
84
|
+
path: string;
|
|
85
|
+
freeBytes: number;
|
|
86
|
+
floorBytes: number;
|
|
87
|
+
} | {
|
|
88
|
+
state: 'low';
|
|
89
|
+
path: string;
|
|
90
|
+
freeBytes: number;
|
|
91
|
+
floorBytes: number;
|
|
92
|
+
}
|
|
93
|
+
/** Unmeasurable, or the gate is switched off. The caller dispatches. */
|
|
94
|
+
| {
|
|
95
|
+
state: 'unknown';
|
|
96
|
+
path: string;
|
|
97
|
+
};
|
|
98
|
+
/** Render a byte count the way an operator reading a log line wants it. */
|
|
99
|
+
export declare function formatBytes(bytes: number): string;
|
|
100
|
+
/**
|
|
101
|
+
* Measure free space for the volume that will hold the shift's work.
|
|
102
|
+
*
|
|
103
|
+
* Returns `unknown` — never `low` — when the floor is disabled or the disk
|
|
104
|
+
* cannot be measured, so that a caller written as `if (state === 'low')` treats
|
|
105
|
+
* every uncertainty as permission to proceed.
|
|
106
|
+
*/
|
|
107
|
+
export declare function checkDiskFloor(workRoot: string, floorBytes: number, fs?: StatfsProbe): DiskSpace;
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Is there enough room on this disk to START another piece of work?
|
|
3
|
+
*
|
|
4
|
+
* A shift's children are the heaviest writers on the machine: a clone, a
|
|
5
|
+
* dependency install, a build tree, and a transcript per agent turn. When the
|
|
6
|
+
* volume holding the work root fills, those children do not fail politely. The
|
|
7
|
+
* incident this exists for is recorded in `loop.ts`'s `noteWorkerFailure`
|
|
8
|
+
* docstring: a worker "exited 1 with no signal, killed by ENOSPC inside its own
|
|
9
|
+
* error logging" — the process died while trying to say why it was dying, so
|
|
10
|
+
* the shift saw a bare non-zero exit and re-offered the order onto the same
|
|
11
|
+
* full disk, forever.
|
|
12
|
+
*
|
|
13
|
+
* WHY THIS REFUSES WORK INSTEAD OF STOPPING THE SHIFT. A full disk is a LOCAL
|
|
14
|
+
* fault, like the vanished `TMPDIR` that `host-preflight.ts` answers for, but
|
|
15
|
+
* unlike it a full disk SELF-HEALS: a large file gets deleted, a cache is
|
|
16
|
+
* pruned, another process finishes and releases its scratch space. Exiting the
|
|
17
|
+
* shift would turn a condition that commonly clears in minutes into an outage
|
|
18
|
+
* that needs a human to notice and restart a daemon. So this gate does the one
|
|
19
|
+
* thing that is both safe and reversible — it declines to take on NEW work,
|
|
20
|
+
* keeps polling, and starts dispatching again by itself the moment space
|
|
21
|
+
* returns. Work already in flight is left alone: killing a nearly-finished
|
|
22
|
+
* child frees little and destroys everything it had done.
|
|
23
|
+
*
|
|
24
|
+
* WHAT IT DOES NOT PROMISE. This is a floor for STARTING work, not a guarantee
|
|
25
|
+
* that a step will finish. Nothing here knows how much space the next order
|
|
26
|
+
* needs, and a step that wants 40 GiB will still die on a disk holding 2. The
|
|
27
|
+
* value of the gate is not that it prevents every ENOSPC; it is that when the
|
|
28
|
+
* disk is the reason nothing is moving, a record says so by name instead of the
|
|
29
|
+
* operator reading a wall of unexplained exit-1s.
|
|
30
|
+
*
|
|
31
|
+
* FAIL-OPEN, DELIBERATELY. If the free-space probe itself fails — an
|
|
32
|
+
* unsupported filesystem, a platform without `statfs`, a path that races away
|
|
33
|
+
* between the walk and the call — this reports `unknown` and the caller
|
|
34
|
+
* dispatches. A disk check that refused work whenever it could not measure the
|
|
35
|
+
* disk would convert its own blind spot into the total outage it exists to
|
|
36
|
+
* prevent, and it would do so on exactly the exotic hosts least likely to be
|
|
37
|
+
* debugged quickly. The measured failure mode is a full disk, not an
|
|
38
|
+
* unmeasurable one.
|
|
39
|
+
*/
|
|
40
|
+
import { statfsSync } from 'node:fs';
|
|
41
|
+
import { dirname, resolve } from 'node:path';
|
|
42
|
+
/** One gibibyte, the default floor. */
|
|
43
|
+
const GIB = 1024 * 1024 * 1024;
|
|
44
|
+
/**
|
|
45
|
+
* How much room the shift insists on before starting anything new.
|
|
46
|
+
*
|
|
47
|
+
* Chosen as a DISTRESS threshold rather than a sufficiency threshold. Below
|
|
48
|
+
* roughly a gibibyte a general-purpose developer machine cannot complete a
|
|
49
|
+
* dependency install or a build of any size, so nearly every order a shift
|
|
50
|
+
* takes will die; above it, whether an order succeeds depends on the order, and
|
|
51
|
+
* this module has no business guessing. The error to avoid is a floor set high
|
|
52
|
+
* enough to refuse work that would have succeeded, because a shift that will
|
|
53
|
+
* not take work is worse than a shift that occasionally loses one to ENOSPC —
|
|
54
|
+
* which is why the default is low and the operator can lower it further.
|
|
55
|
+
*/
|
|
56
|
+
export const DEFAULT_DISK_FLOOR_BYTES = GIB;
|
|
57
|
+
/**
|
|
58
|
+
* `--disk-floor` > `settings.diskFloorBytes` > the default, matching every
|
|
59
|
+
* other shift tunable. `0` disables the gate: an operator who is deliberately
|
|
60
|
+
* running a machine close to full, or who has hit a floor this module got
|
|
61
|
+
* wrong, must be able to say so without downgrading the CLI.
|
|
62
|
+
*/
|
|
63
|
+
export function resolveDiskFloorBytes(flagFloor, settingsFloor) {
|
|
64
|
+
return flagFloor ?? settingsFloor ?? DEFAULT_DISK_FLOOR_BYTES;
|
|
65
|
+
}
|
|
66
|
+
export const realStatfsProbe = {
|
|
67
|
+
freeBytes: (path) => {
|
|
68
|
+
try {
|
|
69
|
+
const fs = statfsSync(path);
|
|
70
|
+
/*
|
|
71
|
+
* `bavail`, NOT `bfree`. `bfree` counts blocks the filesystem has reserved
|
|
72
|
+
* for root, which this process cannot write into — reading it would let
|
|
73
|
+
* the gate report several hundred megabytes of headroom that no child can
|
|
74
|
+
* actually use, on precisely the full disk the gate exists to catch.
|
|
75
|
+
*/
|
|
76
|
+
const free = Number(fs.bavail) * Number(fs.bsize);
|
|
77
|
+
return Number.isFinite(free) && free >= 0 ? free : undefined;
|
|
78
|
+
}
|
|
79
|
+
catch {
|
|
80
|
+
return undefined;
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
exists: (path) => {
|
|
84
|
+
try {
|
|
85
|
+
statfsSync(path);
|
|
86
|
+
return true;
|
|
87
|
+
}
|
|
88
|
+
catch {
|
|
89
|
+
return false;
|
|
90
|
+
}
|
|
91
|
+
},
|
|
92
|
+
};
|
|
93
|
+
/**
|
|
94
|
+
* The nearest ancestor of `path` that exists, so free space can be measured for
|
|
95
|
+
* a directory the shift has not created yet.
|
|
96
|
+
*
|
|
97
|
+
* A work root is normally absent on a first run — `resolveWorkRoot` defaults it
|
|
98
|
+
* to `<cacheDir>/work` and `prepareWorkdir` mkdir -p's it on demand — so a
|
|
99
|
+
* check that required the path to exist would report `unknown` on every clean
|
|
100
|
+
* install and never fire when it mattered. This is the same premise that an
|
|
101
|
+
* earlier draft of `host-preflight.ts` got wrong; the difference is that a
|
|
102
|
+
* missing directory is not a fault here, merely a reason to ask its parent.
|
|
103
|
+
*
|
|
104
|
+
* Bounded by the walk reaching the filesystem root, where `dirname` becomes a
|
|
105
|
+
* fixed point, so a path that exists nowhere terminates rather than looping.
|
|
106
|
+
*/
|
|
107
|
+
export function nearestExistingAncestor(path, fs) {
|
|
108
|
+
let current = resolve(path);
|
|
109
|
+
for (;;) {
|
|
110
|
+
if (fs.exists(current))
|
|
111
|
+
return current;
|
|
112
|
+
const parent = dirname(current);
|
|
113
|
+
if (parent === current)
|
|
114
|
+
return undefined;
|
|
115
|
+
current = parent;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
/** Render a byte count the way an operator reading a log line wants it. */
|
|
119
|
+
export function formatBytes(bytes) {
|
|
120
|
+
if (bytes >= GIB)
|
|
121
|
+
return `${(bytes / GIB).toFixed(1)} GiB`;
|
|
122
|
+
if (bytes >= 1024 * 1024)
|
|
123
|
+
return `${Math.round(bytes / (1024 * 1024))} MiB`;
|
|
124
|
+
return `${bytes} B`;
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Measure free space for the volume that will hold the shift's work.
|
|
128
|
+
*
|
|
129
|
+
* Returns `unknown` — never `low` — when the floor is disabled or the disk
|
|
130
|
+
* cannot be measured, so that a caller written as `if (state === 'low')` treats
|
|
131
|
+
* every uncertainty as permission to proceed.
|
|
132
|
+
*/
|
|
133
|
+
export function checkDiskFloor(workRoot, floorBytes, fs = realStatfsProbe) {
|
|
134
|
+
if (floorBytes <= 0)
|
|
135
|
+
return { state: 'unknown', path: workRoot };
|
|
136
|
+
const probed = nearestExistingAncestor(workRoot, fs);
|
|
137
|
+
if (probed === undefined)
|
|
138
|
+
return { state: 'unknown', path: workRoot };
|
|
139
|
+
const free = fs.freeBytes(probed);
|
|
140
|
+
if (free === undefined)
|
|
141
|
+
return { state: 'unknown', path: probed };
|
|
142
|
+
return {
|
|
143
|
+
state: free < floorBytes ? 'low' : 'ok',
|
|
144
|
+
path: probed,
|
|
145
|
+
freeBytes: free,
|
|
146
|
+
floorBytes,
|
|
147
|
+
};
|
|
148
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Is this machine still the machine the shift started on?
|
|
3
|
+
*
|
|
4
|
+
* A shift daemon is a long-lived process holding paths it resolved once, at
|
|
5
|
+
* start, and one of those paths is not stable for the life of the process:
|
|
6
|
+
* macOS gives each boot a fresh per-user temp directory
|
|
7
|
+
* (`/var/folders/<a>/<b>/T`). A process that survives a reboot — or was started
|
|
8
|
+
* before one and re-parented — keeps the OLD `TMPDIR` in its environment, and
|
|
9
|
+
* that directory is gone.
|
|
10
|
+
*
|
|
11
|
+
* The fault does not announce itself. The shift keeps polling, every hub call
|
|
12
|
+
* and every child spawn fails for a reason the error text attributes to the
|
|
13
|
+
* network ("fetch failed"), and the daemon looks alive to `pgrep` and answers
|
|
14
|
+
* `shift end` while dispatching nothing. It was measured wedged this way for 42
|
|
15
|
+
* minutes with no record naming the cause.
|
|
16
|
+
*
|
|
17
|
+
* WHAT THIS MODULE IS FOR, AND WHAT IT IS NOT FOR. It answers exactly one
|
|
18
|
+
* question: "is a LOCAL, non-self-healing fault the reason this shift cannot
|
|
19
|
+
* work?" It is deliberately not a health check and not a reachability probe. A
|
|
20
|
+
* hub outage, a Cloudflare 1015 ban, an expired credential and a flaky DHCP
|
|
21
|
+
* lease are all REMOTE and all self-heal without operator action, so this
|
|
22
|
+
* module must report them clean — the caller's whole decision rests on that
|
|
23
|
+
* split, because a clean answer means "keep polling" and a dirty one means
|
|
24
|
+
* "stop, a human must fix the host".
|
|
25
|
+
*
|
|
26
|
+
* PURE AND INJECTABLE. Every filesystem call is taken through `FsProbe` so the
|
|
27
|
+
* caller's tests can present a missing temp directory without one, and so a
|
|
28
|
+
* probe that itself throws is reported as a fault rather than crashing the
|
|
29
|
+
* shift.
|
|
30
|
+
*/
|
|
31
|
+
/** Which of the shift's own directories a fault was found in. */
|
|
32
|
+
export type HostPathRole = 'temp-dir';
|
|
33
|
+
export interface HostFault {
|
|
34
|
+
/**
|
|
35
|
+
* The stable machine discriminator. The three kinds are separate because
|
|
36
|
+
* they have different operator fixes: a missing temp directory means the host
|
|
37
|
+
* rebooted and the shift must be restarted from a fresh shell, while an
|
|
38
|
+
* unwritable one means a permission change that restarting will not repair.
|
|
39
|
+
*/
|
|
40
|
+
kind: 'missing' | 'not-a-directory' | 'unwritable';
|
|
41
|
+
role: HostPathRole;
|
|
42
|
+
path: string;
|
|
43
|
+
/** Operator-facing text: what is wrong, why it happened, and what to do. */
|
|
44
|
+
message: string;
|
|
45
|
+
}
|
|
46
|
+
/** The filesystem calls this check makes, as one injectable seam. */
|
|
47
|
+
export interface FsProbe {
|
|
48
|
+
isDirectory: (path: string) => boolean;
|
|
49
|
+
exists: (path: string) => boolean;
|
|
50
|
+
isWritable: (path: string) => boolean;
|
|
51
|
+
}
|
|
52
|
+
export declare const realFsProbe: FsProbe;
|
|
53
|
+
/**
|
|
54
|
+
* The path checked, and DELIBERATELY only this one.
|
|
55
|
+
*
|
|
56
|
+
* A shift also holds a state directory, a bundle cache directory and a work
|
|
57
|
+
* root, and none of them is here: the shift CREATES all three on demand
|
|
58
|
+
* (`resolveWorkRoot` defaults the work root to `<cacheDir>/work`, and
|
|
59
|
+
* `prepareWorkdir` mkdir -p's it), so their absence is a normal first-run
|
|
60
|
+
* condition rather than a fault. A check that treated it as one would refuse to
|
|
61
|
+
* start every fresh install — which is exactly what an earlier draft of this
|
|
62
|
+
* module did, and what the suite caught. The temp directory is the one path the
|
|
63
|
+
* shift depends on, does not create, and cannot recover on its own.
|
|
64
|
+
*
|
|
65
|
+
* The operator's `--work-root` list is a different thing again: it NARROWS
|
|
66
|
+
* which roots an order may name rather than supplying a directory the shift
|
|
67
|
+
* writes to, and losing one of several does not stop the shift serving orders
|
|
68
|
+
* that name the others. If it is ever worth checking, it is worth checking as
|
|
69
|
+
* its own decision, not folded in here.
|
|
70
|
+
*/
|
|
71
|
+
export interface HostPaths {
|
|
72
|
+
/** Absent means "read the live `TMPDIR`", which is what the shift itself uses. */
|
|
73
|
+
tempDir?: string;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Report every local host fault, in a fixed role order so a reader comparing
|
|
77
|
+
* two records sees the same fault in the same position.
|
|
78
|
+
*
|
|
79
|
+
* The return is an array rather than a single fault so that adding a second
|
|
80
|
+
* role later does not change this function's contract, and so a caller never
|
|
81
|
+
* has to ask whether the one fault it was handed was the only one. An empty
|
|
82
|
+
* array means "no local fault" — which the caller must read as "the cause is
|
|
83
|
+
* elsewhere", never as "the shift is healthy".
|
|
84
|
+
*/
|
|
85
|
+
export declare function checkHost(paths: HostPaths, fs?: FsProbe): HostFault[];
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Is this machine still the machine the shift started on?
|
|
3
|
+
*
|
|
4
|
+
* A shift daemon is a long-lived process holding paths it resolved once, at
|
|
5
|
+
* start, and one of those paths is not stable for the life of the process:
|
|
6
|
+
* macOS gives each boot a fresh per-user temp directory
|
|
7
|
+
* (`/var/folders/<a>/<b>/T`). A process that survives a reboot — or was started
|
|
8
|
+
* before one and re-parented — keeps the OLD `TMPDIR` in its environment, and
|
|
9
|
+
* that directory is gone.
|
|
10
|
+
*
|
|
11
|
+
* The fault does not announce itself. The shift keeps polling, every hub call
|
|
12
|
+
* and every child spawn fails for a reason the error text attributes to the
|
|
13
|
+
* network ("fetch failed"), and the daemon looks alive to `pgrep` and answers
|
|
14
|
+
* `shift end` while dispatching nothing. It was measured wedged this way for 42
|
|
15
|
+
* minutes with no record naming the cause.
|
|
16
|
+
*
|
|
17
|
+
* WHAT THIS MODULE IS FOR, AND WHAT IT IS NOT FOR. It answers exactly one
|
|
18
|
+
* question: "is a LOCAL, non-self-healing fault the reason this shift cannot
|
|
19
|
+
* work?" It is deliberately not a health check and not a reachability probe. A
|
|
20
|
+
* hub outage, a Cloudflare 1015 ban, an expired credential and a flaky DHCP
|
|
21
|
+
* lease are all REMOTE and all self-heal without operator action, so this
|
|
22
|
+
* module must report them clean — the caller's whole decision rests on that
|
|
23
|
+
* split, because a clean answer means "keep polling" and a dirty one means
|
|
24
|
+
* "stop, a human must fix the host".
|
|
25
|
+
*
|
|
26
|
+
* PURE AND INJECTABLE. Every filesystem call is taken through `FsProbe` so the
|
|
27
|
+
* caller's tests can present a missing temp directory without one, and so a
|
|
28
|
+
* probe that itself throws is reported as a fault rather than crashing the
|
|
29
|
+
* shift.
|
|
30
|
+
*/
|
|
31
|
+
import { accessSync, constants, statSync } from 'node:fs';
|
|
32
|
+
import { tmpdir } from 'node:os';
|
|
33
|
+
export const realFsProbe = {
|
|
34
|
+
exists: (path) => {
|
|
35
|
+
try {
|
|
36
|
+
statSync(path);
|
|
37
|
+
return true;
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
return false;
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
isDirectory: (path) => {
|
|
44
|
+
try {
|
|
45
|
+
return statSync(path).isDirectory();
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
},
|
|
51
|
+
isWritable: (path) => {
|
|
52
|
+
try {
|
|
53
|
+
accessSync(path, constants.W_OK);
|
|
54
|
+
return true;
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
},
|
|
60
|
+
};
|
|
61
|
+
const REMEDY = {
|
|
62
|
+
'temp-dir': 'the host\'s temp directory changed, which a reboot does — stop this shift and start a new one from a fresh shell so it inherits the current TMPDIR',
|
|
63
|
+
};
|
|
64
|
+
const LABEL = {
|
|
65
|
+
'temp-dir': 'TMPDIR',
|
|
66
|
+
};
|
|
67
|
+
function checkOne(role, path, fs) {
|
|
68
|
+
if (!fs.exists(path)) {
|
|
69
|
+
return { kind: 'missing', role, path, message: `${LABEL[role]} '${path}' no longer exists — ${REMEDY[role]}` };
|
|
70
|
+
}
|
|
71
|
+
if (!fs.isDirectory(path)) {
|
|
72
|
+
return {
|
|
73
|
+
kind: 'not-a-directory',
|
|
74
|
+
role,
|
|
75
|
+
path,
|
|
76
|
+
message: `${LABEL[role]} '${path}' is not a directory — ${REMEDY[role]}`,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
if (!fs.isWritable(path)) {
|
|
80
|
+
return {
|
|
81
|
+
kind: 'unwritable',
|
|
82
|
+
role,
|
|
83
|
+
path,
|
|
84
|
+
message: `${LABEL[role]} '${path}' is not writable by this process — ${REMEDY[role]}`,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
return undefined;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Report every local host fault, in a fixed role order so a reader comparing
|
|
91
|
+
* two records sees the same fault in the same position.
|
|
92
|
+
*
|
|
93
|
+
* The return is an array rather than a single fault so that adding a second
|
|
94
|
+
* role later does not change this function's contract, and so a caller never
|
|
95
|
+
* has to ask whether the one fault it was handed was the only one. An empty
|
|
96
|
+
* array means "no local fault" — which the caller must read as "the cause is
|
|
97
|
+
* elsewhere", never as "the shift is healthy".
|
|
98
|
+
*/
|
|
99
|
+
export function checkHost(paths, fs = realFsProbe) {
|
|
100
|
+
const candidates = [
|
|
101
|
+
['temp-dir', paths.tempDir ?? tmpdir()],
|
|
102
|
+
];
|
|
103
|
+
const faults = [];
|
|
104
|
+
for (const [role, path] of candidates) {
|
|
105
|
+
if (path === undefined || path === '')
|
|
106
|
+
continue;
|
|
107
|
+
const fault = checkOne(role, path, fs);
|
|
108
|
+
if (fault !== undefined)
|
|
109
|
+
faults.push(fault);
|
|
110
|
+
}
|
|
111
|
+
return faults;
|
|
112
|
+
}
|
|
@@ -4,6 +4,8 @@ import { type WorkOrder } from '../hub/types.ts';
|
|
|
4
4
|
import type { FetchedStep } from '../bundle/types.ts';
|
|
5
5
|
import { type ReconcileOptions, type Liveness } from './state.ts';
|
|
6
6
|
import { sweepWorkDirs as sweepWorkDirsImpl } from '../agent/workdir.ts';
|
|
7
|
+
import { type HostFault } from './host-preflight.ts';
|
|
8
|
+
import { type DiskSpace } from './disk-floor.ts';
|
|
7
9
|
import type { Spawner, WorkerFailure } from './spawn.ts';
|
|
8
10
|
import { type ShiftEvent } from './protocol.ts';
|
|
9
11
|
export interface ShiftLoopOptions {
|
|
@@ -104,6 +106,22 @@ export interface ShiftLoopOptions {
|
|
|
104
106
|
/** PHASE 4 — injected in tests so the reaper can be exercised without a
|
|
105
107
|
* filesystem. Defaults to `sweepWorkDirs` from `src/agent/workdir.ts`. */
|
|
106
108
|
sweepWorkDirs?: typeof sweepWorkDirsImpl;
|
|
109
|
+
/**
|
|
110
|
+
* Does this machine still have the directories the shift needs? Injected in
|
|
111
|
+
* tests so a vanished temp directory can be presented without one, and so the
|
|
112
|
+
* check itself can be made to throw. Defaults to `checkHost`, which is
|
|
113
|
+
* `stat`-only and issues no hub traffic. An empty array means "no LOCAL
|
|
114
|
+
* fault", never "healthy" — see `host-preflight.ts`.
|
|
115
|
+
*/
|
|
116
|
+
hostPreflight?: () => HostFault[];
|
|
117
|
+
/**
|
|
118
|
+
* Is there room on this disk to start another order? Injected in tests so a
|
|
119
|
+
* full volume can be presented without filling one. Defaults to a real
|
|
120
|
+
* `statfs` of `workRoot` (or its nearest existing ancestor) against the
|
|
121
|
+
* resolved floor. `unknown` means "could not measure, or switched off" and is
|
|
122
|
+
* treated as permission to dispatch — see `disk-floor.ts`.
|
|
123
|
+
*/
|
|
124
|
+
diskSpace?: () => DiskSpace;
|
|
107
125
|
}
|
|
108
126
|
export interface LockedRemovalOptions {
|
|
109
127
|
dispatchLockOptions?: Pick<AcquireFileLockOpts, 'beforeOpen'>;
|