@ours.network/fleet 0.10.0-nightly.4 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +138 -21
- package/dist/atomic-file.d.ts +30 -0
- package/dist/atomic-file.js +86 -0
- package/dist/briefing.d.ts +6 -0
- package/dist/briefing.js +43 -13
- package/dist/cli.js +98 -22
- package/dist/config.d.ts +24 -3
- package/dist/config.js +84 -11
- package/dist/creation.d.ts +179 -0
- package/dist/creation.js +254 -0
- package/dist/docs.d.ts +28 -1
- package/dist/docs.js +155 -8
- package/dist/doctor.js +75 -17
- package/dist/harness/claude-code.d.ts +39 -3
- package/dist/harness/claude-code.js +128 -26
- package/dist/harness/codex.d.ts +7 -1
- package/dist/harness/codex.js +58 -11
- package/dist/harness/registry.d.ts +2 -0
- package/dist/harness/registry.js +19 -0
- package/dist/harness/types.d.ts +51 -4
- package/dist/isolation/bubblewrap.js +7 -1
- package/dist/isolation/policy.d.ts +34 -5
- package/dist/isolation/policy.js +114 -7
- package/dist/isolation/resources.d.ts +6 -3
- package/dist/isolation/resources.js +6 -3
- package/dist/isolation/types.d.ts +19 -1
- package/dist/monitor.d.ts +33 -4
- package/dist/monitor.js +150 -32
- package/dist/ops.d.ts +15 -2
- package/dist/ops.js +32 -9
- package/dist/permissions.d.ts +70 -0
- package/dist/permissions.js +97 -0
- package/dist/runner.d.ts +65 -2
- package/dist/runner.js +262 -27
- package/dist/session/acp.d.ts +25 -2
- package/dist/session/acp.js +143 -26
- package/dist/session/control.d.ts +49 -1
- package/dist/session/control.js +116 -12
- package/dist/session/tmux.d.ts +9 -2
- package/dist/session/tmux.js +36 -4
- package/dist/session/types.d.ts +99 -2
- package/dist/session/types.js +42 -1
- package/dist/spawn.d.ts +27 -2
- package/dist/spawn.js +153 -15
- package/dist/supervisor/launchd.d.ts +50 -0
- package/dist/supervisor/launchd.js +121 -4
- package/dist/supervisor/none.js +22 -4
- package/dist/supervisor/systemd.d.ts +8 -1
- package/dist/supervisor/systemd.js +94 -4
- package/dist/supervisor/types.d.ts +36 -3
- package/dist/tmux.d.ts +34 -2
- package/dist/tmux.js +48 -11
- package/package.json +1 -1
package/dist/isolation/policy.js
CHANGED
|
@@ -1,5 +1,57 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { realpathSync } from 'node:fs';
|
|
2
|
+
import { basename, dirname, join, resolve, sep } from 'node:path';
|
|
2
3
|
import { BACKENDS, ON_UNAVAILABLE, NETWORK_MODES, } from './types.js';
|
|
4
|
+
/** A mount that the forbidden-path policy refuses. Raised before any launch. */
|
|
5
|
+
export class IsolationPolicyError extends Error {
|
|
6
|
+
constructor(message) {
|
|
7
|
+
super(message);
|
|
8
|
+
this.name = 'IsolationPolicyError';
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Resolve a path to its canonical form, following symlinks as far as the
|
|
13
|
+
* filesystem allows and normalising the rest. Without this, `~/link-to-ssh`
|
|
14
|
+
* and `/home/u/.ssh` are different strings for the same directory, and a
|
|
15
|
+
* string comparison against the forbidden list is trivially side-stepped.
|
|
16
|
+
*/
|
|
17
|
+
export function canonicalPath(p) {
|
|
18
|
+
const abs = resolve(p);
|
|
19
|
+
let head = abs;
|
|
20
|
+
let tail = '';
|
|
21
|
+
for (;;) {
|
|
22
|
+
try {
|
|
23
|
+
return tail ? join(realpathSync.native(head), tail) : realpathSync.native(head);
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
const parent = dirname(head);
|
|
27
|
+
if (parent === head)
|
|
28
|
+
return abs; // nothing on this path exists yet
|
|
29
|
+
tail = tail ? join(basename(head), tail) : basename(head);
|
|
30
|
+
head = parent;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
const within = (child, parent) => child === parent || child.startsWith(parent + sep);
|
|
35
|
+
/**
|
|
36
|
+
* How a canonical mount path collides with a canonical forbidden path.
|
|
37
|
+
*
|
|
38
|
+
* `parent` matters as much as the other two: binding `$HOME` does not name
|
|
39
|
+
* `~/.ssh`, but it exposes it just as completely.
|
|
40
|
+
*/
|
|
41
|
+
export function mountConflict(mount, forbidden) {
|
|
42
|
+
if (mount === forbidden)
|
|
43
|
+
return 'exact';
|
|
44
|
+
if (within(mount, forbidden))
|
|
45
|
+
return 'descendant';
|
|
46
|
+
if (within(forbidden, mount))
|
|
47
|
+
return 'parent';
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
const CONFLICT_WORDING = {
|
|
51
|
+
exact: 'is',
|
|
52
|
+
descendant: 'is inside',
|
|
53
|
+
parent: 'would expose',
|
|
54
|
+
};
|
|
3
55
|
/** Read-only system dirs exposed under the allowlist model. */
|
|
4
56
|
const SYSTEM_RO = ['/usr', '/bin', '/sbin', '/lib', '/lib64', '/etc'];
|
|
5
57
|
/** Ephemeral scratch mounts. */
|
|
@@ -52,6 +104,12 @@ export function validateIsolationConfig(raw) {
|
|
|
52
104
|
}
|
|
53
105
|
return problems;
|
|
54
106
|
}
|
|
107
|
+
/**
|
|
108
|
+
* Where a role's per-role harness runtime state lives (5.1). Under the agent's
|
|
109
|
+
* own state directory, so it is covered by the state dir's existing lifecycle
|
|
110
|
+
* and by the forbidden-path exception, and is never shared with a peer.
|
|
111
|
+
*/
|
|
112
|
+
export const harnessRuntimeDir = (stateDir, harnessId) => join(stateDir, 'harness', harnessId);
|
|
55
113
|
/** Parse a `host:container` secret pair; a bare path maps to itself. */
|
|
56
114
|
function parseSecret(pair) {
|
|
57
115
|
const i = pair.indexOf(':');
|
|
@@ -61,12 +119,17 @@ function parseSecret(pair) {
|
|
|
61
119
|
}
|
|
62
120
|
/**
|
|
63
121
|
* Resolve a raw (already validated) isolation block against runtime context into
|
|
64
|
-
* a defaults-filled, backend-agnostic policy
|
|
122
|
+
* a defaults-filled, backend-agnostic policy, and REFUSE any mount that would
|
|
123
|
+
* breach the forbidden-path list.
|
|
124
|
+
*
|
|
125
|
+
* The mount model is an allowlist: only the durable set (state dir, cwd, harness
|
|
126
|
+
* config, declared fs/secrets) plus read-only system dirs are exposed. The
|
|
127
|
+
* forbidden list — the ours key store, sibling agent state dirs, ~/.ssh, ~/.aws
|
|
128
|
+
* — is now enforced on top of that, so a role cannot ask its way back in.
|
|
65
129
|
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
* is simply never mounted, and thus absent inside the sandbox (§5.2).
|
|
130
|
+
* Not pure: canonicalising a path reads the filesystem, because symlink aliases
|
|
131
|
+
* are one of the ways a forbidden path gets requested. Throws
|
|
132
|
+
* `IsolationPolicyError`; callers surface it against the role.
|
|
70
133
|
*/
|
|
71
134
|
export function resolveIsolation(cfg, ctx) {
|
|
72
135
|
const { stateDir, runCwd, home } = ctx;
|
|
@@ -75,10 +138,26 @@ export function resolveIsolation(cfg, ctx) {
|
|
|
75
138
|
mounts.push({ src: p, dst: p, mode: 'rw' }); };
|
|
76
139
|
const addRo = (p) => { if (!mounts.some(m => m.src === p))
|
|
77
140
|
mounts.push({ src: p, dst: p, mode: 'ro' }); };
|
|
141
|
+
/** Writable bind whose destination differs from its source (the per-role home). */
|
|
142
|
+
const addRw2 = (src, dst) => {
|
|
143
|
+
if (!mounts.some(m => m.src === src && m.dst === dst))
|
|
144
|
+
mounts.push({ src, dst, mode: 'rw' });
|
|
145
|
+
};
|
|
78
146
|
// Durable set: state dir + cwd, then only the active harness's config/auth roots.
|
|
79
147
|
addRw(stateDir);
|
|
80
148
|
addRw(runCwd);
|
|
81
|
-
if (ctx.
|
|
149
|
+
if (ctx.harnessHome && ctx.harnessRuntimeDir) {
|
|
150
|
+
// The harness home is backed by a PER-ROLE directory (5.1): the agent gets a
|
|
151
|
+
// writable home for its sessions, caches and history, and anything a future
|
|
152
|
+
// CLI version writes lands there too. The shared credentials, global
|
|
153
|
+
// instructions and configuration are then layered back read-only, so they
|
|
154
|
+
// are readable and cannot be rewritten — for this role or for its peers.
|
|
155
|
+
// Order matters: the writable home must precede the read-only overlays.
|
|
156
|
+
addRw2(ctx.harnessRuntimeDir, ctx.harnessHome);
|
|
157
|
+
for (const p of ctx.harnessSharedPaths ?? [])
|
|
158
|
+
addRo(p);
|
|
159
|
+
}
|
|
160
|
+
else if (ctx.harness === 'codex') {
|
|
82
161
|
addRw(join(home, '.codex'));
|
|
83
162
|
addRo(join(home, '.agents'));
|
|
84
163
|
}
|
|
@@ -105,6 +184,34 @@ export function resolveIsolation(cfg, ctx) {
|
|
|
105
184
|
...SENSITIVE_HOME.map(p => join(home, p)),
|
|
106
185
|
agentsRoot, // sibling agents' state dirs (this agent's own is explicitly mounted)
|
|
107
186
|
];
|
|
187
|
+
// ENFORCE the list, before anything builds a backend argv. Until now it was
|
|
188
|
+
// observational: the allowlist model kept these paths out by default, but a
|
|
189
|
+
// role that asked for one in `fs.write`, `secrets`, or a Codex `add_dirs` got
|
|
190
|
+
// it mounted anyway, and the "blocklist" recorded a guarantee it never made.
|
|
191
|
+
//
|
|
192
|
+
// The role's OWN state dir is the one legitimate descendant of the agents
|
|
193
|
+
// root, so it is excepted by exact canonical identity — which does not
|
|
194
|
+
// exempt its parent, and does not exempt a sibling.
|
|
195
|
+
const forbidden = blocklist.map(canonicalPath);
|
|
196
|
+
const ownStateDir = canonicalPath(stateDir);
|
|
197
|
+
for (const m of mounts) {
|
|
198
|
+
for (const [role, path] of [['source', m.src], ['destination', m.dst]]) {
|
|
199
|
+
const canon = canonicalPath(path);
|
|
200
|
+
// The role's own state dir and anything inside it (its per-role harness
|
|
201
|
+
// runtime home, 5.1) is the one legitimate descendant of the agents root.
|
|
202
|
+
// This does not exempt the root above it, nor a sibling beside it.
|
|
203
|
+
if (within(canon, ownStateDir))
|
|
204
|
+
continue;
|
|
205
|
+
for (let i = 0; i < forbidden.length; i++) {
|
|
206
|
+
const kind = mountConflict(canon, forbidden[i]);
|
|
207
|
+
if (!kind)
|
|
208
|
+
continue;
|
|
209
|
+
const alias = canon === resolve(path) ? '' : ` (resolves to '${canon}')`;
|
|
210
|
+
throw new IsolationPolicyError(`isolation: refusing to mount ${role} '${path}'${alias} — it ` +
|
|
211
|
+
`${CONFLICT_WORDING[kind]} the forbidden path '${blocklist[i]}'`);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
108
215
|
return {
|
|
109
216
|
backend: cfg.backend ?? 'auto',
|
|
110
217
|
onUnavailable: cfg.on_unavailable ?? 'warn',
|
|
@@ -5,9 +5,12 @@ export interface ResourceArgs {
|
|
|
5
5
|
}
|
|
6
6
|
/**
|
|
7
7
|
* Build the `systemd-run --user --scope -p … --` prefix that caps the pane's
|
|
8
|
-
* cgroup-v2 scope. Composed OUTSIDE the sandbox wrap (§5.3/§5.4):
|
|
9
|
-
*
|
|
10
|
-
* only reliable per-agent limit is a transient scope at the pane itself.
|
|
8
|
+
* cgroup-v2 scope. Composed OUTSIDE the sandbox wrap (§5.3/§5.4): a tmux pane is
|
|
9
|
+
* a child of a tmux SERVER rather than of the role's own runner process, so the
|
|
10
|
+
* only reliable per-agent limit is a transient scope at the pane itself. (Since
|
|
11
|
+
* #32 that server is per role rather than fleet-wide, which is what keeps one
|
|
12
|
+
* role's `stop` off every other role's pane — the limit still belongs on the
|
|
13
|
+
* pane.)
|
|
11
14
|
*
|
|
12
15
|
* mem/pids are always enforced (their controllers are delegated to `--user` by
|
|
13
16
|
* default). cpu degrades to a warning when the cpu controller is not delegated.
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { readFileSync } from 'node:fs';
|
|
2
2
|
/**
|
|
3
3
|
* Build the `systemd-run --user --scope -p … --` prefix that caps the pane's
|
|
4
|
-
* cgroup-v2 scope. Composed OUTSIDE the sandbox wrap (§5.3/§5.4):
|
|
5
|
-
*
|
|
6
|
-
* only reliable per-agent limit is a transient scope at the pane itself.
|
|
4
|
+
* cgroup-v2 scope. Composed OUTSIDE the sandbox wrap (§5.3/§5.4): a tmux pane is
|
|
5
|
+
* a child of a tmux SERVER rather than of the role's own runner process, so the
|
|
6
|
+
* only reliable per-agent limit is a transient scope at the pane itself. (Since
|
|
7
|
+
* #32 that server is per role rather than fleet-wide, which is what keeps one
|
|
8
|
+
* role's `stop` off every other role's pane — the limit still belongs on the
|
|
9
|
+
* pane.)
|
|
7
10
|
*
|
|
8
11
|
* mem/pids are always enforced (their controllers are delegated to `--user` by
|
|
9
12
|
* default). cpu degrades to a warning when the cpu controller is not delegated.
|
|
@@ -50,6 +50,19 @@ export interface WrapContext {
|
|
|
50
50
|
harness?: string;
|
|
51
51
|
/** Harness-declared writable roots (for example Codex --add-dir). */
|
|
52
52
|
additionalWriteDirs?: string[];
|
|
53
|
+
/**
|
|
54
|
+
* The harness's home directory on the host (`~/.claude`, `~/.codex`). Mounted
|
|
55
|
+
* from `harnessRuntimeDir` so the agent's own runtime state is per-role (5.1).
|
|
56
|
+
*/
|
|
57
|
+
harnessHome?: string;
|
|
58
|
+
/** Per-role writable directory backing `harnessHome` inside the sandbox. */
|
|
59
|
+
harnessRuntimeDir?: string;
|
|
60
|
+
/**
|
|
61
|
+
* Shared credentials, global instructions and configuration. Mounted READ-ONLY
|
|
62
|
+
* on top of the per-role home, so an agent can read them and cannot rewrite
|
|
63
|
+
* them for itself or for its peers.
|
|
64
|
+
*/
|
|
65
|
+
harnessSharedPaths?: string[];
|
|
53
66
|
brokerEndpoint?: string;
|
|
54
67
|
}
|
|
55
68
|
/**
|
|
@@ -68,7 +81,12 @@ export interface ResolvedIsolation {
|
|
|
68
81
|
system: string[];
|
|
69
82
|
/** ephemeral scratch tmpfs mounts (/tmp, ~/.cache). */
|
|
70
83
|
tmpfs: string[];
|
|
71
|
-
/**
|
|
84
|
+
/**
|
|
85
|
+
* Sensitive host paths that are ENFORCED absent from the sandbox: any mount
|
|
86
|
+
* that is, sits inside, or would expose one of these is refused by
|
|
87
|
+
* `resolveIsolation` before a backend argv is built. Retained on the resolved
|
|
88
|
+
* policy for diagnostics — doctor and `config` report what is being enforced.
|
|
89
|
+
*/
|
|
72
90
|
blocklist: string[];
|
|
73
91
|
}
|
|
74
92
|
/** A pluggable isolation backend (bubblewrap, podman, none). */
|
package/dist/monitor.d.ts
CHANGED
|
@@ -38,14 +38,29 @@ export interface MonitorDeps {
|
|
|
38
38
|
set(fn: () => void, ms: number): ReturnType<typeof setTimeout>;
|
|
39
39
|
clear(t: ReturnType<typeof setTimeout>): void;
|
|
40
40
|
};
|
|
41
|
-
/**
|
|
41
|
+
/**
|
|
42
|
+
* Structured prompt delivery used by ACP sessions. Tmux remains the fallback.
|
|
43
|
+
* `succeeded` is the turn's TERMINAL result, not merely that the session took
|
|
44
|
+
* the prompt: a refused or cancelled wake was seen and not acted on, and must
|
|
45
|
+
* not commit the cursor.
|
|
46
|
+
*/
|
|
42
47
|
delivery?: {
|
|
43
|
-
submit(text: string
|
|
44
|
-
|
|
48
|
+
submit(text: string, options?: {
|
|
49
|
+
interrupt?: boolean;
|
|
50
|
+
}): Promise<{
|
|
51
|
+
succeeded: boolean;
|
|
52
|
+
outcome: string;
|
|
45
53
|
detail?: string;
|
|
46
54
|
}>;
|
|
47
55
|
};
|
|
48
56
|
}
|
|
57
|
+
/**
|
|
58
|
+
* Why the monitor is not healthy. Each cause clears on its OWN recovery signal
|
|
59
|
+
* and nothing else — a successful poll proves the stream works, and proves
|
|
60
|
+
* nothing whatsoever about whether wakes are being delivered or whether the
|
|
61
|
+
* turns they trigger keep dying.
|
|
62
|
+
*/
|
|
63
|
+
export type StatusCause = 'connectivity' | 'delivery' | 'modal' | 'offline' | 'turns-failing' | 'auth';
|
|
49
64
|
/** Best-effort daemon config (issue #17): the fields the MCP client reads. */
|
|
50
65
|
interface DaemonConfig {
|
|
51
66
|
apiToken?: string;
|
|
@@ -149,6 +164,8 @@ export declare class Monitor {
|
|
|
149
164
|
private currentAbort;
|
|
150
165
|
private apiErrorStreak;
|
|
151
166
|
private readonly turnFailThreshold;
|
|
167
|
+
/** Active degradations, keyed by cause. Empty means armed. */
|
|
168
|
+
private readonly causes;
|
|
152
169
|
constructor(o: MonitorOpts);
|
|
153
170
|
/** Resume the last delivered cursor; only a brand-new monitor primes at stream tip. */
|
|
154
171
|
prime(): Promise<void>;
|
|
@@ -190,7 +207,19 @@ export declare class Monitor {
|
|
|
190
207
|
private readPersistedCursor;
|
|
191
208
|
/** Atomically persist body-free delivery state; restart always resumes from deliveredCursor. */
|
|
192
209
|
private persistState;
|
|
193
|
-
|
|
210
|
+
/** Record a degradation under its own cause and republish the status. */
|
|
211
|
+
private degrade;
|
|
212
|
+
/**
|
|
213
|
+
* Clear exactly the causes this recovery signal speaks to. Anything else
|
|
214
|
+
* stays: one successful poll must never be able to erase `turns failing`.
|
|
215
|
+
*/
|
|
216
|
+
private recover;
|
|
217
|
+
/**
|
|
218
|
+
* One line per active cause, each dated; `armed` when there are none. Every
|
|
219
|
+
* line carries an ISO timestamp so an operator can tell a live status from a
|
|
220
|
+
* stale one left behind by a monitor that stopped writing.
|
|
221
|
+
*/
|
|
222
|
+
private writeStatus;
|
|
194
223
|
}
|
|
195
224
|
export declare function createMonitor(o: MonitorOpts): Monitor;
|
|
196
225
|
export {};
|
package/dist/monitor.js
CHANGED
|
@@ -234,11 +234,73 @@ export function looksRunning(pane) {
|
|
|
234
234
|
return true; // "(12s · … tokens)" elapsed meter
|
|
235
235
|
return false;
|
|
236
236
|
}
|
|
237
|
-
|
|
237
|
+
const COMPOSER_TOP = /^[^\S\n]*[╭┌][─━]/;
|
|
238
|
+
const COMPOSER_BOTTOM = /^[^\S\n]*[╰└][─━]/;
|
|
239
|
+
const COMPOSER_PROMPT = /^[^\S\n]*(?:[│┃|][^\S\n]*)?[❯›>][^\S\n]*$/;
|
|
240
|
+
/**
|
|
241
|
+
* Remove wrapping-only whitespace and box chrome from composer rows. Notification
|
|
242
|
+
* lines contain no meaningful whitespace distinction, so this lets a fragment
|
|
243
|
+
* cross an arbitrary terminal wrap without matching unrelated transcript text.
|
|
244
|
+
*/
|
|
245
|
+
function normalizeComposerRows(rows) {
|
|
246
|
+
return rows.map((raw, i) => {
|
|
247
|
+
let row = raw;
|
|
248
|
+
if (i > 0)
|
|
249
|
+
row = row.replace(/^[^\S\n]*(?:[│┃|][^\S\n]*)?/, '');
|
|
250
|
+
row = row.replace(/[^\S\n]*(?:[│┃|])?[^\S\n]*$/, '');
|
|
251
|
+
return row;
|
|
252
|
+
}).join('').replace(/\s+/g, '');
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* Is the injected line still sitting unsubmitted in the composer?
|
|
256
|
+
*
|
|
257
|
+
* The footer has variable height and the line may wrap across any number of
|
|
258
|
+
* rows, so a fixed tail window cannot identify the composer. Prefer the final
|
|
259
|
+
* bordered composer region; for a borderless/truncated capture, require the
|
|
260
|
+
* notification prefix to follow a composer prompt. This keeps old submitted
|
|
261
|
+
* wake lines in the transcript from causing stray Enters.
|
|
262
|
+
*/
|
|
238
263
|
function stillInComposer(pane, line) {
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
264
|
+
if (!pane)
|
|
265
|
+
return false; // dead pane: do not waste Enters
|
|
266
|
+
const lines = pane.split('\n');
|
|
267
|
+
let bottom = -1;
|
|
268
|
+
for (let i = lines.length - 1; i >= 0; i--) {
|
|
269
|
+
if (COMPOSER_BOTTOM.test(lines[i])) {
|
|
270
|
+
bottom = i;
|
|
271
|
+
break;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
let top = -1;
|
|
275
|
+
if (bottom >= 0) {
|
|
276
|
+
for (let i = bottom - 1; i >= 0; i--) {
|
|
277
|
+
if (COMPOSER_TOP.test(lines[i])) {
|
|
278
|
+
top = i;
|
|
279
|
+
break;
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
const from = top >= 0 ? top + 1 : 0;
|
|
284
|
+
const to = bottom >= 0 ? bottom : lines.length;
|
|
285
|
+
let wakeRow = -1;
|
|
286
|
+
let wakeColumn = -1;
|
|
287
|
+
for (let i = to - 1; i >= from; i--) {
|
|
288
|
+
const column = lines[i].lastIndexOf(PREFIX);
|
|
289
|
+
if (column >= 0) {
|
|
290
|
+
wakeRow = i;
|
|
291
|
+
wakeColumn = column;
|
|
292
|
+
break;
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
if (wakeRow < 0)
|
|
296
|
+
return false;
|
|
297
|
+
// Without both box boundaries, only trust text visibly in a composer prompt.
|
|
298
|
+
// This is the safe fallback for borderless TUIs and truncated captures.
|
|
299
|
+
if (top < 0 && !COMPOSER_PROMPT.test(lines[wakeRow].slice(0, wakeColumn)))
|
|
300
|
+
return false;
|
|
301
|
+
const rows = lines.slice(wakeRow, to);
|
|
302
|
+
rows[0] = rows[0].slice(wakeColumn);
|
|
303
|
+
return normalizeComposerRows(rows).includes(line.replace(/\s+/g, ''));
|
|
242
304
|
}
|
|
243
305
|
export class Monitor {
|
|
244
306
|
name;
|
|
@@ -260,6 +322,8 @@ export class Monitor {
|
|
|
260
322
|
// ended in an API error with no completed turn in between.
|
|
261
323
|
apiErrorStreak = 0;
|
|
262
324
|
turnFailThreshold;
|
|
325
|
+
/** Active degradations, keyed by cause. Empty means armed. */
|
|
326
|
+
causes = new Map();
|
|
263
327
|
constructor(o) {
|
|
264
328
|
this.name = o.name;
|
|
265
329
|
this.identity = o.identity ?? o.name;
|
|
@@ -278,23 +342,23 @@ export class Monitor {
|
|
|
278
342
|
if (persisted !== null) {
|
|
279
343
|
this.cursor = persisted;
|
|
280
344
|
this.deliveredCursor = persisted;
|
|
281
|
-
this.
|
|
345
|
+
this.writeStatus();
|
|
282
346
|
return;
|
|
283
347
|
}
|
|
284
348
|
try {
|
|
285
349
|
const body = await this.doFetch('tip', LONGPOLL_TIMEOUT_MS);
|
|
286
350
|
this.cursor = typeof body.cursor === 'number' ? body.cursor : 0;
|
|
287
351
|
this.persistCursor();
|
|
288
|
-
this.
|
|
352
|
+
this.writeStatus();
|
|
289
353
|
}
|
|
290
354
|
catch (e) {
|
|
291
355
|
if (e instanceof AuthError) {
|
|
292
356
|
this.fatal = true;
|
|
293
|
-
this.
|
|
357
|
+
this.degrade('auth', e.message, 'failed');
|
|
294
358
|
}
|
|
295
359
|
else {
|
|
296
360
|
this.cursor = null;
|
|
297
|
-
this.
|
|
361
|
+
this.degrade('connectivity', `prime failed (${msg(e)})`);
|
|
298
362
|
}
|
|
299
363
|
}
|
|
300
364
|
}
|
|
@@ -307,24 +371,26 @@ export class Monitor {
|
|
|
307
371
|
const pending = [];
|
|
308
372
|
while (!this.stopped) {
|
|
309
373
|
if (!this.deps.isAlive(pid)) {
|
|
310
|
-
this.
|
|
374
|
+
this.degrade('offline', 'session offline');
|
|
311
375
|
return;
|
|
312
376
|
}
|
|
313
377
|
let body;
|
|
314
378
|
try {
|
|
315
379
|
body = await this.doFetch(String(this.cursor ?? 0), LONGPOLL_TIMEOUT_MS);
|
|
316
380
|
backoff = 0;
|
|
381
|
+
// A poll that worked proves the stream is healthy — and only that.
|
|
382
|
+
this.recover('connectivity');
|
|
317
383
|
}
|
|
318
384
|
catch (e) {
|
|
319
385
|
if (this.stopped)
|
|
320
386
|
return;
|
|
321
387
|
if (e instanceof AuthError) {
|
|
322
388
|
this.fatal = true;
|
|
323
|
-
this.
|
|
389
|
+
this.degrade('auth', e.message, 'failed');
|
|
324
390
|
return;
|
|
325
391
|
}
|
|
326
392
|
backoff = Math.min(backoff + BACKOFF_STEP_MS, BACKOFF_MAX_MS);
|
|
327
|
-
this.
|
|
393
|
+
this.degrade('connectivity', `stream hiccup (${msg(e)})`);
|
|
328
394
|
await this.deps.sleep(backoff);
|
|
329
395
|
continue;
|
|
330
396
|
}
|
|
@@ -350,7 +416,7 @@ export class Monitor {
|
|
|
350
416
|
accepted = await this.deliver(pid, pending);
|
|
351
417
|
}
|
|
352
418
|
catch (e) {
|
|
353
|
-
this.
|
|
419
|
+
this.degrade('delivery', `delivery failed (${msg(e)})`);
|
|
354
420
|
}
|
|
355
421
|
if (accepted) {
|
|
356
422
|
pending.length = 0;
|
|
@@ -381,20 +447,27 @@ export class Monitor {
|
|
|
381
447
|
async deliver(pid, batch) {
|
|
382
448
|
const line = formatNotificationLine(batch);
|
|
383
449
|
if (this.deps.delivery) {
|
|
384
|
-
const result = await this.deps.delivery.submit(line);
|
|
385
|
-
if (!result.
|
|
386
|
-
|
|
450
|
+
const result = await this.deps.delivery.submit(line, { interrupt: this.cfg.interrupt });
|
|
451
|
+
if (!result.succeeded) {
|
|
452
|
+
// Name the reason: "refused" and "cancelled" are the agent's answer,
|
|
453
|
+
// not a transport problem, and an operator has to be able to tell them
|
|
454
|
+
// apart from a dead socket.
|
|
455
|
+
this.degrade('delivery', `wake ${result.outcome}${result.detail ? ` (${result.detail})` : ''}`);
|
|
387
456
|
return false;
|
|
388
457
|
}
|
|
389
|
-
this.
|
|
458
|
+
this.recover('delivery', 'modal');
|
|
459
|
+
if (result.detail !== 'injected' && result.detail !== 'startedNewTurn')
|
|
460
|
+
this.recordTurn('completed');
|
|
390
461
|
return true;
|
|
391
462
|
}
|
|
463
|
+
if (this.cfg.interrupt)
|
|
464
|
+
await this.deps.tmux.sendKey(this.name, 'C-c');
|
|
392
465
|
const state = await this.awaitInjectable(pid);
|
|
393
466
|
if (state !== 'ready') {
|
|
394
467
|
if (state === 'offline')
|
|
395
|
-
this.
|
|
468
|
+
this.degrade('offline', 'offline during delivery');
|
|
396
469
|
else if (state === 'modal')
|
|
397
|
-
this.
|
|
470
|
+
this.degrade('modal', `modal wedge — pane held a dialog for ` +
|
|
398
471
|
`${MODAL_GIVE_UP_MS / 1000}s, wake not injected`);
|
|
399
472
|
return false;
|
|
400
473
|
}
|
|
@@ -404,11 +477,25 @@ export class Monitor {
|
|
|
404
477
|
// Verify submission for THIS line even if stop() arrives mid-flight: the text
|
|
405
478
|
// is already in the composer and we want it submitted (at-least-once). A truly
|
|
406
479
|
// dead pane makes safeCapture return '' ⇒ not-in-composer ⇒ breaks, no wasted Enter.
|
|
407
|
-
for (let i = 0; i < MAX_ENTER_RETRIES;
|
|
480
|
+
for (let i = 0; i < MAX_ENTER_RETRIES;) {
|
|
408
481
|
await this.deps.sleep(POST_VERIFY_MS);
|
|
409
482
|
const capture = await safeCapture(this.deps.tmux, this.name);
|
|
410
483
|
if (!capture.ok) {
|
|
411
|
-
this.
|
|
484
|
+
this.degrade('delivery', 'capture failed during injection verification');
|
|
485
|
+
return false;
|
|
486
|
+
}
|
|
487
|
+
// A dialog can appear after the initial send. Never let a verification
|
|
488
|
+
// retry confirm it. Wait under the same bounded modal policy as initial
|
|
489
|
+
// injection, then re-capture immediately before considering Enter.
|
|
490
|
+
if (looksModal(capture.pane)) {
|
|
491
|
+
const state = await this.awaitInjectable(pid);
|
|
492
|
+
if (state === 'ready')
|
|
493
|
+
continue;
|
|
494
|
+
if (state === 'offline')
|
|
495
|
+
this.degrade('offline', 'offline during injection verification');
|
|
496
|
+
else if (state === 'modal')
|
|
497
|
+
this.degrade('modal', `modal wedge during injection verification — ` +
|
|
498
|
+
`no Enter sent for ${MODAL_GIVE_UP_MS / 1000}s`);
|
|
412
499
|
return false;
|
|
413
500
|
}
|
|
414
501
|
if (!stillInComposer(capture.pane, line)) {
|
|
@@ -416,11 +503,13 @@ export class Monitor {
|
|
|
416
503
|
break;
|
|
417
504
|
}
|
|
418
505
|
await this.deps.tmux.sendKey(this.name, 'Enter');
|
|
506
|
+
i++;
|
|
419
507
|
}
|
|
420
508
|
if (!delivered) {
|
|
421
|
-
this.
|
|
509
|
+
this.degrade('delivery', 'injection unverified');
|
|
422
510
|
return false;
|
|
423
511
|
}
|
|
512
|
+
this.recover('delivery', 'modal');
|
|
424
513
|
// The wake landed and a turn started; observe how that turn terminates so a
|
|
425
514
|
// refusal-wedge (every turn dies with `API Error:` while delivery stays green)
|
|
426
515
|
// becomes visible in `.monitor-status` instead of masquerading as armed (#19).
|
|
@@ -442,7 +531,7 @@ export class Monitor {
|
|
|
442
531
|
return; // loop marks offline
|
|
443
532
|
const capture = await safeCapture(this.deps.tmux, this.name);
|
|
444
533
|
if (!capture.ok) {
|
|
445
|
-
this.
|
|
534
|
+
this.degrade('delivery', 'capture failed during turn observation');
|
|
446
535
|
return;
|
|
447
536
|
}
|
|
448
537
|
if (looksApiError(capture.pane)) {
|
|
@@ -459,14 +548,17 @@ export class Monitor {
|
|
|
459
548
|
}
|
|
460
549
|
/** Update the consecutive-API-error streak and derive `.monitor-status` from it. */
|
|
461
550
|
recordTurn(outcome) {
|
|
551
|
+
if (outcome === 'inconclusive')
|
|
552
|
+
return; // no evidence either way; leave the streak
|
|
462
553
|
if (outcome === 'api-error')
|
|
463
554
|
this.apiErrorStreak++;
|
|
464
|
-
else
|
|
555
|
+
else
|
|
465
556
|
this.apiErrorStreak = 0;
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
557
|
+
if (this.apiErrorStreak >= this.turnFailThreshold)
|
|
558
|
+
this.degrade('turns-failing', 'turns failing (api error)');
|
|
559
|
+
else if (outcome === 'completed')
|
|
560
|
+
// A turn that ran to the end is the ONLY thing that clears this.
|
|
561
|
+
this.recover('turns-failing');
|
|
470
562
|
}
|
|
471
563
|
/**
|
|
472
564
|
* Reset the composer to empty before typing a wake. Without this, any
|
|
@@ -500,7 +592,7 @@ export class Monitor {
|
|
|
500
592
|
}
|
|
501
593
|
const capture = await safeCapture(this.deps.tmux, this.name);
|
|
502
594
|
if (!capture.ok) {
|
|
503
|
-
this.
|
|
595
|
+
this.degrade('delivery', 'capture failed while checking session readiness');
|
|
504
596
|
await this.deps.sleep(MODAL_RETRY_MS);
|
|
505
597
|
continue;
|
|
506
598
|
}
|
|
@@ -592,15 +684,41 @@ export class Monitor {
|
|
|
592
684
|
this.deps.log(`[${this.name}] monitor: failed to persist state: ${msg(e)}`);
|
|
593
685
|
}
|
|
594
686
|
}
|
|
595
|
-
|
|
687
|
+
/** Record a degradation under its own cause and republish the status. */
|
|
688
|
+
degrade(cause, detail, level = 'degraded') {
|
|
689
|
+
const previous = this.causes.get(cause);
|
|
690
|
+
this.causes.set(cause, { level, detail, at: new Date(this.deps.now()).toISOString() });
|
|
691
|
+
this.writeStatus();
|
|
692
|
+
if (previous?.detail !== detail)
|
|
693
|
+
this.deps.log(`[${this.name}] monitor ${level}: ${cause} — ${detail}`);
|
|
694
|
+
}
|
|
695
|
+
/**
|
|
696
|
+
* Clear exactly the causes this recovery signal speaks to. Anything else
|
|
697
|
+
* stays: one successful poll must never be able to erase `turns failing`.
|
|
698
|
+
*/
|
|
699
|
+
recover(...causes) {
|
|
700
|
+
let changed = false;
|
|
701
|
+
for (const cause of causes)
|
|
702
|
+
changed = this.causes.delete(cause) || changed;
|
|
703
|
+
if (changed)
|
|
704
|
+
this.deps.log(`[${this.name}] monitor recovered: ${causes.join(', ')}`);
|
|
705
|
+
this.writeStatus();
|
|
706
|
+
}
|
|
707
|
+
/**
|
|
708
|
+
* One line per active cause, each dated; `armed` when there are none. Every
|
|
709
|
+
* line carries an ISO timestamp so an operator can tell a live status from a
|
|
710
|
+
* stale one left behind by a monitor that stopped writing.
|
|
711
|
+
*/
|
|
712
|
+
writeStatus() {
|
|
713
|
+
const lines = this.causes.size
|
|
714
|
+
? [...this.causes.entries()].map(([cause, e]) => `${e.level}: ${cause} at ${e.at} — ${e.detail}`)
|
|
715
|
+
: [`armed at ${new Date(this.deps.now()).toISOString()}`];
|
|
596
716
|
try {
|
|
597
|
-
writeFileSync(this.statusPath,
|
|
717
|
+
writeFileSync(this.statusPath, lines.join('\n') + '\n');
|
|
598
718
|
}
|
|
599
719
|
catch (e) {
|
|
600
720
|
this.deps.log(`[${this.name}] monitor: failed to write status: ${msg(e)}`);
|
|
601
721
|
}
|
|
602
|
-
if (!s.startsWith('armed'))
|
|
603
|
-
this.deps.log(`[${this.name}] monitor ${s}`);
|
|
604
722
|
}
|
|
605
723
|
}
|
|
606
724
|
export function createMonitor(o) {
|
package/dist/ops.d.ts
CHANGED
|
@@ -1,18 +1,31 @@
|
|
|
1
1
|
import type { FleetConfig, ResolvedRole } from './config.js';
|
|
2
|
-
import type { SupervisorBackend } from './supervisor/types.js';
|
|
2
|
+
import type { InstallOutcome as BackendInstallOutcome, SupervisorBackend } from './supervisor/types.js';
|
|
3
|
+
/** An install outcome tagged with the role it belongs to. */
|
|
4
|
+
export interface InstallOutcome extends BackendInstallOutcome {
|
|
5
|
+
role: string;
|
|
6
|
+
}
|
|
3
7
|
export interface OpsDeps {
|
|
4
8
|
backend: SupervisorBackend;
|
|
5
9
|
binPath: string;
|
|
6
10
|
log(line: string): void;
|
|
11
|
+
/**
|
|
12
|
+
* Called the INSTANT a registration is created, before anything else can
|
|
13
|
+
* fail. A creation transaction that learns about registrations only from
|
|
14
|
+
* `up()`'s return value learns nothing when `up()` throws — and the service
|
|
15
|
+
* it just registered is then invisible to rollback (6.2). Optional: plain
|
|
16
|
+
* `ours-fleet up` has no transaction to tell.
|
|
17
|
+
*/
|
|
18
|
+
onInstalled?(outcome: InstallOutcome): void;
|
|
7
19
|
}
|
|
8
20
|
/** Materialize a role's state dir from config: briefing + markers. Returns the dir. */
|
|
9
21
|
export declare function applyRole(role: ResolvedRole, opts?: {
|
|
10
22
|
fresh?: boolean;
|
|
11
23
|
temp?: boolean;
|
|
12
24
|
configPath?: string;
|
|
25
|
+
identityGuarantee?: 'verified' | 'created' | 'unverified';
|
|
13
26
|
}): string;
|
|
14
27
|
/** Create/start roles declaratively. Idempotent; active roles keep their context. */
|
|
15
|
-
export declare function up(cfg: FleetConfig, names: string[], deps: OpsDeps, configPath?: string): Promise<
|
|
28
|
+
export declare function up(cfg: FleetConfig, names: string[], deps: OpsDeps, configPath?: string, identityGuarantee?: 'verified' | 'created' | 'unverified'): Promise<InstallOutcome[]>;
|
|
16
29
|
export declare function down(cfg: FleetConfig, names: string[], deps: OpsDeps): Promise<void>;
|
|
17
30
|
/** Re-sync from config + bounce. mode 'keep' resumes context; 'fresh' wipes it. */
|
|
18
31
|
export declare function restartRoles(cfg: FleetConfig, names: string[], deps: OpsDeps, mode: 'keep' | 'fresh', configPath?: string): Promise<void>;
|