@ours.network/fleet 0.10.0-nightly.4 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +138 -21
- package/dist/atomic-file.d.ts +30 -0
- package/dist/atomic-file.js +86 -0
- package/dist/briefing.d.ts +6 -0
- package/dist/briefing.js +43 -13
- package/dist/cli.js +98 -22
- package/dist/config.d.ts +24 -3
- package/dist/config.js +84 -11
- package/dist/creation.d.ts +179 -0
- package/dist/creation.js +254 -0
- package/dist/docs.d.ts +28 -1
- package/dist/docs.js +155 -8
- package/dist/doctor.js +75 -17
- package/dist/harness/claude-code.d.ts +39 -3
- package/dist/harness/claude-code.js +128 -26
- package/dist/harness/codex.d.ts +7 -1
- package/dist/harness/codex.js +58 -11
- package/dist/harness/registry.d.ts +2 -0
- package/dist/harness/registry.js +19 -0
- package/dist/harness/types.d.ts +51 -4
- package/dist/isolation/bubblewrap.js +7 -1
- package/dist/isolation/policy.d.ts +34 -5
- package/dist/isolation/policy.js +114 -7
- package/dist/isolation/resources.d.ts +6 -3
- package/dist/isolation/resources.js +6 -3
- package/dist/isolation/types.d.ts +19 -1
- package/dist/monitor.d.ts +44 -7
- package/dist/monitor.js +157 -35
- package/dist/ops.d.ts +15 -2
- package/dist/ops.js +32 -9
- package/dist/permissions.d.ts +70 -0
- package/dist/permissions.js +97 -0
- package/dist/runner.d.ts +72 -2
- package/dist/runner.js +291 -28
- package/dist/session/acp.d.ts +25 -2
- package/dist/session/acp.js +143 -26
- package/dist/session/control.d.ts +49 -1
- package/dist/session/control.js +116 -12
- package/dist/session/tmux.d.ts +9 -2
- package/dist/session/tmux.js +36 -4
- package/dist/session/types.d.ts +99 -2
- package/dist/session/types.js +42 -1
- package/dist/spawn.d.ts +27 -2
- package/dist/spawn.js +153 -15
- package/dist/supervisor/launchd.d.ts +50 -0
- package/dist/supervisor/launchd.js +121 -4
- package/dist/supervisor/none.js +22 -4
- package/dist/supervisor/systemd.d.ts +8 -1
- package/dist/supervisor/systemd.js +94 -4
- package/dist/supervisor/types.d.ts +36 -3
- package/dist/tmux.d.ts +34 -2
- package/dist/tmux.js +48 -11
- package/package.json +1 -1
package/dist/isolation/policy.js
CHANGED
|
@@ -1,5 +1,57 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { realpathSync } from 'node:fs';
|
|
2
|
+
import { basename, dirname, join, resolve, sep } from 'node:path';
|
|
2
3
|
import { BACKENDS, ON_UNAVAILABLE, NETWORK_MODES, } from './types.js';
|
|
4
|
+
/** A mount that the forbidden-path policy refuses. Raised before any launch. */
|
|
5
|
+
export class IsolationPolicyError extends Error {
|
|
6
|
+
constructor(message) {
|
|
7
|
+
super(message);
|
|
8
|
+
this.name = 'IsolationPolicyError';
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Resolve a path to its canonical form, following symlinks as far as the
|
|
13
|
+
* filesystem allows and normalising the rest. Without this, `~/link-to-ssh`
|
|
14
|
+
* and `/home/u/.ssh` are different strings for the same directory, and a
|
|
15
|
+
* string comparison against the forbidden list is trivially side-stepped.
|
|
16
|
+
*/
|
|
17
|
+
export function canonicalPath(p) {
|
|
18
|
+
const abs = resolve(p);
|
|
19
|
+
let head = abs;
|
|
20
|
+
let tail = '';
|
|
21
|
+
for (;;) {
|
|
22
|
+
try {
|
|
23
|
+
return tail ? join(realpathSync.native(head), tail) : realpathSync.native(head);
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
const parent = dirname(head);
|
|
27
|
+
if (parent === head)
|
|
28
|
+
return abs; // nothing on this path exists yet
|
|
29
|
+
tail = tail ? join(basename(head), tail) : basename(head);
|
|
30
|
+
head = parent;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
const within = (child, parent) => child === parent || child.startsWith(parent + sep);
|
|
35
|
+
/**
|
|
36
|
+
* How a canonical mount path collides with a canonical forbidden path.
|
|
37
|
+
*
|
|
38
|
+
* `parent` matters as much as the other two: binding `$HOME` does not name
|
|
39
|
+
* `~/.ssh`, but it exposes it just as completely.
|
|
40
|
+
*/
|
|
41
|
+
export function mountConflict(mount, forbidden) {
|
|
42
|
+
if (mount === forbidden)
|
|
43
|
+
return 'exact';
|
|
44
|
+
if (within(mount, forbidden))
|
|
45
|
+
return 'descendant';
|
|
46
|
+
if (within(forbidden, mount))
|
|
47
|
+
return 'parent';
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
const CONFLICT_WORDING = {
|
|
51
|
+
exact: 'is',
|
|
52
|
+
descendant: 'is inside',
|
|
53
|
+
parent: 'would expose',
|
|
54
|
+
};
|
|
3
55
|
/** Read-only system dirs exposed under the allowlist model. */
|
|
4
56
|
const SYSTEM_RO = ['/usr', '/bin', '/sbin', '/lib', '/lib64', '/etc'];
|
|
5
57
|
/** Ephemeral scratch mounts. */
|
|
@@ -52,6 +104,12 @@ export function validateIsolationConfig(raw) {
|
|
|
52
104
|
}
|
|
53
105
|
return problems;
|
|
54
106
|
}
|
|
107
|
+
/**
|
|
108
|
+
* Where a role's per-role harness runtime state lives (5.1). Under the agent's
|
|
109
|
+
* own state directory, so it is covered by the state dir's existing lifecycle
|
|
110
|
+
* and by the forbidden-path exception, and is never shared with a peer.
|
|
111
|
+
*/
|
|
112
|
+
export const harnessRuntimeDir = (stateDir, harnessId) => join(stateDir, 'harness', harnessId);
|
|
55
113
|
/** Parse a `host:container` secret pair; a bare path maps to itself. */
|
|
56
114
|
function parseSecret(pair) {
|
|
57
115
|
const i = pair.indexOf(':');
|
|
@@ -61,12 +119,17 @@ function parseSecret(pair) {
|
|
|
61
119
|
}
|
|
62
120
|
/**
|
|
63
121
|
* Resolve a raw (already validated) isolation block against runtime context into
|
|
64
|
-
* a defaults-filled, backend-agnostic policy
|
|
122
|
+
* a defaults-filled, backend-agnostic policy, and REFUSE any mount that would
|
|
123
|
+
* breach the forbidden-path list.
|
|
124
|
+
*
|
|
125
|
+
* The mount model is an allowlist: only the durable set (state dir, cwd, harness
|
|
126
|
+
* config, declared fs/secrets) plus read-only system dirs are exposed. The
|
|
127
|
+
* forbidden list — the ours key store, sibling agent state dirs, ~/.ssh, ~/.aws
|
|
128
|
+
* — is now enforced on top of that, so a role cannot ask its way back in.
|
|
65
129
|
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
* is simply never mounted, and thus absent inside the sandbox (§5.2).
|
|
130
|
+
* Not pure: canonicalising a path reads the filesystem, because symlink aliases
|
|
131
|
+
* are one of the ways a forbidden path gets requested. Throws
|
|
132
|
+
* `IsolationPolicyError`; callers surface it against the role.
|
|
70
133
|
*/
|
|
71
134
|
export function resolveIsolation(cfg, ctx) {
|
|
72
135
|
const { stateDir, runCwd, home } = ctx;
|
|
@@ -75,10 +138,26 @@ export function resolveIsolation(cfg, ctx) {
|
|
|
75
138
|
mounts.push({ src: p, dst: p, mode: 'rw' }); };
|
|
76
139
|
const addRo = (p) => { if (!mounts.some(m => m.src === p))
|
|
77
140
|
mounts.push({ src: p, dst: p, mode: 'ro' }); };
|
|
141
|
+
/** Writable bind whose destination differs from its source (the per-role home). */
|
|
142
|
+
const addRw2 = (src, dst) => {
|
|
143
|
+
if (!mounts.some(m => m.src === src && m.dst === dst))
|
|
144
|
+
mounts.push({ src, dst, mode: 'rw' });
|
|
145
|
+
};
|
|
78
146
|
// Durable set: state dir + cwd, then only the active harness's config/auth roots.
|
|
79
147
|
addRw(stateDir);
|
|
80
148
|
addRw(runCwd);
|
|
81
|
-
if (ctx.
|
|
149
|
+
if (ctx.harnessHome && ctx.harnessRuntimeDir) {
|
|
150
|
+
// The harness home is backed by a PER-ROLE directory (5.1): the agent gets a
|
|
151
|
+
// writable home for its sessions, caches and history, and anything a future
|
|
152
|
+
// CLI version writes lands there too. The shared credentials, global
|
|
153
|
+
// instructions and configuration are then layered back read-only, so they
|
|
154
|
+
// are readable and cannot be rewritten — for this role or for its peers.
|
|
155
|
+
// Order matters: the writable home must precede the read-only overlays.
|
|
156
|
+
addRw2(ctx.harnessRuntimeDir, ctx.harnessHome);
|
|
157
|
+
for (const p of ctx.harnessSharedPaths ?? [])
|
|
158
|
+
addRo(p);
|
|
159
|
+
}
|
|
160
|
+
else if (ctx.harness === 'codex') {
|
|
82
161
|
addRw(join(home, '.codex'));
|
|
83
162
|
addRo(join(home, '.agents'));
|
|
84
163
|
}
|
|
@@ -105,6 +184,34 @@ export function resolveIsolation(cfg, ctx) {
|
|
|
105
184
|
...SENSITIVE_HOME.map(p => join(home, p)),
|
|
106
185
|
agentsRoot, // sibling agents' state dirs (this agent's own is explicitly mounted)
|
|
107
186
|
];
|
|
187
|
+
// ENFORCE the list, before anything builds a backend argv. Until now it was
|
|
188
|
+
// observational: the allowlist model kept these paths out by default, but a
|
|
189
|
+
// role that asked for one in `fs.write`, `secrets`, or a Codex `add_dirs` got
|
|
190
|
+
// it mounted anyway, and the "blocklist" recorded a guarantee it never made.
|
|
191
|
+
//
|
|
192
|
+
// The role's OWN state dir is the one legitimate descendant of the agents
|
|
193
|
+
// root, so it is excepted by exact canonical identity — which does not
|
|
194
|
+
// exempt its parent, and does not exempt a sibling.
|
|
195
|
+
const forbidden = blocklist.map(canonicalPath);
|
|
196
|
+
const ownStateDir = canonicalPath(stateDir);
|
|
197
|
+
for (const m of mounts) {
|
|
198
|
+
for (const [role, path] of [['source', m.src], ['destination', m.dst]]) {
|
|
199
|
+
const canon = canonicalPath(path);
|
|
200
|
+
// The role's own state dir and anything inside it (its per-role harness
|
|
201
|
+
// runtime home, 5.1) is the one legitimate descendant of the agents root.
|
|
202
|
+
// This does not exempt the root above it, nor a sibling beside it.
|
|
203
|
+
if (within(canon, ownStateDir))
|
|
204
|
+
continue;
|
|
205
|
+
for (let i = 0; i < forbidden.length; i++) {
|
|
206
|
+
const kind = mountConflict(canon, forbidden[i]);
|
|
207
|
+
if (!kind)
|
|
208
|
+
continue;
|
|
209
|
+
const alias = canon === resolve(path) ? '' : ` (resolves to '${canon}')`;
|
|
210
|
+
throw new IsolationPolicyError(`isolation: refusing to mount ${role} '${path}'${alias} — it ` +
|
|
211
|
+
`${CONFLICT_WORDING[kind]} the forbidden path '${blocklist[i]}'`);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
108
215
|
return {
|
|
109
216
|
backend: cfg.backend ?? 'auto',
|
|
110
217
|
onUnavailable: cfg.on_unavailable ?? 'warn',
|
|
@@ -5,9 +5,12 @@ export interface ResourceArgs {
|
|
|
5
5
|
}
|
|
6
6
|
/**
|
|
7
7
|
* Build the `systemd-run --user --scope -p … --` prefix that caps the pane's
|
|
8
|
-
* cgroup-v2 scope. Composed OUTSIDE the sandbox wrap (§5.3/§5.4):
|
|
9
|
-
*
|
|
10
|
-
* only reliable per-agent limit is a transient scope at the pane itself.
|
|
8
|
+
* cgroup-v2 scope. Composed OUTSIDE the sandbox wrap (§5.3/§5.4): a tmux pane is
|
|
9
|
+
* a child of a tmux SERVER rather than of the role's own runner process, so the
|
|
10
|
+
* only reliable per-agent limit is a transient scope at the pane itself. (Since
|
|
11
|
+
* #32 that server is per role rather than fleet-wide, which is what keeps one
|
|
12
|
+
* role's `stop` off every other role's pane — the limit still belongs on the
|
|
13
|
+
* pane.)
|
|
11
14
|
*
|
|
12
15
|
* mem/pids are always enforced (their controllers are delegated to `--user` by
|
|
13
16
|
* default). cpu degrades to a warning when the cpu controller is not delegated.
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { readFileSync } from 'node:fs';
|
|
2
2
|
/**
|
|
3
3
|
* Build the `systemd-run --user --scope -p … --` prefix that caps the pane's
|
|
4
|
-
* cgroup-v2 scope. Composed OUTSIDE the sandbox wrap (§5.3/§5.4):
|
|
5
|
-
*
|
|
6
|
-
* only reliable per-agent limit is a transient scope at the pane itself.
|
|
4
|
+
* cgroup-v2 scope. Composed OUTSIDE the sandbox wrap (§5.3/§5.4): a tmux pane is
|
|
5
|
+
* a child of a tmux SERVER rather than of the role's own runner process, so the
|
|
6
|
+
* only reliable per-agent limit is a transient scope at the pane itself. (Since
|
|
7
|
+
* #32 that server is per role rather than fleet-wide, which is what keeps one
|
|
8
|
+
* role's `stop` off every other role's pane — the limit still belongs on the
|
|
9
|
+
* pane.)
|
|
7
10
|
*
|
|
8
11
|
* mem/pids are always enforced (their controllers are delegated to `--user` by
|
|
9
12
|
* default). cpu degrades to a warning when the cpu controller is not delegated.
|
|
@@ -50,6 +50,19 @@ export interface WrapContext {
|
|
|
50
50
|
harness?: string;
|
|
51
51
|
/** Harness-declared writable roots (for example Codex --add-dir). */
|
|
52
52
|
additionalWriteDirs?: string[];
|
|
53
|
+
/**
|
|
54
|
+
* The harness's home directory on the host (`~/.claude`, `~/.codex`). Mounted
|
|
55
|
+
* from `harnessRuntimeDir` so the agent's own runtime state is per-role (5.1).
|
|
56
|
+
*/
|
|
57
|
+
harnessHome?: string;
|
|
58
|
+
/** Per-role writable directory backing `harnessHome` inside the sandbox. */
|
|
59
|
+
harnessRuntimeDir?: string;
|
|
60
|
+
/**
|
|
61
|
+
* Shared credentials, global instructions and configuration. Mounted READ-ONLY
|
|
62
|
+
* on top of the per-role home, so an agent can read them and cannot rewrite
|
|
63
|
+
* them for itself or for its peers.
|
|
64
|
+
*/
|
|
65
|
+
harnessSharedPaths?: string[];
|
|
53
66
|
brokerEndpoint?: string;
|
|
54
67
|
}
|
|
55
68
|
/**
|
|
@@ -68,7 +81,12 @@ export interface ResolvedIsolation {
|
|
|
68
81
|
system: string[];
|
|
69
82
|
/** ephemeral scratch tmpfs mounts (/tmp, ~/.cache). */
|
|
70
83
|
tmpfs: string[];
|
|
71
|
-
/**
|
|
84
|
+
/**
|
|
85
|
+
* Sensitive host paths that are ENFORCED absent from the sandbox: any mount
|
|
86
|
+
* that is, sits inside, or would expose one of these is refused by
|
|
87
|
+
* `resolveIsolation` before a backend argv is built. Retained on the resolved
|
|
88
|
+
* policy for diagnostics — doctor and `config` report what is being enforced.
|
|
89
|
+
*/
|
|
72
90
|
blocklist: string[];
|
|
73
91
|
}
|
|
74
92
|
/** A pluggable isolation backend (bubblewrap, podman, none). */
|
package/dist/monitor.d.ts
CHANGED
|
@@ -38,14 +38,29 @@ export interface MonitorDeps {
|
|
|
38
38
|
set(fn: () => void, ms: number): ReturnType<typeof setTimeout>;
|
|
39
39
|
clear(t: ReturnType<typeof setTimeout>): void;
|
|
40
40
|
};
|
|
41
|
-
/**
|
|
41
|
+
/**
|
|
42
|
+
* Structured prompt delivery used by ACP sessions. Tmux remains the fallback.
|
|
43
|
+
* `succeeded` is the turn's TERMINAL result, not merely that the session took
|
|
44
|
+
* the prompt: a refused or cancelled wake was seen and not acted on, and must
|
|
45
|
+
* not commit the cursor.
|
|
46
|
+
*/
|
|
42
47
|
delivery?: {
|
|
43
|
-
submit(text: string
|
|
44
|
-
|
|
48
|
+
submit(text: string, options?: {
|
|
49
|
+
interrupt?: boolean;
|
|
50
|
+
}): Promise<{
|
|
51
|
+
succeeded: boolean;
|
|
52
|
+
outcome: string;
|
|
45
53
|
detail?: string;
|
|
46
54
|
}>;
|
|
47
55
|
};
|
|
48
56
|
}
|
|
57
|
+
/**
|
|
58
|
+
* Why the monitor is not healthy. Each cause clears on its OWN recovery signal
|
|
59
|
+
* and nothing else — a successful poll proves the stream works, and proves
|
|
60
|
+
* nothing whatsoever about whether wakes are being delivered or whether the
|
|
61
|
+
* turns they trigger keep dying.
|
|
62
|
+
*/
|
|
63
|
+
export type StatusCause = 'connectivity' | 'delivery' | 'modal' | 'offline' | 'turns-failing' | 'auth';
|
|
49
64
|
/** Best-effort daemon config (issue #17): the fields the MCP client reads. */
|
|
50
65
|
interface DaemonConfig {
|
|
51
66
|
apiToken?: string;
|
|
@@ -127,7 +142,9 @@ export interface MonitorOpts {
|
|
|
127
142
|
}
|
|
128
143
|
/** The lifecycle surface the runner drives: prime pre-launch, run, stop on pid death. */
|
|
129
144
|
export interface MonitorHandle {
|
|
130
|
-
prime(
|
|
145
|
+
prime(options?: {
|
|
146
|
+
resetCursor?: boolean;
|
|
147
|
+
}): Promise<void>;
|
|
131
148
|
run(pid: number): Promise<void>;
|
|
132
149
|
stop(): void;
|
|
133
150
|
}
|
|
@@ -149,9 +166,17 @@ export declare class Monitor {
|
|
|
149
166
|
private currentAbort;
|
|
150
167
|
private apiErrorStreak;
|
|
151
168
|
private readonly turnFailThreshold;
|
|
169
|
+
/** Active degradations, keyed by cause. Empty means armed. */
|
|
170
|
+
private readonly causes;
|
|
152
171
|
constructor(o: MonitorOpts);
|
|
153
|
-
/**
|
|
154
|
-
|
|
172
|
+
/**
|
|
173
|
+
* Resume the last delivered cursor during ordinary fleet-owned restarts.
|
|
174
|
+
* A native→fleet ownership transition resets at stream tip because the native
|
|
175
|
+
* owner was responsible for arrivals while the supervisor was inactive.
|
|
176
|
+
*/
|
|
177
|
+
prime(options?: {
|
|
178
|
+
resetCursor?: boolean;
|
|
179
|
+
}): Promise<void>;
|
|
155
180
|
/** Long-poll → filter → coalesce → inject, until the pane pid dies or stop(). */
|
|
156
181
|
run(pid: number): Promise<void>;
|
|
157
182
|
stop(): void;
|
|
@@ -190,7 +215,19 @@ export declare class Monitor {
|
|
|
190
215
|
private readPersistedCursor;
|
|
191
216
|
/** Atomically persist body-free delivery state; restart always resumes from deliveredCursor. */
|
|
192
217
|
private persistState;
|
|
193
|
-
|
|
218
|
+
/** Record a degradation under its own cause and republish the status. */
|
|
219
|
+
private degrade;
|
|
220
|
+
/**
|
|
221
|
+
* Clear exactly the causes this recovery signal speaks to. Anything else
|
|
222
|
+
* stays: one successful poll must never be able to erase `turns failing`.
|
|
223
|
+
*/
|
|
224
|
+
private recover;
|
|
225
|
+
/**
|
|
226
|
+
* One line per active cause, each dated; `armed` when there are none. Every
|
|
227
|
+
* line carries an ISO timestamp so an operator can tell a live status from a
|
|
228
|
+
* stale one left behind by a monitor that stopped writing.
|
|
229
|
+
*/
|
|
230
|
+
private writeStatus;
|
|
194
231
|
}
|
|
195
232
|
export declare function createMonitor(o: MonitorOpts): Monitor;
|
|
196
233
|
export {};
|
package/dist/monitor.js
CHANGED
|
@@ -234,11 +234,73 @@ export function looksRunning(pane) {
|
|
|
234
234
|
return true; // "(12s · … tokens)" elapsed meter
|
|
235
235
|
return false;
|
|
236
236
|
}
|
|
237
|
-
|
|
237
|
+
const COMPOSER_TOP = /^[^\S\n]*[╭┌][─━]/;
|
|
238
|
+
const COMPOSER_BOTTOM = /^[^\S\n]*[╰└][─━]/;
|
|
239
|
+
const COMPOSER_PROMPT = /^[^\S\n]*(?:[│┃|][^\S\n]*)?[❯›>][^\S\n]*$/;
|
|
240
|
+
/**
|
|
241
|
+
* Remove wrapping-only whitespace and box chrome from composer rows. Notification
|
|
242
|
+
* lines contain no meaningful whitespace distinction, so this lets a fragment
|
|
243
|
+
* cross an arbitrary terminal wrap without matching unrelated transcript text.
|
|
244
|
+
*/
|
|
245
|
+
function normalizeComposerRows(rows) {
|
|
246
|
+
return rows.map((raw, i) => {
|
|
247
|
+
let row = raw;
|
|
248
|
+
if (i > 0)
|
|
249
|
+
row = row.replace(/^[^\S\n]*(?:[│┃|][^\S\n]*)?/, '');
|
|
250
|
+
row = row.replace(/[^\S\n]*(?:[│┃|])?[^\S\n]*$/, '');
|
|
251
|
+
return row;
|
|
252
|
+
}).join('').replace(/\s+/g, '');
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* Is the injected line still sitting unsubmitted in the composer?
|
|
256
|
+
*
|
|
257
|
+
* The footer has variable height and the line may wrap across any number of
|
|
258
|
+
* rows, so a fixed tail window cannot identify the composer. Prefer the final
|
|
259
|
+
* bordered composer region; for a borderless/truncated capture, require the
|
|
260
|
+
* notification prefix to follow a composer prompt. This keeps old submitted
|
|
261
|
+
* wake lines in the transcript from causing stray Enters.
|
|
262
|
+
*/
|
|
238
263
|
function stillInComposer(pane, line) {
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
264
|
+
if (!pane)
|
|
265
|
+
return false; // dead pane: do not waste Enters
|
|
266
|
+
const lines = pane.split('\n');
|
|
267
|
+
let bottom = -1;
|
|
268
|
+
for (let i = lines.length - 1; i >= 0; i--) {
|
|
269
|
+
if (COMPOSER_BOTTOM.test(lines[i])) {
|
|
270
|
+
bottom = i;
|
|
271
|
+
break;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
let top = -1;
|
|
275
|
+
if (bottom >= 0) {
|
|
276
|
+
for (let i = bottom - 1; i >= 0; i--) {
|
|
277
|
+
if (COMPOSER_TOP.test(lines[i])) {
|
|
278
|
+
top = i;
|
|
279
|
+
break;
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
const from = top >= 0 ? top + 1 : 0;
|
|
284
|
+
const to = bottom >= 0 ? bottom : lines.length;
|
|
285
|
+
let wakeRow = -1;
|
|
286
|
+
let wakeColumn = -1;
|
|
287
|
+
for (let i = to - 1; i >= from; i--) {
|
|
288
|
+
const column = lines[i].lastIndexOf(PREFIX);
|
|
289
|
+
if (column >= 0) {
|
|
290
|
+
wakeRow = i;
|
|
291
|
+
wakeColumn = column;
|
|
292
|
+
break;
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
if (wakeRow < 0)
|
|
296
|
+
return false;
|
|
297
|
+
// Without both box boundaries, only trust text visibly in a composer prompt.
|
|
298
|
+
// This is the safe fallback for borderless TUIs and truncated captures.
|
|
299
|
+
if (top < 0 && !COMPOSER_PROMPT.test(lines[wakeRow].slice(0, wakeColumn)))
|
|
300
|
+
return false;
|
|
301
|
+
const rows = lines.slice(wakeRow, to);
|
|
302
|
+
rows[0] = rows[0].slice(wakeColumn);
|
|
303
|
+
return normalizeComposerRows(rows).includes(line.replace(/\s+/g, ''));
|
|
242
304
|
}
|
|
243
305
|
export class Monitor {
|
|
244
306
|
name;
|
|
@@ -260,6 +322,8 @@ export class Monitor {
|
|
|
260
322
|
// ended in an API error with no completed turn in between.
|
|
261
323
|
apiErrorStreak = 0;
|
|
262
324
|
turnFailThreshold;
|
|
325
|
+
/** Active degradations, keyed by cause. Empty means armed. */
|
|
326
|
+
causes = new Map();
|
|
263
327
|
constructor(o) {
|
|
264
328
|
this.name = o.name;
|
|
265
329
|
this.identity = o.identity ?? o.name;
|
|
@@ -272,29 +336,33 @@ export class Monitor {
|
|
|
272
336
|
const n = o.cfg.turn_fail_threshold;
|
|
273
337
|
this.turnFailThreshold = typeof n === 'number' && n >= 1 ? n : DEFAULT_TURN_FAIL_THRESHOLD;
|
|
274
338
|
}
|
|
275
|
-
/**
|
|
276
|
-
|
|
277
|
-
|
|
339
|
+
/**
|
|
340
|
+
* Resume the last delivered cursor during ordinary fleet-owned restarts.
|
|
341
|
+
* A native→fleet ownership transition resets at stream tip because the native
|
|
342
|
+
* owner was responsible for arrivals while the supervisor was inactive.
|
|
343
|
+
*/
|
|
344
|
+
async prime(options = {}) {
|
|
345
|
+
const persisted = options.resetCursor ? null : this.readPersistedCursor();
|
|
278
346
|
if (persisted !== null) {
|
|
279
347
|
this.cursor = persisted;
|
|
280
348
|
this.deliveredCursor = persisted;
|
|
281
|
-
this.
|
|
349
|
+
this.writeStatus();
|
|
282
350
|
return;
|
|
283
351
|
}
|
|
284
352
|
try {
|
|
285
353
|
const body = await this.doFetch('tip', LONGPOLL_TIMEOUT_MS);
|
|
286
354
|
this.cursor = typeof body.cursor === 'number' ? body.cursor : 0;
|
|
287
355
|
this.persistCursor();
|
|
288
|
-
this.
|
|
356
|
+
this.writeStatus();
|
|
289
357
|
}
|
|
290
358
|
catch (e) {
|
|
291
359
|
if (e instanceof AuthError) {
|
|
292
360
|
this.fatal = true;
|
|
293
|
-
this.
|
|
361
|
+
this.degrade('auth', e.message, 'failed');
|
|
294
362
|
}
|
|
295
363
|
else {
|
|
296
364
|
this.cursor = null;
|
|
297
|
-
this.
|
|
365
|
+
this.degrade('connectivity', `prime failed (${msg(e)})`);
|
|
298
366
|
}
|
|
299
367
|
}
|
|
300
368
|
}
|
|
@@ -307,24 +375,26 @@ export class Monitor {
|
|
|
307
375
|
const pending = [];
|
|
308
376
|
while (!this.stopped) {
|
|
309
377
|
if (!this.deps.isAlive(pid)) {
|
|
310
|
-
this.
|
|
378
|
+
this.degrade('offline', 'session offline');
|
|
311
379
|
return;
|
|
312
380
|
}
|
|
313
381
|
let body;
|
|
314
382
|
try {
|
|
315
383
|
body = await this.doFetch(String(this.cursor ?? 0), LONGPOLL_TIMEOUT_MS);
|
|
316
384
|
backoff = 0;
|
|
385
|
+
// A poll that worked proves the stream is healthy — and only that.
|
|
386
|
+
this.recover('connectivity');
|
|
317
387
|
}
|
|
318
388
|
catch (e) {
|
|
319
389
|
if (this.stopped)
|
|
320
390
|
return;
|
|
321
391
|
if (e instanceof AuthError) {
|
|
322
392
|
this.fatal = true;
|
|
323
|
-
this.
|
|
393
|
+
this.degrade('auth', e.message, 'failed');
|
|
324
394
|
return;
|
|
325
395
|
}
|
|
326
396
|
backoff = Math.min(backoff + BACKOFF_STEP_MS, BACKOFF_MAX_MS);
|
|
327
|
-
this.
|
|
397
|
+
this.degrade('connectivity', `stream hiccup (${msg(e)})`);
|
|
328
398
|
await this.deps.sleep(backoff);
|
|
329
399
|
continue;
|
|
330
400
|
}
|
|
@@ -350,7 +420,7 @@ export class Monitor {
|
|
|
350
420
|
accepted = await this.deliver(pid, pending);
|
|
351
421
|
}
|
|
352
422
|
catch (e) {
|
|
353
|
-
this.
|
|
423
|
+
this.degrade('delivery', `delivery failed (${msg(e)})`);
|
|
354
424
|
}
|
|
355
425
|
if (accepted) {
|
|
356
426
|
pending.length = 0;
|
|
@@ -381,20 +451,27 @@ export class Monitor {
|
|
|
381
451
|
async deliver(pid, batch) {
|
|
382
452
|
const line = formatNotificationLine(batch);
|
|
383
453
|
if (this.deps.delivery) {
|
|
384
|
-
const result = await this.deps.delivery.submit(line);
|
|
385
|
-
if (!result.
|
|
386
|
-
|
|
454
|
+
const result = await this.deps.delivery.submit(line, { interrupt: this.cfg.interrupt });
|
|
455
|
+
if (!result.succeeded) {
|
|
456
|
+
// Name the reason: "refused" and "cancelled" are the agent's answer,
|
|
457
|
+
// not a transport problem, and an operator has to be able to tell them
|
|
458
|
+
// apart from a dead socket.
|
|
459
|
+
this.degrade('delivery', `wake ${result.outcome}${result.detail ? ` (${result.detail})` : ''}`);
|
|
387
460
|
return false;
|
|
388
461
|
}
|
|
389
|
-
this.
|
|
462
|
+
this.recover('delivery', 'modal');
|
|
463
|
+
if (result.detail !== 'injected' && result.detail !== 'startedNewTurn')
|
|
464
|
+
this.recordTurn('completed');
|
|
390
465
|
return true;
|
|
391
466
|
}
|
|
467
|
+
if (this.cfg.interrupt)
|
|
468
|
+
await this.deps.tmux.sendKey(this.name, 'C-c');
|
|
392
469
|
const state = await this.awaitInjectable(pid);
|
|
393
470
|
if (state !== 'ready') {
|
|
394
471
|
if (state === 'offline')
|
|
395
|
-
this.
|
|
472
|
+
this.degrade('offline', 'offline during delivery');
|
|
396
473
|
else if (state === 'modal')
|
|
397
|
-
this.
|
|
474
|
+
this.degrade('modal', `modal wedge — pane held a dialog for ` +
|
|
398
475
|
`${MODAL_GIVE_UP_MS / 1000}s, wake not injected`);
|
|
399
476
|
return false;
|
|
400
477
|
}
|
|
@@ -404,11 +481,25 @@ export class Monitor {
|
|
|
404
481
|
// Verify submission for THIS line even if stop() arrives mid-flight: the text
|
|
405
482
|
// is already in the composer and we want it submitted (at-least-once). A truly
|
|
406
483
|
// dead pane makes safeCapture return '' ⇒ not-in-composer ⇒ breaks, no wasted Enter.
|
|
407
|
-
for (let i = 0; i < MAX_ENTER_RETRIES;
|
|
484
|
+
for (let i = 0; i < MAX_ENTER_RETRIES;) {
|
|
408
485
|
await this.deps.sleep(POST_VERIFY_MS);
|
|
409
486
|
const capture = await safeCapture(this.deps.tmux, this.name);
|
|
410
487
|
if (!capture.ok) {
|
|
411
|
-
this.
|
|
488
|
+
this.degrade('delivery', 'capture failed during injection verification');
|
|
489
|
+
return false;
|
|
490
|
+
}
|
|
491
|
+
// A dialog can appear after the initial send. Never let a verification
|
|
492
|
+
// retry confirm it. Wait under the same bounded modal policy as initial
|
|
493
|
+
// injection, then re-capture immediately before considering Enter.
|
|
494
|
+
if (looksModal(capture.pane)) {
|
|
495
|
+
const state = await this.awaitInjectable(pid);
|
|
496
|
+
if (state === 'ready')
|
|
497
|
+
continue;
|
|
498
|
+
if (state === 'offline')
|
|
499
|
+
this.degrade('offline', 'offline during injection verification');
|
|
500
|
+
else if (state === 'modal')
|
|
501
|
+
this.degrade('modal', `modal wedge during injection verification — ` +
|
|
502
|
+
`no Enter sent for ${MODAL_GIVE_UP_MS / 1000}s`);
|
|
412
503
|
return false;
|
|
413
504
|
}
|
|
414
505
|
if (!stillInComposer(capture.pane, line)) {
|
|
@@ -416,11 +507,13 @@ export class Monitor {
|
|
|
416
507
|
break;
|
|
417
508
|
}
|
|
418
509
|
await this.deps.tmux.sendKey(this.name, 'Enter');
|
|
510
|
+
i++;
|
|
419
511
|
}
|
|
420
512
|
if (!delivered) {
|
|
421
|
-
this.
|
|
513
|
+
this.degrade('delivery', 'injection unverified');
|
|
422
514
|
return false;
|
|
423
515
|
}
|
|
516
|
+
this.recover('delivery', 'modal');
|
|
424
517
|
// The wake landed and a turn started; observe how that turn terminates so a
|
|
425
518
|
// refusal-wedge (every turn dies with `API Error:` while delivery stays green)
|
|
426
519
|
// becomes visible in `.monitor-status` instead of masquerading as armed (#19).
|
|
@@ -442,7 +535,7 @@ export class Monitor {
|
|
|
442
535
|
return; // loop marks offline
|
|
443
536
|
const capture = await safeCapture(this.deps.tmux, this.name);
|
|
444
537
|
if (!capture.ok) {
|
|
445
|
-
this.
|
|
538
|
+
this.degrade('delivery', 'capture failed during turn observation');
|
|
446
539
|
return;
|
|
447
540
|
}
|
|
448
541
|
if (looksApiError(capture.pane)) {
|
|
@@ -459,14 +552,17 @@ export class Monitor {
|
|
|
459
552
|
}
|
|
460
553
|
/** Update the consecutive-API-error streak and derive `.monitor-status` from it. */
|
|
461
554
|
recordTurn(outcome) {
|
|
555
|
+
if (outcome === 'inconclusive')
|
|
556
|
+
return; // no evidence either way; leave the streak
|
|
462
557
|
if (outcome === 'api-error')
|
|
463
558
|
this.apiErrorStreak++;
|
|
464
|
-
else
|
|
559
|
+
else
|
|
465
560
|
this.apiErrorStreak = 0;
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
561
|
+
if (this.apiErrorStreak >= this.turnFailThreshold)
|
|
562
|
+
this.degrade('turns-failing', 'turns failing (api error)');
|
|
563
|
+
else if (outcome === 'completed')
|
|
564
|
+
// A turn that ran to the end is the ONLY thing that clears this.
|
|
565
|
+
this.recover('turns-failing');
|
|
470
566
|
}
|
|
471
567
|
/**
|
|
472
568
|
* Reset the composer to empty before typing a wake. Without this, any
|
|
@@ -500,7 +596,7 @@ export class Monitor {
|
|
|
500
596
|
}
|
|
501
597
|
const capture = await safeCapture(this.deps.tmux, this.name);
|
|
502
598
|
if (!capture.ok) {
|
|
503
|
-
this.
|
|
599
|
+
this.degrade('delivery', 'capture failed while checking session readiness');
|
|
504
600
|
await this.deps.sleep(MODAL_RETRY_MS);
|
|
505
601
|
continue;
|
|
506
602
|
}
|
|
@@ -592,15 +688,41 @@ export class Monitor {
|
|
|
592
688
|
this.deps.log(`[${this.name}] monitor: failed to persist state: ${msg(e)}`);
|
|
593
689
|
}
|
|
594
690
|
}
|
|
595
|
-
|
|
691
|
+
/** Record a degradation under its own cause and republish the status. */
|
|
692
|
+
degrade(cause, detail, level = 'degraded') {
|
|
693
|
+
const previous = this.causes.get(cause);
|
|
694
|
+
this.causes.set(cause, { level, detail, at: new Date(this.deps.now()).toISOString() });
|
|
695
|
+
this.writeStatus();
|
|
696
|
+
if (previous?.detail !== detail)
|
|
697
|
+
this.deps.log(`[${this.name}] monitor ${level}: ${cause} — ${detail}`);
|
|
698
|
+
}
|
|
699
|
+
/**
|
|
700
|
+
* Clear exactly the causes this recovery signal speaks to. Anything else
|
|
701
|
+
* stays: one successful poll must never be able to erase `turns failing`.
|
|
702
|
+
*/
|
|
703
|
+
recover(...causes) {
|
|
704
|
+
let changed = false;
|
|
705
|
+
for (const cause of causes)
|
|
706
|
+
changed = this.causes.delete(cause) || changed;
|
|
707
|
+
if (changed)
|
|
708
|
+
this.deps.log(`[${this.name}] monitor recovered: ${causes.join(', ')}`);
|
|
709
|
+
this.writeStatus();
|
|
710
|
+
}
|
|
711
|
+
/**
|
|
712
|
+
* One line per active cause, each dated; `armed` when there are none. Every
|
|
713
|
+
* line carries an ISO timestamp so an operator can tell a live status from a
|
|
714
|
+
* stale one left behind by a monitor that stopped writing.
|
|
715
|
+
*/
|
|
716
|
+
writeStatus() {
|
|
717
|
+
const lines = this.causes.size
|
|
718
|
+
? [...this.causes.entries()].map(([cause, e]) => `${e.level}: ${cause} at ${e.at} — ${e.detail}`)
|
|
719
|
+
: [`armed at ${new Date(this.deps.now()).toISOString()}`];
|
|
596
720
|
try {
|
|
597
|
-
writeFileSync(this.statusPath,
|
|
721
|
+
writeFileSync(this.statusPath, lines.join('\n') + '\n');
|
|
598
722
|
}
|
|
599
723
|
catch (e) {
|
|
600
724
|
this.deps.log(`[${this.name}] monitor: failed to write status: ${msg(e)}`);
|
|
601
725
|
}
|
|
602
|
-
if (!s.startsWith('armed'))
|
|
603
|
-
this.deps.log(`[${this.name}] monitor ${s}`);
|
|
604
726
|
}
|
|
605
727
|
}
|
|
606
728
|
export function createMonitor(o) {
|