@ours.network/fleet 0.10.0-nightly.4 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/README.md +138 -21
  2. package/dist/atomic-file.d.ts +30 -0
  3. package/dist/atomic-file.js +86 -0
  4. package/dist/briefing.d.ts +6 -0
  5. package/dist/briefing.js +43 -13
  6. package/dist/cli.js +98 -22
  7. package/dist/config.d.ts +24 -3
  8. package/dist/config.js +84 -11
  9. package/dist/creation.d.ts +179 -0
  10. package/dist/creation.js +254 -0
  11. package/dist/docs.d.ts +28 -1
  12. package/dist/docs.js +155 -8
  13. package/dist/doctor.js +75 -17
  14. package/dist/harness/claude-code.d.ts +39 -3
  15. package/dist/harness/claude-code.js +128 -26
  16. package/dist/harness/codex.d.ts +7 -1
  17. package/dist/harness/codex.js +58 -11
  18. package/dist/harness/registry.d.ts +2 -0
  19. package/dist/harness/registry.js +19 -0
  20. package/dist/harness/types.d.ts +51 -4
  21. package/dist/isolation/bubblewrap.js +7 -1
  22. package/dist/isolation/policy.d.ts +34 -5
  23. package/dist/isolation/policy.js +114 -7
  24. package/dist/isolation/resources.d.ts +6 -3
  25. package/dist/isolation/resources.js +6 -3
  26. package/dist/isolation/types.d.ts +19 -1
  27. package/dist/monitor.d.ts +33 -4
  28. package/dist/monitor.js +150 -32
  29. package/dist/ops.d.ts +15 -2
  30. package/dist/ops.js +32 -9
  31. package/dist/permissions.d.ts +70 -0
  32. package/dist/permissions.js +97 -0
  33. package/dist/runner.d.ts +65 -2
  34. package/dist/runner.js +262 -27
  35. package/dist/session/acp.d.ts +25 -2
  36. package/dist/session/acp.js +143 -26
  37. package/dist/session/control.d.ts +49 -1
  38. package/dist/session/control.js +116 -12
  39. package/dist/session/tmux.d.ts +9 -2
  40. package/dist/session/tmux.js +36 -4
  41. package/dist/session/types.d.ts +99 -2
  42. package/dist/session/types.js +42 -1
  43. package/dist/spawn.d.ts +27 -2
  44. package/dist/spawn.js +153 -15
  45. package/dist/supervisor/launchd.d.ts +50 -0
  46. package/dist/supervisor/launchd.js +121 -4
  47. package/dist/supervisor/none.js +22 -4
  48. package/dist/supervisor/systemd.d.ts +8 -1
  49. package/dist/supervisor/systemd.js +94 -4
  50. package/dist/supervisor/types.d.ts +36 -3
  51. package/dist/tmux.d.ts +34 -2
  52. package/dist/tmux.js +48 -11
  53. package/package.json +1 -1
@@ -1,5 +1,57 @@
1
- import { join, dirname } from 'node:path';
1
+ import { realpathSync } from 'node:fs';
2
+ import { basename, dirname, join, resolve, sep } from 'node:path';
2
3
  import { BACKENDS, ON_UNAVAILABLE, NETWORK_MODES, } from './types.js';
4
+ /** A mount that the forbidden-path policy refuses. Raised before any launch. */
5
+ export class IsolationPolicyError extends Error {
6
+ constructor(message) {
7
+ super(message);
8
+ this.name = 'IsolationPolicyError';
9
+ }
10
+ }
11
+ /**
12
+ * Resolve a path to its canonical form, following symlinks as far as the
13
+ * filesystem allows and normalising the rest. Without this, `~/link-to-ssh`
14
+ * and `/home/u/.ssh` are different strings for the same directory, and a
15
+ * string comparison against the forbidden list is trivially side-stepped.
16
+ */
17
+ export function canonicalPath(p) {
18
+ const abs = resolve(p);
19
+ let head = abs;
20
+ let tail = '';
21
+ for (;;) {
22
+ try {
23
+ return tail ? join(realpathSync.native(head), tail) : realpathSync.native(head);
24
+ }
25
+ catch {
26
+ const parent = dirname(head);
27
+ if (parent === head)
28
+ return abs; // nothing on this path exists yet
29
+ tail = tail ? join(basename(head), tail) : basename(head);
30
+ head = parent;
31
+ }
32
+ }
33
+ }
34
+ const within = (child, parent) => child === parent || child.startsWith(parent + sep);
35
+ /**
36
+ * How a canonical mount path collides with a canonical forbidden path.
37
+ *
38
+ * `parent` matters as much as the other two: binding `$HOME` does not name
39
+ * `~/.ssh`, but it exposes it just as completely.
40
+ */
41
+ export function mountConflict(mount, forbidden) {
42
+ if (mount === forbidden)
43
+ return 'exact';
44
+ if (within(mount, forbidden))
45
+ return 'descendant';
46
+ if (within(forbidden, mount))
47
+ return 'parent';
48
+ return null;
49
+ }
50
+ const CONFLICT_WORDING = {
51
+ exact: 'is',
52
+ descendant: 'is inside',
53
+ parent: 'would expose',
54
+ };
3
55
  /** Read-only system dirs exposed under the allowlist model. */
4
56
  const SYSTEM_RO = ['/usr', '/bin', '/sbin', '/lib', '/lib64', '/etc'];
5
57
  /** Ephemeral scratch mounts. */
@@ -52,6 +104,12 @@ export function validateIsolationConfig(raw) {
52
104
  }
53
105
  return problems;
54
106
  }
107
+ /**
108
+ * Where a role's per-role harness runtime state lives (5.1). Under the agent's
109
+ * own state directory, so it is covered by the state dir's existing lifecycle
110
+ * and by the forbidden-path exception, and is never shared with a peer.
111
+ */
112
+ export const harnessRuntimeDir = (stateDir, harnessId) => join(stateDir, 'harness', harnessId);
55
113
  /** Parse a `host:container` secret pair; a bare path maps to itself. */
56
114
  function parseSecret(pair) {
57
115
  const i = pair.indexOf(':');
@@ -61,12 +119,17 @@ function parseSecret(pair) {
61
119
  }
62
120
  /**
63
121
  * Resolve a raw (already validated) isolation block against runtime context into
64
- * a defaults-filled, backend-agnostic policy. Pure no I/O, no probing.
122
+ * a defaults-filled, backend-agnostic policy, and REFUSE any mount that would
123
+ * breach the forbidden-path list.
124
+ *
125
+ * The mount model is an allowlist: only the durable set (state dir, cwd, harness
126
+ * config, declared fs/secrets) plus read-only system dirs are exposed. The
127
+ * forbidden list — the ours key store, sibling agent state dirs, ~/.ssh, ~/.aws
128
+ * — is now enforced on top of that, so a role cannot ask its way back in.
65
129
  *
66
- * The mount model is an allowlist: only the durable set (state dir, cwd, Claude
67
- * config, declared fs/secrets) plus read-only system dirs are exposed; everything
68
- * else on the host the ours key store, sibling agent state dirs, ~/.ssh, ~/.aws —
69
- * is simply never mounted, and thus absent inside the sandbox (§5.2).
130
+ * Not pure: canonicalising a path reads the filesystem, because symlink aliases
131
+ * are one of the ways a forbidden path gets requested. Throws
132
+ * `IsolationPolicyError`; callers surface it against the role.
70
133
  */
71
134
  export function resolveIsolation(cfg, ctx) {
72
135
  const { stateDir, runCwd, home } = ctx;
@@ -75,10 +138,26 @@ export function resolveIsolation(cfg, ctx) {
75
138
  mounts.push({ src: p, dst: p, mode: 'rw' }); };
76
139
  const addRo = (p) => { if (!mounts.some(m => m.src === p))
77
140
  mounts.push({ src: p, dst: p, mode: 'ro' }); };
141
+ /** Writable bind whose destination differs from its source (the per-role home). */
142
+ const addRw2 = (src, dst) => {
143
+ if (!mounts.some(m => m.src === src && m.dst === dst))
144
+ mounts.push({ src, dst, mode: 'rw' });
145
+ };
78
146
  // Durable set: state dir + cwd, then only the active harness's config/auth roots.
79
147
  addRw(stateDir);
80
148
  addRw(runCwd);
81
- if (ctx.harness === 'codex') {
149
+ if (ctx.harnessHome && ctx.harnessRuntimeDir) {
150
+ // The harness home is backed by a PER-ROLE directory (5.1): the agent gets a
151
+ // writable home for its sessions, caches and history, and anything a future
152
+ // CLI version writes lands there too. The shared credentials, global
153
+ // instructions and configuration are then layered back read-only, so they
154
+ // are readable and cannot be rewritten — for this role or for its peers.
155
+ // Order matters: the writable home must precede the read-only overlays.
156
+ addRw2(ctx.harnessRuntimeDir, ctx.harnessHome);
157
+ for (const p of ctx.harnessSharedPaths ?? [])
158
+ addRo(p);
159
+ }
160
+ else if (ctx.harness === 'codex') {
82
161
  addRw(join(home, '.codex'));
83
162
  addRo(join(home, '.agents'));
84
163
  }
@@ -105,6 +184,34 @@ export function resolveIsolation(cfg, ctx) {
105
184
  ...SENSITIVE_HOME.map(p => join(home, p)),
106
185
  agentsRoot, // sibling agents' state dirs (this agent's own is explicitly mounted)
107
186
  ];
187
+ // ENFORCE the list, before anything builds a backend argv. Until now it was
188
+ // observational: the allowlist model kept these paths out by default, but a
189
+ // role that asked for one in `fs.write`, `secrets`, or a Codex `add_dirs` got
190
+ // it mounted anyway, and the "blocklist" recorded a guarantee it never made.
191
+ //
192
+ // The role's OWN state dir is the one legitimate descendant of the agents
193
+ // root, so it is excepted by exact canonical identity — which does not
194
+ // exempt its parent, and does not exempt a sibling.
195
+ const forbidden = blocklist.map(canonicalPath);
196
+ const ownStateDir = canonicalPath(stateDir);
197
+ for (const m of mounts) {
198
+ for (const [role, path] of [['source', m.src], ['destination', m.dst]]) {
199
+ const canon = canonicalPath(path);
200
+ // The role's own state dir and anything inside it (its per-role harness
201
+ // runtime home, 5.1) is the one legitimate descendant of the agents root.
202
+ // This does not exempt the root above it, nor a sibling beside it.
203
+ if (within(canon, ownStateDir))
204
+ continue;
205
+ for (let i = 0; i < forbidden.length; i++) {
206
+ const kind = mountConflict(canon, forbidden[i]);
207
+ if (!kind)
208
+ continue;
209
+ const alias = canon === resolve(path) ? '' : ` (resolves to '${canon}')`;
210
+ throw new IsolationPolicyError(`isolation: refusing to mount ${role} '${path}'${alias} — it ` +
211
+ `${CONFLICT_WORDING[kind]} the forbidden path '${blocklist[i]}'`);
212
+ }
213
+ }
214
+ }
108
215
  return {
109
216
  backend: cfg.backend ?? 'auto',
110
217
  onUnavailable: cfg.on_unavailable ?? 'warn',
@@ -5,9 +5,12 @@ export interface ResourceArgs {
5
5
  }
6
6
  /**
7
7
  * Build the `systemd-run --user --scope -p … --` prefix that caps the pane's
8
- * cgroup-v2 scope. Composed OUTSIDE the sandbox wrap (§5.3/§5.4): because tmux
9
- * panes are children of the shared tmux server rather than the per-role unit, the
10
- * only reliable per-agent limit is a transient scope at the pane itself.
8
+ * cgroup-v2 scope. Composed OUTSIDE the sandbox wrap (§5.3/§5.4): a tmux pane is
9
+ * a child of a tmux SERVER rather than of the role's own runner process, so the
10
+ * only reliable per-agent limit is a transient scope at the pane itself. (Since
11
+ * #32 that server is per role rather than fleet-wide, which is what keeps one
12
+ * role's `stop` off every other role's pane — the limit still belongs on the
13
+ * pane.)
11
14
  *
12
15
  * mem/pids are always enforced (their controllers are delegated to `--user` by
13
16
  * default). cpu degrades to a warning when the cpu controller is not delegated.
@@ -1,9 +1,12 @@
1
1
  import { readFileSync } from 'node:fs';
2
2
  /**
3
3
  * Build the `systemd-run --user --scope -p … --` prefix that caps the pane's
4
- * cgroup-v2 scope. Composed OUTSIDE the sandbox wrap (§5.3/§5.4): because tmux
5
- * panes are children of the shared tmux server rather than the per-role unit, the
6
- * only reliable per-agent limit is a transient scope at the pane itself.
4
+ * cgroup-v2 scope. Composed OUTSIDE the sandbox wrap (§5.3/§5.4): a tmux pane is
5
+ * a child of a tmux SERVER rather than of the role's own runner process, so the
6
+ * only reliable per-agent limit is a transient scope at the pane itself. (Since
7
+ * #32 that server is per role rather than fleet-wide, which is what keeps one
8
+ * role's `stop` off every other role's pane — the limit still belongs on the
9
+ * pane.)
7
10
  *
8
11
  * mem/pids are always enforced (their controllers are delegated to `--user` by
9
12
  * default). cpu degrades to a warning when the cpu controller is not delegated.
@@ -50,6 +50,19 @@ export interface WrapContext {
50
50
  harness?: string;
51
51
  /** Harness-declared writable roots (for example Codex --add-dir). */
52
52
  additionalWriteDirs?: string[];
53
+ /**
54
+ * The harness's home directory on the host (`~/.claude`, `~/.codex`). Mounted
55
+ * from `harnessRuntimeDir` so the agent's own runtime state is per-role (5.1).
56
+ */
57
+ harnessHome?: string;
58
+ /** Per-role writable directory backing `harnessHome` inside the sandbox. */
59
+ harnessRuntimeDir?: string;
60
+ /**
61
+ * Shared credentials, global instructions and configuration. Mounted READ-ONLY
62
+ * on top of the per-role home, so an agent can read them and cannot rewrite
63
+ * them for itself or for its peers.
64
+ */
65
+ harnessSharedPaths?: string[];
53
66
  brokerEndpoint?: string;
54
67
  }
55
68
  /**
@@ -68,7 +81,12 @@ export interface ResolvedIsolation {
68
81
  system: string[];
69
82
  /** ephemeral scratch tmpfs mounts (/tmp, ~/.cache). */
70
83
  tmpfs: string[];
71
- /** sensitive host paths guaranteed absent from the sandbox (observability). */
84
+ /**
85
+ * Sensitive host paths that are ENFORCED absent from the sandbox: any mount
86
+ * that is, sits inside, or would expose one of these is refused by
87
+ * `resolveIsolation` before a backend argv is built. Retained on the resolved
88
+ * policy for diagnostics — doctor and `config` report what is being enforced.
89
+ */
72
90
  blocklist: string[];
73
91
  }
74
92
  /** A pluggable isolation backend (bubblewrap, podman, none). */
package/dist/monitor.d.ts CHANGED
@@ -38,14 +38,29 @@ export interface MonitorDeps {
38
38
  set(fn: () => void, ms: number): ReturnType<typeof setTimeout>;
39
39
  clear(t: ReturnType<typeof setTimeout>): void;
40
40
  };
41
- /** Structured prompt delivery used by ACP sessions. Tmux remains the fallback. */
41
+ /**
42
+ * Structured prompt delivery used by ACP sessions. Tmux remains the fallback.
43
+ * `succeeded` is the turn's TERMINAL result, not merely that the session took
44
+ * the prompt: a refused or cancelled wake was seen and not acted on, and must
45
+ * not commit the cursor.
46
+ */
42
47
  delivery?: {
43
- submit(text: string): Promise<{
44
- accepted: boolean;
48
+ submit(text: string, options?: {
49
+ interrupt?: boolean;
50
+ }): Promise<{
51
+ succeeded: boolean;
52
+ outcome: string;
45
53
  detail?: string;
46
54
  }>;
47
55
  };
48
56
  }
57
+ /**
58
+ * Why the monitor is not healthy. Each cause clears on its OWN recovery signal
59
+ * and nothing else — a successful poll proves the stream works, and proves
60
+ * nothing whatsoever about whether wakes are being delivered or whether the
61
+ * turns they trigger keep dying.
62
+ */
63
+ export type StatusCause = 'connectivity' | 'delivery' | 'modal' | 'offline' | 'turns-failing' | 'auth';
49
64
  /** Best-effort daemon config (issue #17): the fields the MCP client reads. */
50
65
  interface DaemonConfig {
51
66
  apiToken?: string;
@@ -149,6 +164,8 @@ export declare class Monitor {
149
164
  private currentAbort;
150
165
  private apiErrorStreak;
151
166
  private readonly turnFailThreshold;
167
+ /** Active degradations, keyed by cause. Empty means armed. */
168
+ private readonly causes;
152
169
  constructor(o: MonitorOpts);
153
170
  /** Resume the last delivered cursor; only a brand-new monitor primes at stream tip. */
154
171
  prime(): Promise<void>;
@@ -190,7 +207,19 @@ export declare class Monitor {
190
207
  private readPersistedCursor;
191
208
  /** Atomically persist body-free delivery state; restart always resumes from deliveredCursor. */
192
209
  private persistState;
193
- private setStatus;
210
+ /** Record a degradation under its own cause and republish the status. */
211
+ private degrade;
212
+ /**
213
+ * Clear exactly the causes this recovery signal speaks to. Anything else
214
+ * stays: one successful poll must never be able to erase `turns failing`.
215
+ */
216
+ private recover;
217
+ /**
218
+ * One line per active cause, each dated; `armed` when there are none. Every
219
+ * line carries an ISO timestamp so an operator can tell a live status from a
220
+ * stale one left behind by a monitor that stopped writing.
221
+ */
222
+ private writeStatus;
194
223
  }
195
224
  export declare function createMonitor(o: MonitorOpts): Monitor;
196
225
  export {};
package/dist/monitor.js CHANGED
@@ -234,11 +234,73 @@ export function looksRunning(pane) {
234
234
  return true; // "(12s · … tokens)" elapsed meter
235
235
  return false;
236
236
  }
237
- /** Is the injected line still sitting unsubmitted in the composer (bottom of pane)? */
237
+ const COMPOSER_TOP = /^[^\S\n]*[╭┌][─━]/;
238
+ const COMPOSER_BOTTOM = /^[^\S\n]*[╰└][─━]/;
239
+ const COMPOSER_PROMPT = /^[^\S\n]*(?:[│┃|][^\S\n]*)?[❯›>][^\S\n]*$/;
240
+ /**
241
+ * Remove wrapping-only whitespace and box chrome from composer rows. Notification
242
+ * lines contain no meaningful whitespace distinction, so this lets a fragment
243
+ * cross an arbitrary terminal wrap without matching unrelated transcript text.
244
+ */
245
+ function normalizeComposerRows(rows) {
246
+ return rows.map((raw, i) => {
247
+ let row = raw;
248
+ if (i > 0)
249
+ row = row.replace(/^[^\S\n]*(?:[│┃|][^\S\n]*)?/, '');
250
+ row = row.replace(/[^\S\n]*(?:[│┃|])?[^\S\n]*$/, '');
251
+ return row;
252
+ }).join('').replace(/\s+/g, '');
253
+ }
254
+ /**
255
+ * Is the injected line still sitting unsubmitted in the composer?
256
+ *
257
+ * The footer has variable height and the line may wrap across any number of
258
+ * rows, so a fixed tail window cannot identify the composer. Prefer the final
259
+ * bordered composer region; for a borderless/truncated capture, require the
260
+ * notification prefix to follow a composer prompt. This keeps old submitted
261
+ * wake lines in the transcript from causing stray Enters.
262
+ */
238
263
  function stillInComposer(pane, line) {
239
- const frag = line.slice(0, 48);
240
- const tail = pane.split('\n').slice(-4).join('\n');
241
- return tail.includes(frag);
264
+ if (!pane)
265
+ return false; // dead pane: do not waste Enters
266
+ const lines = pane.split('\n');
267
+ let bottom = -1;
268
+ for (let i = lines.length - 1; i >= 0; i--) {
269
+ if (COMPOSER_BOTTOM.test(lines[i])) {
270
+ bottom = i;
271
+ break;
272
+ }
273
+ }
274
+ let top = -1;
275
+ if (bottom >= 0) {
276
+ for (let i = bottom - 1; i >= 0; i--) {
277
+ if (COMPOSER_TOP.test(lines[i])) {
278
+ top = i;
279
+ break;
280
+ }
281
+ }
282
+ }
283
+ const from = top >= 0 ? top + 1 : 0;
284
+ const to = bottom >= 0 ? bottom : lines.length;
285
+ let wakeRow = -1;
286
+ let wakeColumn = -1;
287
+ for (let i = to - 1; i >= from; i--) {
288
+ const column = lines[i].lastIndexOf(PREFIX);
289
+ if (column >= 0) {
290
+ wakeRow = i;
291
+ wakeColumn = column;
292
+ break;
293
+ }
294
+ }
295
+ if (wakeRow < 0)
296
+ return false;
297
+ // Without both box boundaries, only trust text visibly in a composer prompt.
298
+ // This is the safe fallback for borderless TUIs and truncated captures.
299
+ if (top < 0 && !COMPOSER_PROMPT.test(lines[wakeRow].slice(0, wakeColumn)))
300
+ return false;
301
+ const rows = lines.slice(wakeRow, to);
302
+ rows[0] = rows[0].slice(wakeColumn);
303
+ return normalizeComposerRows(rows).includes(line.replace(/\s+/g, ''));
242
304
  }
243
305
  export class Monitor {
244
306
  name;
@@ -260,6 +322,8 @@ export class Monitor {
260
322
  // ended in an API error with no completed turn in between.
261
323
  apiErrorStreak = 0;
262
324
  turnFailThreshold;
325
+ /** Active degradations, keyed by cause. Empty means armed. */
326
+ causes = new Map();
263
327
  constructor(o) {
264
328
  this.name = o.name;
265
329
  this.identity = o.identity ?? o.name;
@@ -278,23 +342,23 @@ export class Monitor {
278
342
  if (persisted !== null) {
279
343
  this.cursor = persisted;
280
344
  this.deliveredCursor = persisted;
281
- this.setStatus('armed');
345
+ this.writeStatus();
282
346
  return;
283
347
  }
284
348
  try {
285
349
  const body = await this.doFetch('tip', LONGPOLL_TIMEOUT_MS);
286
350
  this.cursor = typeof body.cursor === 'number' ? body.cursor : 0;
287
351
  this.persistCursor();
288
- this.setStatus('armed');
352
+ this.writeStatus();
289
353
  }
290
354
  catch (e) {
291
355
  if (e instanceof AuthError) {
292
356
  this.fatal = true;
293
- this.setStatus(`failed: ${e.message}`);
357
+ this.degrade('auth', e.message, 'failed');
294
358
  }
295
359
  else {
296
360
  this.cursor = null;
297
- this.setStatus(`degraded: prime failed (${msg(e)})`);
361
+ this.degrade('connectivity', `prime failed (${msg(e)})`);
298
362
  }
299
363
  }
300
364
  }
@@ -307,24 +371,26 @@ export class Monitor {
307
371
  const pending = [];
308
372
  while (!this.stopped) {
309
373
  if (!this.deps.isAlive(pid)) {
310
- this.setStatus('degraded: session offline');
374
+ this.degrade('offline', 'session offline');
311
375
  return;
312
376
  }
313
377
  let body;
314
378
  try {
315
379
  body = await this.doFetch(String(this.cursor ?? 0), LONGPOLL_TIMEOUT_MS);
316
380
  backoff = 0;
381
+ // A poll that worked proves the stream is healthy — and only that.
382
+ this.recover('connectivity');
317
383
  }
318
384
  catch (e) {
319
385
  if (this.stopped)
320
386
  return;
321
387
  if (e instanceof AuthError) {
322
388
  this.fatal = true;
323
- this.setStatus(`failed: ${e.message}`);
389
+ this.degrade('auth', e.message, 'failed');
324
390
  return;
325
391
  }
326
392
  backoff = Math.min(backoff + BACKOFF_STEP_MS, BACKOFF_MAX_MS);
327
- this.setStatus(`degraded: stream hiccup (${msg(e)})`);
393
+ this.degrade('connectivity', `stream hiccup (${msg(e)})`);
328
394
  await this.deps.sleep(backoff);
329
395
  continue;
330
396
  }
@@ -350,7 +416,7 @@ export class Monitor {
350
416
  accepted = await this.deliver(pid, pending);
351
417
  }
352
418
  catch (e) {
353
- this.setStatus(`degraded: delivery failed (${msg(e)})`);
419
+ this.degrade('delivery', `delivery failed (${msg(e)})`);
354
420
  }
355
421
  if (accepted) {
356
422
  pending.length = 0;
@@ -381,20 +447,27 @@ export class Monitor {
381
447
  async deliver(pid, batch) {
382
448
  const line = formatNotificationLine(batch);
383
449
  if (this.deps.delivery) {
384
- const result = await this.deps.delivery.submit(line);
385
- if (!result.accepted) {
386
- this.setStatus(`degraded: ACP prompt not accepted${result.detail ? ` (${result.detail})` : ''}`);
450
+ const result = await this.deps.delivery.submit(line, { interrupt: this.cfg.interrupt });
451
+ if (!result.succeeded) {
452
+ // Name the reason: "refused" and "cancelled" are the agent's answer,
453
+ // not a transport problem, and an operator has to be able to tell them
454
+ // apart from a dead socket.
455
+ this.degrade('delivery', `wake ${result.outcome}${result.detail ? ` (${result.detail})` : ''}`);
387
456
  return false;
388
457
  }
389
- this.recordTurn('completed');
458
+ this.recover('delivery', 'modal');
459
+ if (result.detail !== 'injected' && result.detail !== 'startedNewTurn')
460
+ this.recordTurn('completed');
390
461
  return true;
391
462
  }
463
+ if (this.cfg.interrupt)
464
+ await this.deps.tmux.sendKey(this.name, 'C-c');
392
465
  const state = await this.awaitInjectable(pid);
393
466
  if (state !== 'ready') {
394
467
  if (state === 'offline')
395
- this.setStatus('degraded: offline during delivery');
468
+ this.degrade('offline', 'offline during delivery');
396
469
  else if (state === 'modal')
397
- this.setStatus(`degraded: modal wedge — pane held a dialog for ` +
470
+ this.degrade('modal', `modal wedge — pane held a dialog for ` +
398
471
  `${MODAL_GIVE_UP_MS / 1000}s, wake not injected`);
399
472
  return false;
400
473
  }
@@ -404,11 +477,25 @@ export class Monitor {
404
477
  // Verify submission for THIS line even if stop() arrives mid-flight: the text
405
478
  // is already in the composer and we want it submitted (at-least-once). A truly
406
479
  // dead pane makes safeCapture return '' ⇒ not-in-composer ⇒ breaks, no wasted Enter.
407
- for (let i = 0; i < MAX_ENTER_RETRIES; i++) {
480
+ for (let i = 0; i < MAX_ENTER_RETRIES;) {
408
481
  await this.deps.sleep(POST_VERIFY_MS);
409
482
  const capture = await safeCapture(this.deps.tmux, this.name);
410
483
  if (!capture.ok) {
411
- this.setStatus('degraded: capture failed during injection verification');
484
+ this.degrade('delivery', 'capture failed during injection verification');
485
+ return false;
486
+ }
487
+ // A dialog can appear after the initial send. Never let a verification
488
+ // retry confirm it. Wait under the same bounded modal policy as initial
489
+ // injection, then re-capture immediately before considering Enter.
490
+ if (looksModal(capture.pane)) {
491
+ const state = await this.awaitInjectable(pid);
492
+ if (state === 'ready')
493
+ continue;
494
+ if (state === 'offline')
495
+ this.degrade('offline', 'offline during injection verification');
496
+ else if (state === 'modal')
497
+ this.degrade('modal', `modal wedge during injection verification — ` +
498
+ `no Enter sent for ${MODAL_GIVE_UP_MS / 1000}s`);
412
499
  return false;
413
500
  }
414
501
  if (!stillInComposer(capture.pane, line)) {
@@ -416,11 +503,13 @@ export class Monitor {
416
503
  break;
417
504
  }
418
505
  await this.deps.tmux.sendKey(this.name, 'Enter');
506
+ i++;
419
507
  }
420
508
  if (!delivered) {
421
- this.setStatus('degraded: injection unverified');
509
+ this.degrade('delivery', 'injection unverified');
422
510
  return false;
423
511
  }
512
+ this.recover('delivery', 'modal');
424
513
  // The wake landed and a turn started; observe how that turn terminates so a
425
514
  // refusal-wedge (every turn dies with `API Error:` while delivery stays green)
426
515
  // becomes visible in `.monitor-status` instead of masquerading as armed (#19).
@@ -442,7 +531,7 @@ export class Monitor {
442
531
  return; // loop marks offline
443
532
  const capture = await safeCapture(this.deps.tmux, this.name);
444
533
  if (!capture.ok) {
445
- this.setStatus('degraded: capture failed during turn observation');
534
+ this.degrade('delivery', 'capture failed during turn observation');
446
535
  return;
447
536
  }
448
537
  if (looksApiError(capture.pane)) {
@@ -459,14 +548,17 @@ export class Monitor {
459
548
  }
460
549
  /** Update the consecutive-API-error streak and derive `.monitor-status` from it. */
461
550
  recordTurn(outcome) {
551
+ if (outcome === 'inconclusive')
552
+ return; // no evidence either way; leave the streak
462
553
  if (outcome === 'api-error')
463
554
  this.apiErrorStreak++;
464
- else if (outcome === 'completed')
555
+ else
465
556
  this.apiErrorStreak = 0;
466
- // 'inconclusive' leaves the streak (and therefore the status) unchanged.
467
- this.setStatus(this.apiErrorStreak >= this.turnFailThreshold
468
- ? 'degraded: turns failing (api error)'
469
- : 'armed');
557
+ if (this.apiErrorStreak >= this.turnFailThreshold)
558
+ this.degrade('turns-failing', 'turns failing (api error)');
559
+ else if (outcome === 'completed')
560
+ // A turn that ran to the end is the ONLY thing that clears this.
561
+ this.recover('turns-failing');
470
562
  }
471
563
  /**
472
564
  * Reset the composer to empty before typing a wake. Without this, any
@@ -500,7 +592,7 @@ export class Monitor {
500
592
  }
501
593
  const capture = await safeCapture(this.deps.tmux, this.name);
502
594
  if (!capture.ok) {
503
- this.setStatus('degraded: capture failed while checking session readiness');
595
+ this.degrade('delivery', 'capture failed while checking session readiness');
504
596
  await this.deps.sleep(MODAL_RETRY_MS);
505
597
  continue;
506
598
  }
@@ -592,15 +684,41 @@ export class Monitor {
592
684
  this.deps.log(`[${this.name}] monitor: failed to persist state: ${msg(e)}`);
593
685
  }
594
686
  }
595
- setStatus(s) {
687
+ /** Record a degradation under its own cause and republish the status. */
688
+ degrade(cause, detail, level = 'degraded') {
689
+ const previous = this.causes.get(cause);
690
+ this.causes.set(cause, { level, detail, at: new Date(this.deps.now()).toISOString() });
691
+ this.writeStatus();
692
+ if (previous?.detail !== detail)
693
+ this.deps.log(`[${this.name}] monitor ${level}: ${cause} — ${detail}`);
694
+ }
695
+ /**
696
+ * Clear exactly the causes this recovery signal speaks to. Anything else
697
+ * stays: one successful poll must never be able to erase `turns failing`.
698
+ */
699
+ recover(...causes) {
700
+ let changed = false;
701
+ for (const cause of causes)
702
+ changed = this.causes.delete(cause) || changed;
703
+ if (changed)
704
+ this.deps.log(`[${this.name}] monitor recovered: ${causes.join(', ')}`);
705
+ this.writeStatus();
706
+ }
707
+ /**
708
+ * One line per active cause, each dated; `armed` when there are none. Every
709
+ * line carries an ISO timestamp so an operator can tell a live status from a
710
+ * stale one left behind by a monitor that stopped writing.
711
+ */
712
+ writeStatus() {
713
+ const lines = this.causes.size
714
+ ? [...this.causes.entries()].map(([cause, e]) => `${e.level}: ${cause} at ${e.at} — ${e.detail}`)
715
+ : [`armed at ${new Date(this.deps.now()).toISOString()}`];
596
716
  try {
597
- writeFileSync(this.statusPath, `${s}\n`);
717
+ writeFileSync(this.statusPath, lines.join('\n') + '\n');
598
718
  }
599
719
  catch (e) {
600
720
  this.deps.log(`[${this.name}] monitor: failed to write status: ${msg(e)}`);
601
721
  }
602
- if (!s.startsWith('armed'))
603
- this.deps.log(`[${this.name}] monitor ${s}`);
604
722
  }
605
723
  }
606
724
  export function createMonitor(o) {
package/dist/ops.d.ts CHANGED
@@ -1,18 +1,31 @@
1
1
  import type { FleetConfig, ResolvedRole } from './config.js';
2
- import type { SupervisorBackend } from './supervisor/types.js';
2
+ import type { InstallOutcome as BackendInstallOutcome, SupervisorBackend } from './supervisor/types.js';
3
+ /** An install outcome tagged with the role it belongs to. */
4
+ export interface InstallOutcome extends BackendInstallOutcome {
5
+ role: string;
6
+ }
3
7
  export interface OpsDeps {
4
8
  backend: SupervisorBackend;
5
9
  binPath: string;
6
10
  log(line: string): void;
11
+ /**
12
+ * Called the INSTANT a registration is created, before anything else can
13
+ * fail. A creation transaction that learns about registrations only from
14
+ * `up()`'s return value learns nothing when `up()` throws — and the service
15
+ * it just registered is then invisible to rollback (6.2). Optional: plain
16
+ * `ours-fleet up` has no transaction to tell.
17
+ */
18
+ onInstalled?(outcome: InstallOutcome): void;
7
19
  }
8
20
  /** Materialize a role's state dir from config: briefing + markers. Returns the dir. */
9
21
  export declare function applyRole(role: ResolvedRole, opts?: {
10
22
  fresh?: boolean;
11
23
  temp?: boolean;
12
24
  configPath?: string;
25
+ identityGuarantee?: 'verified' | 'created' | 'unverified';
13
26
  }): string;
14
27
  /** Create/start roles declaratively. Idempotent; active roles keep their context. */
15
- export declare function up(cfg: FleetConfig, names: string[], deps: OpsDeps, configPath?: string): Promise<void>;
28
+ export declare function up(cfg: FleetConfig, names: string[], deps: OpsDeps, configPath?: string, identityGuarantee?: 'verified' | 'created' | 'unverified'): Promise<InstallOutcome[]>;
16
29
  export declare function down(cfg: FleetConfig, names: string[], deps: OpsDeps): Promise<void>;
17
30
  /** Re-sync from config + bounce. mode 'keep' resumes context; 'fresh' wipes it. */
18
31
  export declare function restartRoles(cfg: FleetConfig, names: string[], deps: OpsDeps, mode: 'keep' | 'fresh', configPath?: string): Promise<void>;