@ours.network/fleet 0.9.4 → 0.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +148 -30
- package/dist/atomic-file.d.ts +30 -0
- package/dist/atomic-file.js +86 -0
- package/dist/briefing.d.ts +6 -0
- package/dist/briefing.js +41 -11
- package/dist/cli.js +238 -26
- package/dist/config.d.ts +39 -1
- package/dist/config.js +126 -3
- package/dist/creation.d.ts +179 -0
- package/dist/creation.js +254 -0
- package/dist/docs.d.ts +34 -0
- package/dist/docs.js +309 -0
- package/dist/doctor.js +123 -21
- package/dist/harness/acp-agent.d.ts +11 -0
- package/dist/harness/acp-agent.js +27 -0
- package/dist/harness/claude-code.d.ts +39 -3
- package/dist/harness/claude-code.js +145 -13
- package/dist/harness/codex.d.ts +7 -1
- package/dist/harness/codex.js +89 -4
- package/dist/harness/registry.d.ts +2 -0
- package/dist/harness/registry.js +19 -0
- package/dist/harness/types.d.ts +59 -1
- package/dist/index.d.ts +6 -3
- package/dist/index.js +3 -1
- package/dist/isolation/bubblewrap.js +7 -1
- package/dist/isolation/policy.d.ts +34 -5
- package/dist/isolation/policy.js +114 -7
- package/dist/isolation/resources.d.ts +6 -3
- package/dist/isolation/resources.js +6 -3
- package/dist/isolation/types.d.ts +19 -1
- package/dist/monitor.d.ts +44 -2
- package/dist/monitor.js +177 -42
- package/dist/ops.d.ts +15 -2
- package/dist/ops.js +32 -9
- package/dist/permissions.d.ts +70 -0
- package/dist/permissions.js +97 -0
- package/dist/runner.d.ts +65 -2
- package/dist/runner.js +307 -32
- package/dist/session/acp.d.ts +70 -0
- package/dist/session/acp.js +364 -0
- package/dist/session/control.d.ts +89 -0
- package/dist/session/control.js +322 -0
- package/dist/session/events.d.ts +14 -0
- package/dist/session/events.js +67 -0
- package/dist/session/tmux.d.ts +27 -0
- package/dist/session/tmux.js +76 -0
- package/dist/session/types.d.ts +138 -0
- package/dist/session/types.js +42 -0
- package/dist/spawn.d.ts +32 -2
- package/dist/spawn.js +177 -16
- package/dist/supervisor/launchd.d.ts +50 -0
- package/dist/supervisor/launchd.js +121 -4
- package/dist/supervisor/none.js +22 -4
- package/dist/supervisor/systemd.d.ts +8 -1
- package/dist/supervisor/systemd.js +94 -4
- package/dist/supervisor/types.d.ts +36 -3
- package/dist/tmux.d.ts +34 -2
- package/dist/tmux.js +48 -11
- package/package.json +7 -2
package/dist/monitor.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { existsSync, readFileSync, writeFileSync } from 'node:fs';
|
|
1
|
+
import { existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
|
|
2
2
|
import { homedir } from 'node:os';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
4
|
// Code constants (not config — YAGNI, design §2).
|
|
@@ -242,12 +242,16 @@ function stillInComposer(pane, line) {
|
|
|
242
242
|
}
|
|
243
243
|
export class Monitor {
|
|
244
244
|
name;
|
|
245
|
+
identity;
|
|
245
246
|
cfg;
|
|
246
247
|
deps;
|
|
247
248
|
ep;
|
|
248
249
|
statusPath;
|
|
249
250
|
cursorPath;
|
|
251
|
+
statePath;
|
|
250
252
|
cursor = null;
|
|
253
|
+
deliveredCursor = null;
|
|
254
|
+
pendingState = null;
|
|
251
255
|
fatal = false;
|
|
252
256
|
stopped = false;
|
|
253
257
|
bootDeadline = 0;
|
|
@@ -256,32 +260,43 @@ export class Monitor {
|
|
|
256
260
|
// ended in an API error with no completed turn in between.
|
|
257
261
|
apiErrorStreak = 0;
|
|
258
262
|
turnFailThreshold;
|
|
263
|
+
/** Active degradations, keyed by cause. Empty means armed. */
|
|
264
|
+
causes = new Map();
|
|
259
265
|
constructor(o) {
|
|
260
266
|
this.name = o.name;
|
|
267
|
+
this.identity = o.identity ?? o.name;
|
|
261
268
|
this.cfg = o.cfg;
|
|
262
269
|
this.deps = o.deps;
|
|
263
270
|
this.ep = resolveEndpoint(o.deps.env);
|
|
264
271
|
this.statusPath = join(o.agentDir, '.monitor-status');
|
|
265
272
|
this.cursorPath = join(o.agentDir, '.notify-cursor');
|
|
273
|
+
this.statePath = join(o.agentDir, '.monitor-state.json');
|
|
266
274
|
const n = o.cfg.turn_fail_threshold;
|
|
267
275
|
this.turnFailThreshold = typeof n === 'number' && n >= 1 ? n : DEFAULT_TURN_FAIL_THRESHOLD;
|
|
268
276
|
}
|
|
269
|
-
/**
|
|
277
|
+
/** Resume the last delivered cursor; only a brand-new monitor primes at stream tip. */
|
|
270
278
|
async prime() {
|
|
279
|
+
const persisted = this.readPersistedCursor();
|
|
280
|
+
if (persisted !== null) {
|
|
281
|
+
this.cursor = persisted;
|
|
282
|
+
this.deliveredCursor = persisted;
|
|
283
|
+
this.writeStatus();
|
|
284
|
+
return;
|
|
285
|
+
}
|
|
271
286
|
try {
|
|
272
287
|
const body = await this.doFetch('tip', LONGPOLL_TIMEOUT_MS);
|
|
273
288
|
this.cursor = typeof body.cursor === 'number' ? body.cursor : 0;
|
|
274
289
|
this.persistCursor();
|
|
275
|
-
this.
|
|
290
|
+
this.writeStatus();
|
|
276
291
|
}
|
|
277
292
|
catch (e) {
|
|
278
293
|
if (e instanceof AuthError) {
|
|
279
294
|
this.fatal = true;
|
|
280
|
-
this.
|
|
295
|
+
this.degrade('auth', e.message, 'failed');
|
|
281
296
|
}
|
|
282
297
|
else {
|
|
283
|
-
this.cursor =
|
|
284
|
-
this.
|
|
298
|
+
this.cursor = null;
|
|
299
|
+
this.degrade('connectivity', `prime failed (${msg(e)})`);
|
|
285
300
|
}
|
|
286
301
|
}
|
|
287
302
|
}
|
|
@@ -291,35 +306,61 @@ export class Monitor {
|
|
|
291
306
|
return;
|
|
292
307
|
this.bootDeadline = this.deps.now() + BOOT_GRACE_MS;
|
|
293
308
|
let backoff = 0;
|
|
309
|
+
const pending = [];
|
|
294
310
|
while (!this.stopped) {
|
|
295
311
|
if (!this.deps.isAlive(pid)) {
|
|
296
|
-
this.
|
|
312
|
+
this.degrade('offline', 'session offline');
|
|
297
313
|
return;
|
|
298
314
|
}
|
|
299
315
|
let body;
|
|
300
316
|
try {
|
|
301
317
|
body = await this.doFetch(String(this.cursor ?? 0), LONGPOLL_TIMEOUT_MS);
|
|
302
318
|
backoff = 0;
|
|
319
|
+
// A poll that worked proves the stream is healthy — and only that.
|
|
320
|
+
this.recover('connectivity');
|
|
303
321
|
}
|
|
304
322
|
catch (e) {
|
|
305
323
|
if (this.stopped)
|
|
306
324
|
return;
|
|
307
325
|
if (e instanceof AuthError) {
|
|
308
326
|
this.fatal = true;
|
|
309
|
-
this.
|
|
327
|
+
this.degrade('auth', e.message, 'failed');
|
|
310
328
|
return;
|
|
311
329
|
}
|
|
312
330
|
backoff = Math.min(backoff + BACKOFF_STEP_MS, BACKOFF_MAX_MS);
|
|
313
|
-
this.
|
|
331
|
+
this.degrade('connectivity', `stream hiccup (${msg(e)})`);
|
|
314
332
|
await this.deps.sleep(backoff);
|
|
315
333
|
continue;
|
|
316
334
|
}
|
|
317
|
-
this.advance(body.cursor);
|
|
335
|
+
this.advance(body.cursor, false);
|
|
318
336
|
const batch = filterEvents(body.events ?? [], this.cfg.wake_sources);
|
|
319
|
-
|
|
337
|
+
pending.push(...batch);
|
|
338
|
+
if (pending.length === 0) {
|
|
339
|
+
this.persistCursor();
|
|
320
340
|
continue;
|
|
321
|
-
|
|
322
|
-
|
|
341
|
+
}
|
|
342
|
+
this.pendingState = {
|
|
343
|
+
count: pending.length,
|
|
344
|
+
eventTypes: uniq(pending.map(event => event.event ?? 'unknown')),
|
|
345
|
+
attempts: (this.pendingState?.attempts ?? 0) + 1,
|
|
346
|
+
};
|
|
347
|
+
this.persistState();
|
|
348
|
+
await this.coalesce(pending);
|
|
349
|
+
// Do not durably commit this cursor until the session explicitly accepts
|
|
350
|
+
// the wake. If delivery fails or the runner crashes, the daemon replays
|
|
351
|
+
// from the last committed cursor and the wake is attempted again.
|
|
352
|
+
let accepted = false;
|
|
353
|
+
try {
|
|
354
|
+
accepted = await this.deliver(pid, pending);
|
|
355
|
+
}
|
|
356
|
+
catch (e) {
|
|
357
|
+
this.degrade('delivery', `delivery failed (${msg(e)})`);
|
|
358
|
+
}
|
|
359
|
+
if (accepted) {
|
|
360
|
+
pending.length = 0;
|
|
361
|
+
this.pendingState = null;
|
|
362
|
+
this.persistCursor();
|
|
363
|
+
}
|
|
323
364
|
}
|
|
324
365
|
}
|
|
325
366
|
stop() {
|
|
@@ -336,22 +377,35 @@ export class Monitor {
|
|
|
336
377
|
return;
|
|
337
378
|
try {
|
|
338
379
|
const more = await this.doFetch(String(this.cursor ?? 0), COALESCE_HOLD_MS);
|
|
339
|
-
this.advance(more.cursor);
|
|
380
|
+
this.advance(more.cursor, false);
|
|
340
381
|
batch.push(...filterEvents(more.events ?? [], this.cfg.wake_sources));
|
|
341
382
|
}
|
|
342
383
|
catch { /* no stragglers / abort — deliver what we have */ }
|
|
343
384
|
}
|
|
344
385
|
async deliver(pid, batch) {
|
|
386
|
+
const line = formatNotificationLine(batch);
|
|
387
|
+
if (this.deps.delivery) {
|
|
388
|
+
const result = await this.deps.delivery.submit(line);
|
|
389
|
+
if (!result.succeeded) {
|
|
390
|
+
// Name the reason: "refused" and "cancelled" are the agent's answer,
|
|
391
|
+
// not a transport problem, and an operator has to be able to tell them
|
|
392
|
+
// apart from a dead socket.
|
|
393
|
+
this.degrade('delivery', `wake ${result.outcome}${result.detail ? ` (${result.detail})` : ''}`);
|
|
394
|
+
return false;
|
|
395
|
+
}
|
|
396
|
+
this.recover('delivery', 'modal');
|
|
397
|
+
this.recordTurn('completed');
|
|
398
|
+
return true;
|
|
399
|
+
}
|
|
345
400
|
const state = await this.awaitInjectable(pid);
|
|
346
401
|
if (state !== 'ready') {
|
|
347
402
|
if (state === 'offline')
|
|
348
|
-
this.
|
|
403
|
+
this.degrade('offline', 'offline during delivery');
|
|
349
404
|
else if (state === 'modal')
|
|
350
|
-
this.
|
|
405
|
+
this.degrade('modal', `modal wedge — pane held a dialog for ` +
|
|
351
406
|
`${MODAL_GIVE_UP_MS / 1000}s, wake not injected`);
|
|
352
|
-
return;
|
|
407
|
+
return false;
|
|
353
408
|
}
|
|
354
|
-
const line = formatNotificationLine(batch);
|
|
355
409
|
await this.clearComposer(); // start from an empty composer
|
|
356
410
|
await this.deps.tmux.sendText(this.name, line); // send-keys -l + Enter
|
|
357
411
|
let delivered = false;
|
|
@@ -360,21 +414,27 @@ export class Monitor {
|
|
|
360
414
|
// dead pane makes safeCapture return '' ⇒ not-in-composer ⇒ breaks, no wasted Enter.
|
|
361
415
|
for (let i = 0; i < MAX_ENTER_RETRIES; i++) {
|
|
362
416
|
await this.deps.sleep(POST_VERIFY_MS);
|
|
363
|
-
const
|
|
364
|
-
if (!
|
|
417
|
+
const capture = await safeCapture(this.deps.tmux, this.name);
|
|
418
|
+
if (!capture.ok) {
|
|
419
|
+
this.degrade('delivery', 'capture failed during injection verification');
|
|
420
|
+
return false;
|
|
421
|
+
}
|
|
422
|
+
if (!stillInComposer(capture.pane, line)) {
|
|
365
423
|
delivered = true;
|
|
366
424
|
break;
|
|
367
425
|
}
|
|
368
426
|
await this.deps.tmux.sendKey(this.name, 'Enter');
|
|
369
427
|
}
|
|
370
428
|
if (!delivered) {
|
|
371
|
-
this.
|
|
372
|
-
return;
|
|
429
|
+
this.degrade('delivery', 'injection unverified');
|
|
430
|
+
return false;
|
|
373
431
|
}
|
|
432
|
+
this.recover('delivery', 'modal');
|
|
374
433
|
// The wake landed and a turn started; observe how that turn terminates so a
|
|
375
434
|
// refusal-wedge (every turn dies with `API Error:` while delivery stays green)
|
|
376
435
|
// becomes visible in `.monitor-status` instead of masquerading as armed (#19).
|
|
377
436
|
await this.observeTurnOutcome(pid);
|
|
437
|
+
return true;
|
|
378
438
|
}
|
|
379
439
|
/**
|
|
380
440
|
* Watch the pane until the just-triggered turn settles, then fold its outcome
|
|
@@ -389,12 +449,16 @@ export class Monitor {
|
|
|
389
449
|
return; // shutting down — leave status
|
|
390
450
|
if (!this.deps.isAlive(pid) || !(await this.deps.tmux.has(this.name)))
|
|
391
451
|
return; // loop marks offline
|
|
392
|
-
const
|
|
393
|
-
if (
|
|
452
|
+
const capture = await safeCapture(this.deps.tmux, this.name);
|
|
453
|
+
if (!capture.ok) {
|
|
454
|
+
this.degrade('delivery', 'capture failed during turn observation');
|
|
455
|
+
return;
|
|
456
|
+
}
|
|
457
|
+
if (looksApiError(capture.pane)) {
|
|
394
458
|
this.recordTurn('api-error');
|
|
395
459
|
return;
|
|
396
460
|
}
|
|
397
|
-
if (!looksRunning(pane)) {
|
|
461
|
+
if (!looksRunning(capture.pane)) {
|
|
398
462
|
this.recordTurn('completed');
|
|
399
463
|
return;
|
|
400
464
|
}
|
|
@@ -404,14 +468,17 @@ export class Monitor {
|
|
|
404
468
|
}
|
|
405
469
|
/** Update the consecutive-API-error streak and derive `.monitor-status` from it. */
|
|
406
470
|
recordTurn(outcome) {
|
|
471
|
+
if (outcome === 'inconclusive')
|
|
472
|
+
return; // no evidence either way; leave the streak
|
|
407
473
|
if (outcome === 'api-error')
|
|
408
474
|
this.apiErrorStreak++;
|
|
409
|
-
else
|
|
475
|
+
else
|
|
410
476
|
this.apiErrorStreak = 0;
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
477
|
+
if (this.apiErrorStreak >= this.turnFailThreshold)
|
|
478
|
+
this.degrade('turns-failing', 'turns failing (api error)');
|
|
479
|
+
else if (outcome === 'completed')
|
|
480
|
+
// A turn that ran to the end is the ONLY thing that clears this.
|
|
481
|
+
this.recover('turns-failing');
|
|
415
482
|
}
|
|
416
483
|
/**
|
|
417
484
|
* Reset the composer to empty before typing a wake. Without this, any
|
|
@@ -443,8 +510,13 @@ export class Monitor {
|
|
|
443
510
|
await this.deps.sleep(this.bootDeadline - now);
|
|
444
511
|
continue;
|
|
445
512
|
}
|
|
446
|
-
const
|
|
447
|
-
if (!
|
|
513
|
+
const capture = await safeCapture(this.deps.tmux, this.name);
|
|
514
|
+
if (!capture.ok) {
|
|
515
|
+
this.degrade('delivery', 'capture failed while checking session readiness');
|
|
516
|
+
await this.deps.sleep(MODAL_RETRY_MS);
|
|
517
|
+
continue;
|
|
518
|
+
}
|
|
519
|
+
if (!looksModal(capture.pane))
|
|
448
520
|
return 'ready';
|
|
449
521
|
if (modalWaits++ >= MAX_MODAL_WAITS)
|
|
450
522
|
return 'modal';
|
|
@@ -457,7 +529,7 @@ export class Monitor {
|
|
|
457
529
|
const timer = this.deps.timers.set(() => ctrl.abort(), holdMs);
|
|
458
530
|
let resp;
|
|
459
531
|
try {
|
|
460
|
-
resp = await this.deps.fetch(`${this.ep.url(this.
|
|
532
|
+
resp = await this.deps.fetch(`${this.ep.url(this.identity)}?since=${since}`, { headers: this.ep.headers, signal: ctrl.signal });
|
|
461
533
|
}
|
|
462
534
|
finally {
|
|
463
535
|
this.deps.timers.clear(timer);
|
|
@@ -469,16 +541,22 @@ export class Monitor {
|
|
|
469
541
|
throw new Error(`daemon returned HTTP ${resp.status}`);
|
|
470
542
|
return resp.json();
|
|
471
543
|
}
|
|
472
|
-
advance(cursor) {
|
|
544
|
+
advance(cursor, persist = true) {
|
|
473
545
|
if (typeof cursor === 'number' && cursor !== this.cursor) {
|
|
474
546
|
this.cursor = cursor;
|
|
475
|
-
|
|
547
|
+
if (persist)
|
|
548
|
+
this.persistCursor();
|
|
549
|
+
else
|
|
550
|
+
this.persistState();
|
|
476
551
|
}
|
|
477
552
|
}
|
|
478
553
|
persistCursor() {
|
|
479
554
|
try {
|
|
480
|
-
if (this.cursor !== null)
|
|
555
|
+
if (this.cursor !== null) {
|
|
556
|
+
this.deliveredCursor = this.cursor;
|
|
481
557
|
writeFileSync(this.cursorPath, `${this.cursor}\n`);
|
|
558
|
+
this.persistState();
|
|
559
|
+
}
|
|
482
560
|
}
|
|
483
561
|
catch (e) {
|
|
484
562
|
this.deps.log(`[${this.name}] monitor: failed to persist cursor: ${msg(e)}`);
|
|
@@ -486,24 +564,81 @@ export class Monitor {
|
|
|
486
564
|
}
|
|
487
565
|
readPersistedCursor() {
|
|
488
566
|
try {
|
|
567
|
+
if (existsSync(this.statePath)) {
|
|
568
|
+
const state = JSON.parse(readFileSync(this.statePath, 'utf8'));
|
|
569
|
+
if ((state.identity !== undefined && state.identity !== this.identity)
|
|
570
|
+
|| (state.profileKey !== undefined && state.profileKey !== this.ep.origin))
|
|
571
|
+
return null;
|
|
572
|
+
if (typeof state.deliveredCursor === 'number') {
|
|
573
|
+
this.deliveredCursor = state.deliveredCursor;
|
|
574
|
+
return state.deliveredCursor;
|
|
575
|
+
}
|
|
576
|
+
}
|
|
489
577
|
if (!existsSync(this.cursorPath))
|
|
490
578
|
return null;
|
|
491
579
|
const n = parseInt(readFileSync(this.cursorPath, 'utf8').trim(), 10);
|
|
580
|
+
if (Number.isFinite(n))
|
|
581
|
+
this.deliveredCursor = n;
|
|
492
582
|
return Number.isFinite(n) ? n : null;
|
|
493
583
|
}
|
|
494
584
|
catch {
|
|
495
585
|
return null;
|
|
496
586
|
}
|
|
497
587
|
}
|
|
498
|
-
|
|
588
|
+
/** Atomically persist body-free delivery state; restart always resumes from deliveredCursor. */
|
|
589
|
+
persistState() {
|
|
590
|
+
try {
|
|
591
|
+
const tmp = `${this.statePath}.${process.pid}.tmp`;
|
|
592
|
+
writeFileSync(tmp, JSON.stringify({
|
|
593
|
+
version: 1,
|
|
594
|
+
identity: this.identity,
|
|
595
|
+
profileKey: this.ep.origin,
|
|
596
|
+
observedCursor: this.cursor,
|
|
597
|
+
deliveredCursor: this.deliveredCursor,
|
|
598
|
+
pending: this.pendingState,
|
|
599
|
+
updatedAt: new Date(this.deps.now()).toISOString(),
|
|
600
|
+
}, null, 2) + '\n', { mode: 0o600 });
|
|
601
|
+
renameSync(tmp, this.statePath);
|
|
602
|
+
}
|
|
603
|
+
catch (e) {
|
|
604
|
+
this.deps.log(`[${this.name}] monitor: failed to persist state: ${msg(e)}`);
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
/** Record a degradation under its own cause and republish the status. */
|
|
608
|
+
degrade(cause, detail, level = 'degraded') {
|
|
609
|
+
const previous = this.causes.get(cause);
|
|
610
|
+
this.causes.set(cause, { level, detail, at: new Date(this.deps.now()).toISOString() });
|
|
611
|
+
this.writeStatus();
|
|
612
|
+
if (previous?.detail !== detail)
|
|
613
|
+
this.deps.log(`[${this.name}] monitor ${level}: ${cause} — ${detail}`);
|
|
614
|
+
}
|
|
615
|
+
/**
|
|
616
|
+
* Clear exactly the causes this recovery signal speaks to. Anything else
|
|
617
|
+
* stays: one successful poll must never be able to erase `turns failing`.
|
|
618
|
+
*/
|
|
619
|
+
recover(...causes) {
|
|
620
|
+
let changed = false;
|
|
621
|
+
for (const cause of causes)
|
|
622
|
+
changed = this.causes.delete(cause) || changed;
|
|
623
|
+
if (changed)
|
|
624
|
+
this.deps.log(`[${this.name}] monitor recovered: ${causes.join(', ')}`);
|
|
625
|
+
this.writeStatus();
|
|
626
|
+
}
|
|
627
|
+
/**
|
|
628
|
+
* One line per active cause, each dated; `armed` when there are none. Every
|
|
629
|
+
* line carries an ISO timestamp so an operator can tell a live status from a
|
|
630
|
+
* stale one left behind by a monitor that stopped writing.
|
|
631
|
+
*/
|
|
632
|
+
writeStatus() {
|
|
633
|
+
const lines = this.causes.size
|
|
634
|
+
? [...this.causes.entries()].map(([cause, e]) => `${e.level}: ${cause} at ${e.at} — ${e.detail}`)
|
|
635
|
+
: [`armed at ${new Date(this.deps.now()).toISOString()}`];
|
|
499
636
|
try {
|
|
500
|
-
writeFileSync(this.statusPath,
|
|
637
|
+
writeFileSync(this.statusPath, lines.join('\n') + '\n');
|
|
501
638
|
}
|
|
502
639
|
catch (e) {
|
|
503
640
|
this.deps.log(`[${this.name}] monitor: failed to write status: ${msg(e)}`);
|
|
504
641
|
}
|
|
505
|
-
if (!s.startsWith('armed'))
|
|
506
|
-
this.deps.log(`[${this.name}] monitor ${s}`);
|
|
507
642
|
}
|
|
508
643
|
}
|
|
509
644
|
export function createMonitor(o) {
|
|
@@ -511,10 +646,10 @@ export function createMonitor(o) {
|
|
|
511
646
|
}
|
|
512
647
|
async function safeCapture(tmux, name) {
|
|
513
648
|
try {
|
|
514
|
-
return await tmux.capture(name);
|
|
649
|
+
return { ok: true, pane: await tmux.capture(name) };
|
|
515
650
|
}
|
|
516
651
|
catch {
|
|
517
|
-
return '';
|
|
652
|
+
return { ok: false, pane: '' };
|
|
518
653
|
}
|
|
519
654
|
}
|
|
520
655
|
const msg = (e) => e?.message ?? String(e);
|
package/dist/ops.d.ts
CHANGED
|
@@ -1,18 +1,31 @@
|
|
|
1
1
|
import type { FleetConfig, ResolvedRole } from './config.js';
|
|
2
|
-
import type { SupervisorBackend } from './supervisor/types.js';
|
|
2
|
+
import type { InstallOutcome as BackendInstallOutcome, SupervisorBackend } from './supervisor/types.js';
|
|
3
|
+
/** An install outcome tagged with the role it belongs to. */
|
|
4
|
+
export interface InstallOutcome extends BackendInstallOutcome {
|
|
5
|
+
role: string;
|
|
6
|
+
}
|
|
3
7
|
export interface OpsDeps {
|
|
4
8
|
backend: SupervisorBackend;
|
|
5
9
|
binPath: string;
|
|
6
10
|
log(line: string): void;
|
|
11
|
+
/**
|
|
12
|
+
* Called the INSTANT a registration is created, before anything else can
|
|
13
|
+
* fail. A creation transaction that learns about registrations only from
|
|
14
|
+
* `up()`'s return value learns nothing when `up()` throws — and the service
|
|
15
|
+
* it just registered is then invisible to rollback (6.2). Optional: plain
|
|
16
|
+
* `ours-fleet up` has no transaction to tell.
|
|
17
|
+
*/
|
|
18
|
+
onInstalled?(outcome: InstallOutcome): void;
|
|
7
19
|
}
|
|
8
20
|
/** Materialize a role's state dir from config: briefing + markers. Returns the dir. */
|
|
9
21
|
export declare function applyRole(role: ResolvedRole, opts?: {
|
|
10
22
|
fresh?: boolean;
|
|
11
23
|
temp?: boolean;
|
|
12
24
|
configPath?: string;
|
|
25
|
+
identityGuarantee?: 'verified' | 'created' | 'unverified';
|
|
13
26
|
}): string;
|
|
14
27
|
/** Create/start roles declaratively. Idempotent; active roles keep their context. */
|
|
15
|
-
export declare function up(cfg: FleetConfig, names: string[], deps: OpsDeps, configPath?: string): Promise<
|
|
28
|
+
export declare function up(cfg: FleetConfig, names: string[], deps: OpsDeps, configPath?: string, identityGuarantee?: 'verified' | 'created' | 'unverified'): Promise<InstallOutcome[]>;
|
|
16
29
|
export declare function down(cfg: FleetConfig, names: string[], deps: OpsDeps): Promise<void>;
|
|
17
30
|
/** Re-sync from config + bounce. mode 'keep' resumes context; 'fresh' wipes it. */
|
|
18
31
|
export declare function restartRoles(cfg: FleetConfig, names: string[], deps: OpsDeps, mode: 'keep' | 'fresh', configPath?: string): Promise<void>;
|
package/dist/ops.js
CHANGED
|
@@ -5,6 +5,7 @@ import { agentDir, fleetDDir } from './paths.js';
|
|
|
5
5
|
import { findRole } from './config.js';
|
|
6
6
|
import { getAdapter } from './harness/registry.js';
|
|
7
7
|
import { generateBriefing } from './briefing.js';
|
|
8
|
+
import { resetRestartLedger } from './runner.js';
|
|
8
9
|
// Launch staggering now lives at the harness-launch point (the runner's start
|
|
9
10
|
// gate, driven by `start_stagger_ms`), so it covers systemd host-boot too — not
|
|
10
11
|
// just the `up`/`restart` command loop below. The old in-loop FLEET_START_STAGGER
|
|
@@ -36,6 +37,7 @@ export function applyRole(role, opts = {}) {
|
|
|
36
37
|
writeFileSync(join(dir, 'briefing.md'), generateBriefing(role, adapter.vocabulary, {
|
|
37
38
|
stateDir: dir, worklogPath: join(dir, 'WORKLOG.md'),
|
|
38
39
|
routinesPath: join(dir, 'ROUTINES.md'), briefingBody,
|
|
40
|
+
identityGuarantee: opts.identityGuarantee,
|
|
39
41
|
}));
|
|
40
42
|
if (opts.fresh)
|
|
41
43
|
for (const f of ['.booted', '.session-id', '.exit-status'])
|
|
@@ -46,32 +48,53 @@ function selectRoles(cfg, names) {
|
|
|
46
48
|
return names.length ? names.map(n => findRole(cfg, n)) : cfg.roles;
|
|
47
49
|
}
|
|
48
50
|
/** Create/start roles declaratively. Idempotent; active roles keep their context. */
|
|
49
|
-
export async function up(cfg, names, deps, configPath) {
|
|
51
|
+
export async function up(cfg, names, deps, configPath, identityGuarantee) {
|
|
52
|
+
const outcomes = [];
|
|
50
53
|
for (const role of selectRoles(cfg, names)) {
|
|
51
|
-
const dir = applyRole(role, { configPath });
|
|
52
|
-
//
|
|
53
|
-
|
|
54
|
-
|
|
54
|
+
const dir = applyRole(role, { configPath, identityGuarantee });
|
|
55
|
+
// Only a *definite* stop boots fresh so the role reads the briefing we just
|
|
56
|
+
// wrote. A running, restarting, or unprobeable role keeps its context —
|
|
57
|
+
// guessing "stopped" from an unanswered probe silently discards a live
|
|
58
|
+
// conversation.
|
|
59
|
+
// An explicit operator `up` is the sanctioned way to release a held-down
|
|
60
|
+
// role: the still-alive runner polls this file and resumes (3.2).
|
|
61
|
+
resetRestartLedger(dir);
|
|
62
|
+
const live = await deps.backend.liveness(role.name)
|
|
63
|
+
.catch(e => ({ state: 'unknown', detail: e instanceof Error ? e.message : String(e) }));
|
|
64
|
+
if (live.state === 'stopped')
|
|
55
65
|
rmSync(join(dir, '.booted'), { force: true });
|
|
56
|
-
|
|
66
|
+
else if (live.state === 'unknown')
|
|
67
|
+
deps.log(` ! ${role.name}: liveness unknown, keeping session context — ${live.detail}`);
|
|
68
|
+
// Report what each install actually did, so a creation transaction can undo
|
|
69
|
+
// only the registrations IT made (6.2). Announced immediately as well as
|
|
70
|
+
// returned: a later role in this same loop can throw, and the registrations
|
|
71
|
+
// already made must still be undoable.
|
|
72
|
+
const outcome = { ...await deps.backend.install(role.name, deps.binPath), role: role.name };
|
|
73
|
+
if (outcome.created)
|
|
74
|
+
deps.onInstalled?.(outcome);
|
|
75
|
+
outcomes.push(outcome);
|
|
57
76
|
deps.log(`↑ up: ${role.name} (harness: ${role.harness}, identity: ${role.identity}${role.cwd ? `, cwd: ${role.cwd}` : ''})`);
|
|
58
77
|
}
|
|
78
|
+
return outcomes;
|
|
59
79
|
}
|
|
60
80
|
export async function down(cfg, names, deps) {
|
|
61
81
|
for (const role of selectRoles(cfg, names)) {
|
|
82
|
+
// Never swallow the backend's reason. "maybe not running" hid real stop
|
|
83
|
+
// failures — a wedged unit, an unreachable user bus — behind a guess.
|
|
62
84
|
try {
|
|
63
85
|
await deps.backend.stop(role.name);
|
|
64
86
|
deps.log(`■ stopped ${role.name}`);
|
|
65
87
|
}
|
|
66
|
-
catch {
|
|
67
|
-
deps.log(`
|
|
88
|
+
catch (e) {
|
|
89
|
+
deps.log(` ! could not stop ${role.name}: ${e instanceof Error ? e.message : String(e)}`);
|
|
68
90
|
}
|
|
69
91
|
}
|
|
70
92
|
}
|
|
71
93
|
/** Re-sync from config + bounce. mode 'keep' resumes context; 'fresh' wipes it. */
|
|
72
94
|
export async function restartRoles(cfg, names, deps, mode, configPath) {
|
|
73
95
|
for (const role of selectRoles(cfg, names)) {
|
|
74
|
-
applyRole(role, { fresh: mode === 'fresh', configPath });
|
|
96
|
+
const dir = applyRole(role, { fresh: mode === 'fresh', configPath });
|
|
97
|
+
resetRestartLedger(dir); // explicit restart closes the circuit
|
|
75
98
|
await deps.backend.restart(role.name);
|
|
76
99
|
deps.log(mode === 'fresh'
|
|
77
100
|
? `↻ ${role.name} — force-restarted (FRESH — context cleared, briefing reloaded)`
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import type { CommonPermissions, ResolvedRole } from './config.js';
|
|
2
|
+
import type { UnattendedCapability } from './harness/types.js';
|
|
3
|
+
/**
|
|
4
|
+
* The capability floor every unattended role must clear. These are not
|
|
5
|
+
* nice-to-haves: an agent that cannot read its briefing, append its worklog,
|
|
6
|
+
* bind its identity, arm its monitor, edit its workspace, or run the status
|
|
7
|
+
* commands its briefing prescribes cannot carry out the job it was spawned for
|
|
8
|
+
* — and, being unattended, will report no error while failing to.
|
|
9
|
+
*/
|
|
10
|
+
export declare const UNATTENDED_FLOOR: readonly UnattendedCapability[];
|
|
11
|
+
export interface FloorResult {
|
|
12
|
+
meets: boolean;
|
|
13
|
+
missing: UnattendedCapability[];
|
|
14
|
+
}
|
|
15
|
+
/** Which floor capabilities a set of granted capabilities fails to cover. */
|
|
16
|
+
export declare function checkUnattendedFloor(granted: readonly UnattendedCapability[]): FloorResult;
|
|
17
|
+
/**
|
|
18
|
+
* One role's neutral permissions, resolved through its harness adapter.
|
|
19
|
+
*
|
|
20
|
+
* Both `ours-fleet config` and `ours-fleet doctor` render this same object, so
|
|
21
|
+
* the two commands cannot disagree about what a configuration actually means.
|
|
22
|
+
* Before this existed, `translatePermissions()` was implemented by every
|
|
23
|
+
* adapter and called by nobody: the warnings it produced — including "this
|
|
24
|
+
* combination is not represented exactly" — were unreachable.
|
|
25
|
+
*/
|
|
26
|
+
export interface RolePermissionAnalysis {
|
|
27
|
+
role: string;
|
|
28
|
+
harness: string;
|
|
29
|
+
permissions: CommonPermissions;
|
|
30
|
+
/** Whether the harness can express neutral permissions at all. */
|
|
31
|
+
supported: boolean;
|
|
32
|
+
/** The harness's own settings, when it can. */
|
|
33
|
+
native?: Record<string, unknown>;
|
|
34
|
+
/** Whether those settings represent the neutral intent exactly. */
|
|
35
|
+
exact?: boolean;
|
|
36
|
+
/** What the native settings actually permit an unattended agent to do. */
|
|
37
|
+
capabilities?: UnattendedCapability[];
|
|
38
|
+
/** Whether those capabilities clear the unattended floor. */
|
|
39
|
+
floor?: FloorResult;
|
|
40
|
+
/**
|
|
41
|
+
* How hard a floor shortfall is. A role that auto-denies (`unattended: deny`)
|
|
42
|
+
* silently does less than asked, so that is a failure; one that waits can at
|
|
43
|
+
* least be rescued by a human attaching a console, so that is a warning.
|
|
44
|
+
*/
|
|
45
|
+
floorSeverity?: 'fail' | 'warn';
|
|
46
|
+
/** Native settings that contradict the neutral block; empty when they agree. */
|
|
47
|
+
conflicts?: PermissionConflict[];
|
|
48
|
+
/** A role-named line when the floor is not met; absent when it is. */
|
|
49
|
+
floorWarning?: string;
|
|
50
|
+
/** Role-named translation warnings, ready to print verbatim by any command. */
|
|
51
|
+
warnings: string[];
|
|
52
|
+
}
|
|
53
|
+
export interface PermissionConflict {
|
|
54
|
+
/** The native setting both sources speak to, e.g. `permission_mode`. */
|
|
55
|
+
key: string;
|
|
56
|
+
/** What the neutral `permissions:` block translates to. */
|
|
57
|
+
fromNeutral: string;
|
|
58
|
+
/** What `harness_options` states directly. */
|
|
59
|
+
fromNative: string;
|
|
60
|
+
/** The role-named line commands print. */
|
|
61
|
+
warning: string;
|
|
62
|
+
}
|
|
63
|
+
/** Resolve one role's permissions through its adapter. Never throws. */
|
|
64
|
+
export declare function analyzeRolePermissions(role: ResolvedRole): RolePermissionAnalysis;
|
|
65
|
+
/** Every line a command should show for a role: translation, conflicts, floor. */
|
|
66
|
+
export declare function allWarnings(a: RolePermissionAnalysis): string[];
|
|
67
|
+
/** Resolve every role's permissions, in config order. */
|
|
68
|
+
export declare function analyzeFleetPermissions(roles: ResolvedRole[]): RolePermissionAnalysis[];
|
|
69
|
+
/** Render an analysis's native settings compactly, for one-line reporting. */
|
|
70
|
+
export declare function formatNative(native: Record<string, unknown> | undefined): string;
|