@phnx-labs/agents-cli 1.22.0 → 1.22.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +115 -0
- package/README.md +2 -0
- package/dist/bin/agents +0 -0
- package/dist/commands/cloud.js +1 -1
- package/dist/commands/doctor.js +34 -2
- package/dist/commands/exec.js +1 -1
- package/dist/commands/projects.js +128 -66
- package/dist/commands/secrets.js +9 -5
- package/dist/commands/teams.js +2 -2
- package/dist/commands/watchdog.js +64 -5
- package/dist/lib/app-bundle-install.d.ts +17 -0
- package/dist/lib/app-bundle-install.js +94 -0
- package/dist/lib/devices/doctor-findings.js +12 -4
- package/dist/lib/devices/doctor-overview-cache.d.ts +45 -0
- package/dist/lib/devices/doctor-overview-cache.js +168 -0
- package/dist/lib/devices/fleet.js +7 -2
- package/dist/lib/devices/self-host.d.ts +9 -0
- package/dist/lib/devices/self-host.js +61 -0
- package/dist/lib/hosts/passthrough.js +8 -6
- package/dist/lib/menubar/MenubarHelper.app/Contents/CodeResources +0 -0
- package/dist/lib/menubar/MenubarHelper.app/Contents/MacOS/MenubarHelper +0 -0
- package/dist/lib/menubar/install-menubar.js +27 -23
- package/dist/lib/project-status.d.ts +7 -0
- package/dist/lib/project-status.js +9 -0
- package/dist/lib/projects.d.ts +26 -2
- package/dist/lib/projects.js +69 -9
- package/dist/lib/rotate.d.ts +16 -1
- package/dist/lib/rotate.js +33 -7
- package/dist/lib/secrets/Agents CLI.app/Contents/CodeResources +0 -0
- package/dist/lib/secrets/Agents CLI.app/Contents/MacOS/Agents CLI +0 -0
- package/dist/lib/secrets/bundles.d.ts +20 -0
- package/dist/lib/secrets/bundles.js +50 -0
- package/dist/lib/secrets/install-helper.js +28 -31
- package/dist/lib/share/config.d.ts +14 -6
- package/dist/lib/share/config.js +23 -5
- package/dist/lib/types.d.ts +10 -0
- package/dist/lib/versions.js +69 -22
- package/dist/lib/watchdog/rotate.d.ts +218 -0
- package/dist/lib/watchdog/rotate.js +378 -0
- package/dist/lib/watchdog/runner.d.ts +33 -1
- package/dist/lib/watchdog/runner.js +303 -0
- package/package.json +1 -1
|
@@ -29,6 +29,7 @@ import { mailboxIdForActiveSession } from '../lib/mailbox-target.js';
|
|
|
29
29
|
import { gcMailbox } from '../lib/mailbox-gc.js';
|
|
30
30
|
import { ensureWatchdogRoutine, isWatchdogRoutineEnabled, watchdogRoutineExists, WATCHDOG_ROUTINE_NAME, WATCHDOG_ROUTINE_SCHEDULE, } from '../lib/watchdog/routine.js';
|
|
31
31
|
import { runWatchdogTick, writePolicySentinel, DEFAULT_THRESHOLDS, } from '../lib/watchdog/runner.js';
|
|
32
|
+
import { isWatchdogRotateEnabled, listRotateStates, setWatchdogRotateEnabled } from '../lib/watchdog/rotate.js';
|
|
32
33
|
/** Default state dir the runner and these subcommands share. */
|
|
33
34
|
function stateDir() {
|
|
34
35
|
return path.join(getRuntimeStateDir(), 'watchdog');
|
|
@@ -79,6 +80,8 @@ async function runMailboxGc(sessions) {
|
|
|
79
80
|
function colorForOutcome(o) {
|
|
80
81
|
if (o.injected)
|
|
81
82
|
return chalk.green;
|
|
83
|
+
if (o.decision === 'rotate')
|
|
84
|
+
return chalk.magenta;
|
|
82
85
|
if (o.addressable === false)
|
|
83
86
|
return chalk.yellow;
|
|
84
87
|
if (o.stall === 'stalled' && o.decision === 'nudge')
|
|
@@ -92,12 +95,14 @@ function printTick(result, willInject) {
|
|
|
92
95
|
console.log(`${chalk.bold('watchdog')} ${mode} ` +
|
|
93
96
|
`${counts.total} live · ${counts.stalled} stalled · ` +
|
|
94
97
|
`${chalk.green(String(counts.nudged))} nudged · ` +
|
|
95
|
-
`${chalk.yellow(String(counts.unaddressable))} un-addressable`
|
|
98
|
+
`${chalk.yellow(String(counts.unaddressable))} un-addressable` +
|
|
99
|
+
(counts.rotating > 0 ? ` · ${chalk.magenta(String(counts.rotating))} rotating` : ''));
|
|
96
100
|
for (const o of result.outcomes) {
|
|
97
101
|
const tag = o.injected ? 'NUDGED'
|
|
98
|
-
: o.
|
|
99
|
-
: o.
|
|
100
|
-
: '
|
|
102
|
+
: o.decision === 'rotate' ? (o.rotatePhase === 'failed' ? 'ROTATE-FAIL' : 'ROTATE')
|
|
103
|
+
: o.addressable === false ? 'FLAGGED'
|
|
104
|
+
: o.decision === 'nudge' ? 'WOULD-NUDGE'
|
|
105
|
+
: 'skip';
|
|
101
106
|
const c = colorForOutcome(o);
|
|
102
107
|
const who = o.label || o.sessionId?.slice(0, 8) || o.kind;
|
|
103
108
|
const where = o.host ? chalk.dim(`[${o.host}]`) : '';
|
|
@@ -194,8 +199,14 @@ export function registerWatchdogCommand(program) {
|
|
|
194
199
|
# Turn on the ALWAYS-ON watchdog (a daemon-fired routine)
|
|
195
200
|
agents watchdog enable
|
|
196
201
|
|
|
202
|
+
# Show the routine, rotate config, and any in-flight in-place rotates
|
|
203
|
+
agents watchdog status
|
|
204
|
+
|
|
197
205
|
# Leave one session detected-but-untouched
|
|
198
206
|
agents watchdog policy <sessionId> handsoff
|
|
207
|
+
|
|
208
|
+
# Opt out of in-place rotate only (nudging stays on)
|
|
209
|
+
agents watchdog rotate off
|
|
199
210
|
`,
|
|
200
211
|
notes: `
|
|
201
212
|
Decision path: a cheap deterministic pre-filter resolves the obvious cases
|
|
@@ -217,6 +228,20 @@ export function registerWatchdogCommand(program) {
|
|
|
217
228
|
Ghostty with no tmux) is flagged for the menu-bar and SKIPPED -- never a
|
|
218
229
|
guessed or frontmost target.
|
|
219
230
|
|
|
231
|
+
Rotate: a stalled session whose tail shows a HARD account limit ("You've
|
|
232
|
+
hit your weekly limit - resets ...") is ROTATED IN PLACE instead of nudged:
|
|
233
|
+
the tick gates on the same healthy-account selection 'agents run auto'
|
|
234
|
+
makes (zero healthy -> one skip event per cooldown window, terminal
|
|
235
|
+
untouched), injects the harness's exit sequence, relaunches
|
|
236
|
+
'agents run auto --interactive --session-id <uuid>' in the SAME tab, waits
|
|
237
|
+
(bounded, 60s) for the new TUI, then injects the resume replay for the old
|
|
238
|
+
session. On timeout the session is flagged and never blind-typed into; the
|
|
239
|
+
flag says the terminal may sit at a bare shell and needs a manual
|
|
240
|
+
'agents run auto'. A failed rotate is suppressed for 15m before retry.
|
|
241
|
+
Default ON; disable with 'agents watchdog rotate off' (writes
|
|
242
|
+
'watchdog.rotate: off' to ~/.agents/agents.yaml; nudging stays on).
|
|
243
|
+
State machine: ~/.agents/.cache/state/watchdog/rotate/<sessionId>.json.
|
|
244
|
+
|
|
220
245
|
Always-on: 'agents watchdog enable' creates + enables a 'watchdog' command
|
|
221
246
|
routine ('${WATCHDOG_ROUTINE_SCHEDULE}' -> agents watchdog --nudge) and reloads the
|
|
222
247
|
daemon; 'disable' pauses it. Inspect it with 'agents routines list'. Defaults
|
|
@@ -244,6 +269,20 @@ export function registerWatchdogCommand(program) {
|
|
|
244
269
|
await reloadDaemonForRoutine(false);
|
|
245
270
|
console.log(chalk.yellow('watchdog: DISABLED (routine paused)'));
|
|
246
271
|
});
|
|
272
|
+
cmd.command('rotate <state>')
|
|
273
|
+
.description('Turn in-place rotate of rate-limited sessions on|off (watchdog.rotate in agents.yaml). ' +
|
|
274
|
+
'Rotate-only: nudging stays on — unlike `watchdog disable`, which pauses the whole watchdog.')
|
|
275
|
+
.action((state) => {
|
|
276
|
+
const s = state.toLowerCase();
|
|
277
|
+
if (s !== 'on' && s !== 'off') {
|
|
278
|
+
console.error(chalk.red(`invalid state '${state}'. Use: on | off`));
|
|
279
|
+
process.exitCode = 1;
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
282
|
+
setWatchdogRotateEnabled(s === 'on');
|
|
283
|
+
console.log(`watchdog: rotate ${s === 'on' ? chalk.green('ON') : chalk.yellow('OFF')} ` +
|
|
284
|
+
chalk.dim(`(watchdog.rotate: ${s} in agents.yaml)`));
|
|
285
|
+
});
|
|
247
286
|
cmd.command('status')
|
|
248
287
|
.description('Show whether the always-on watchdog routine is enabled and where state is written.')
|
|
249
288
|
.option('--json', 'Emit status as JSON (for the menu-bar / scripts)')
|
|
@@ -254,11 +293,31 @@ export function registerWatchdogCommand(program) {
|
|
|
254
293
|
// read it correctly regardless of which command commander bound it to.
|
|
255
294
|
const json = command.optsWithGlobals().json === true;
|
|
256
295
|
const on = isWatchdogRoutineEnabled();
|
|
296
|
+
const rotate = isWatchdogRotateEnabled() ? 'on' : 'off';
|
|
297
|
+
const rotates = listRotateStates(stateDir());
|
|
298
|
+
const inflight = rotates.filter((r) => r.phase !== 'done' && r.phase !== 'failed');
|
|
257
299
|
if (json) {
|
|
258
|
-
console.log(JSON.stringify({
|
|
300
|
+
console.log(JSON.stringify({
|
|
301
|
+
enabled: on,
|
|
302
|
+
routine: WATCHDOG_ROUTINE_NAME,
|
|
303
|
+
stateDir: stateDir(),
|
|
304
|
+
rotate,
|
|
305
|
+
rotates: rotates.map((r) => ({
|
|
306
|
+
sessionId: r.sessionId,
|
|
307
|
+
newSessionId: r.newSessionId,
|
|
308
|
+
agent: r.agent,
|
|
309
|
+
phase: r.phase,
|
|
310
|
+
updatedAtMs: r.updatedAtMs,
|
|
311
|
+
error: r.error,
|
|
312
|
+
})),
|
|
313
|
+
}));
|
|
259
314
|
return;
|
|
260
315
|
}
|
|
261
316
|
console.log(`always-on watchdog: ${on ? chalk.green('ON') : chalk.dim('off')} (routine '${WATCHDOG_ROUTINE_NAME}')`);
|
|
317
|
+
console.log(`rotate: ${rotate === 'on' ? chalk.green('on') : chalk.yellow('off')} (watchdog.rotate in agents.yaml) · ${inflight.length} in-flight`);
|
|
318
|
+
for (const r of inflight) {
|
|
319
|
+
console.log(` ${chalk.magenta(r.phase.padEnd(12))} ${chalk.bold(r.sessionId.slice(0, 8))} → ${r.newSessionId.slice(0, 8)}${r.error ? chalk.red(` ${r.error}`) : ''}`);
|
|
320
|
+
}
|
|
262
321
|
console.log(`state dir: ${chalk.dim(stateDir())}`);
|
|
263
322
|
});
|
|
264
323
|
// --- per-session policy ----------------------------------------------------
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copy an `.app` bundle to `dest` atomically. Stages into a sibling dir, then
|
|
3
|
+
* swaps with renames — the window where `dest` is absent shrinks from the
|
|
4
|
+
* seconds-long `cp` to a single microsecond rename, and a failed copy leaves the
|
|
5
|
+
* existing bundle untouched. Serialize concurrent callers with {@link withInstallLock}
|
|
6
|
+
* so the two-step swap never races another swap.
|
|
7
|
+
*/
|
|
8
|
+
export declare function copyAppBundle(src: string, dest: string, io?: {
|
|
9
|
+
renameSync?: (from: string, to: string) => void;
|
|
10
|
+
}): void;
|
|
11
|
+
/**
|
|
12
|
+
* Serialize installs across the many concurrent `agents` invocations that pass
|
|
13
|
+
* through the helper-install path, so a burst copies once instead of stampeding
|
|
14
|
+
* the atomic swap. Locks a sentinel file beside the bundle (the bundle itself may
|
|
15
|
+
* not exist yet on first install) via the shared {@link withFileLock}.
|
|
16
|
+
*/
|
|
17
|
+
export declare function withInstallLock(dest: string, fn: (heartbeat: () => void) => void): void;
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Atomic, serialized install of a macOS `.app` bundle to a stable user path.
|
|
3
|
+
*
|
|
4
|
+
* Shared by the two helpers agents-cli installs on darwin — the secrets keychain
|
|
5
|
+
* helper (`lib/secrets/install-helper.ts`) and the menu-bar helper
|
|
6
|
+
* (`lib/menubar/install-menubar.ts`) — both of which are (re)installed on the hot
|
|
7
|
+
* path of ordinary `agents` invocations. Both previously did a non-atomic
|
|
8
|
+
* `rm -rf dest` + `cp -R src dest` straight onto the live bundle. That copy takes
|
|
9
|
+
* long enough that a concurrent reader (Gatekeeper, or an exec of the bundle) sees
|
|
10
|
+
* a half-written `.app` — a truncated Mach-O / mismatched `_CodeSignature` hash —
|
|
11
|
+
* which macOS reports as **"is damaged and can't be opened."** On a busy box dozens
|
|
12
|
+
* of concurrent invocations raced the same path, so the dialog fired intermittently.
|
|
13
|
+
*
|
|
14
|
+
* {@link copyAppBundle} stages the copy in a sibling directory and swaps it into
|
|
15
|
+
* place with renames, so a reader sees either the old or the new complete bundle,
|
|
16
|
+
* never a half-written one — the only moment `dest` is briefly absent is the
|
|
17
|
+
* sub-millisecond gap between the two renames (vs the seconds-long `cp`), and a
|
|
18
|
+
* failed copy never touches the live bundle. {@link withInstallLock} serializes concurrent
|
|
19
|
+
* installers (via the shared `withFileLock`) so a burst of invocations installs
|
|
20
|
+
* once instead of stampeding.
|
|
21
|
+
*/
|
|
22
|
+
import * as fs from 'fs';
|
|
23
|
+
import * as path from 'path';
|
|
24
|
+
import { spawnSync } from 'child_process';
|
|
25
|
+
import { withFileLock, ensureLockTarget } from './fs-atomic.js';
|
|
26
|
+
// A helper `cp -R` under load can take a few seconds; the lock must outlast it so
|
|
27
|
+
// a peer never treats a live installer as crashed and interleaves a second swap.
|
|
28
|
+
// (fs-atomic's 5s default is tuned for sub-second read-modify-writes.)
|
|
29
|
+
const INSTALL_LOCK_STALE_MS = 60_000;
|
|
30
|
+
const INSTALL_LOCK_ACQUIRE_TIMEOUT_MS = 60_000;
|
|
31
|
+
/**
|
|
32
|
+
* Copy an `.app` bundle to `dest` atomically. Stages into a sibling dir, then
|
|
33
|
+
* swaps with renames — the window where `dest` is absent shrinks from the
|
|
34
|
+
* seconds-long `cp` to a single microsecond rename, and a failed copy leaves the
|
|
35
|
+
* existing bundle untouched. Serialize concurrent callers with {@link withInstallLock}
|
|
36
|
+
* so the two-step swap never races another swap.
|
|
37
|
+
*/
|
|
38
|
+
export function copyAppBundle(src, dest, io = {}) {
|
|
39
|
+
const rename = io.renameSync ?? fs.renameSync;
|
|
40
|
+
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
41
|
+
const staging = `${dest}.installing.${process.pid}`;
|
|
42
|
+
const backup = `${dest}.replaced.${process.pid}`;
|
|
43
|
+
fs.rmSync(staging, { recursive: true, force: true });
|
|
44
|
+
fs.rmSync(backup, { recursive: true, force: true });
|
|
45
|
+
// `cp -R` preserves the bundle's signature, symlinks, and resource forks;
|
|
46
|
+
// `fs.cpSync({recursive:true})` has historically mishandled xattrs on `.app`
|
|
47
|
+
// bundles, breaking codesign.
|
|
48
|
+
const r = spawnSync('cp', ['-R', src, staging], { stdio: ['ignore', 'pipe', 'pipe'], encoding: 'utf-8' });
|
|
49
|
+
if (r.status !== 0) {
|
|
50
|
+
fs.rmSync(staging, { recursive: true, force: true });
|
|
51
|
+
const msg = (r.stderr || r.stdout || '').toString().trim();
|
|
52
|
+
throw new Error(`Failed to copy ${src} -> ${staging}: ${msg || 'unknown error'}`);
|
|
53
|
+
}
|
|
54
|
+
// rename(2) cannot replace a non-empty directory, so move the current bundle
|
|
55
|
+
// aside, then move staging into place. If the second rename fails, restore the
|
|
56
|
+
// backup so `dest` is never left missing.
|
|
57
|
+
try {
|
|
58
|
+
if (fs.existsSync(dest))
|
|
59
|
+
rename(dest, backup);
|
|
60
|
+
rename(staging, dest);
|
|
61
|
+
}
|
|
62
|
+
catch (err) {
|
|
63
|
+
if (!fs.existsSync(dest) && fs.existsSync(backup)) {
|
|
64
|
+
try {
|
|
65
|
+
rename(backup, dest);
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
/* best-effort restore */
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
fs.rmSync(staging, { recursive: true, force: true });
|
|
72
|
+
throw new Error(`Failed to install ${src} -> ${dest}: ${err.message}`);
|
|
73
|
+
}
|
|
74
|
+
fs.rmSync(backup, { recursive: true, force: true });
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Serialize installs across the many concurrent `agents` invocations that pass
|
|
78
|
+
* through the helper-install path, so a burst copies once instead of stampeding
|
|
79
|
+
* the atomic swap. Locks a sentinel file beside the bundle (the bundle itself may
|
|
80
|
+
* not exist yet on first install) via the shared {@link withFileLock}.
|
|
81
|
+
*/
|
|
82
|
+
export function withInstallLock(dest, fn) {
|
|
83
|
+
const lockTarget = `${dest}.install-lock`;
|
|
84
|
+
ensureLockTarget(lockTarget);
|
|
85
|
+
// Pass proper-lockfile's `heartbeat` straight through: the install body is a
|
|
86
|
+
// fully SYNCHRONOUS chain of blocking `spawnSync`s (`cp -R`, then codesign /
|
|
87
|
+
// spctl), so the event loop never turns and proper-lockfile's own async mtime
|
|
88
|
+
// refresh can't fire. Callers invoke heartbeat() between those steps to keep a
|
|
89
|
+
// long hold from ageing past staleMs and being broken by a contending peer.
|
|
90
|
+
withFileLock(lockTarget, (heartbeat) => fn(heartbeat), {
|
|
91
|
+
staleMs: INSTALL_LOCK_STALE_MS,
|
|
92
|
+
acquireTimeoutMs: INSTALL_LOCK_ACQUIRE_TIMEOUT_MS,
|
|
93
|
+
});
|
|
94
|
+
}
|
|
@@ -304,8 +304,12 @@ export function buildLocalFindings(input) {
|
|
|
304
304
|
}
|
|
305
305
|
}
|
|
306
306
|
if (neverSynced) {
|
|
307
|
-
// Everything is "missing" because it was never synced —
|
|
308
|
-
//
|
|
307
|
+
// Everything is "missing" because it was never synced — collapse to ONE
|
|
308
|
+
// line. A never-synced version is almost always an old/unused install (you
|
|
309
|
+
// don't run a version you never synced), so this is a WARNING, not a "needs
|
|
310
|
+
// you now" critical: it isn't hurting anything until you actually launch it.
|
|
311
|
+
// The real criticals are a logged-out account or a hook/plugin missing from
|
|
312
|
+
// a version you DO keep synced (the `else` branch below).
|
|
309
313
|
const total = missingHooks.length + missingPlugins.length + missingOther.length;
|
|
310
314
|
if (total > 0) {
|
|
311
315
|
const breakdown = [
|
|
@@ -313,7 +317,7 @@ export function buildLocalFindings(input) {
|
|
|
313
317
|
missingPlugins.length ? `${missingPlugins.length} plugin${missingPlugins.length === 1 ? '' : 's'}` : '',
|
|
314
318
|
].filter(Boolean).join(', ');
|
|
315
319
|
out.push(finding({
|
|
316
|
-
severity: '
|
|
320
|
+
severity: 'warning', kind: 'never-synced', device, agent, version,
|
|
317
321
|
message: `never synced — ${total} resource${total === 1 ? '' : 's'}${breakdown ? ` (incl. ${breakdown})` : ''} not installed`,
|
|
318
322
|
}));
|
|
319
323
|
}
|
|
@@ -464,7 +468,11 @@ function duplicateHookFindings(device, dups) {
|
|
|
464
468
|
`${versions.length} version${versions.length === 1 ? '' : 's'} ` +
|
|
465
469
|
`(incl. ${group.slice(0, 2).map((d) => `'${d.name}'`).join(', ')}) — ${authority}`;
|
|
466
470
|
out.push({
|
|
467
|
-
|
|
471
|
+
// A hook that DIFFERS across versions is still installed and firing — it's
|
|
472
|
+
// stale/sync drift, not a missing hook. Resolvable by one sync; a WARNING,
|
|
473
|
+
// not a "needs you now" critical. (A genuinely MISSING hook stays critical
|
|
474
|
+
// via the missing-hook path.)
|
|
475
|
+
severity: 'warning',
|
|
468
476
|
kind: drift ? 'duplicate-hook-drift' : 'duplicate-hook',
|
|
469
477
|
device, agent, versions, message,
|
|
470
478
|
remediation: `agents sync ${agent}@all --yes`,
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/** Serve a cached snapshot without recomputing while it is younger than this. */
|
|
2
|
+
export declare const DOCTOR_OVERVIEW_FRESH_MS = 90000;
|
|
3
|
+
/** Injectable IO + clock so tests exercise the real fs at a temp dir, no mocks. */
|
|
4
|
+
export interface DoctorOverviewCacheDeps {
|
|
5
|
+
/** Cache directory (default: the real `~/.agents/.cache`). */
|
|
6
|
+
dir?: string;
|
|
7
|
+
/** Clock (default: {@link Date.now}). */
|
|
8
|
+
now?: () => number;
|
|
9
|
+
}
|
|
10
|
+
/** Read the last snapshot (best-effort; missing/corrupt/wrong-version → null). */
|
|
11
|
+
export declare function readDoctorOverviewCache(deps?: DoctorOverviewCacheDeps): {
|
|
12
|
+
fetchedAt: number;
|
|
13
|
+
payload: unknown;
|
|
14
|
+
} | null;
|
|
15
|
+
/** Persist a fresh overview payload (best-effort; tmp+rename so reads are atomic). */
|
|
16
|
+
export declare function writeDoctorOverviewCache(payload: unknown, deps?: DoctorOverviewCacheDeps): void;
|
|
17
|
+
/**
|
|
18
|
+
* Result of {@link enterDoctorOverviewGate}.
|
|
19
|
+
* - `cached` non-null → the caller MUST print this string and return; no compute.
|
|
20
|
+
* - `cached` null → the caller holds the singleflight lock: compute the
|
|
21
|
+
* overview, call {@link writeDoctorOverviewCache}, and invoke `release()` on
|
|
22
|
+
* the way out. Call `release()` in a `finally` so a compute that throws still
|
|
23
|
+
* frees the lock promptly (idempotent).
|
|
24
|
+
*/
|
|
25
|
+
export interface OverviewGate {
|
|
26
|
+
cached: string | null;
|
|
27
|
+
release?: () => void;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Enter the doctor-overview singleflight gate. Returns a cached string to print,
|
|
31
|
+
* or a lock token telling the caller to compute (and then write + release).
|
|
32
|
+
*
|
|
33
|
+
* Contract:
|
|
34
|
+
* - Fresh snapshot present (and not `forceRefresh`) → `{ cached }`, no lock.
|
|
35
|
+
* - Otherwise exactly one caller holds the lock and gets `{ cached: null,
|
|
36
|
+
* release }`; everyone else blocks on the lock, then (on acquiring it)
|
|
37
|
+
* double-checks and serves the winner's fresh write — or, if the winner runs
|
|
38
|
+
* past the wait budget, serves the last snapshot — rather than recomputing.
|
|
39
|
+
* - Never throws: any IO/lock failure degrades to a compute token or a served
|
|
40
|
+
* snapshot.
|
|
41
|
+
*/
|
|
42
|
+
export declare function enterDoctorOverviewGate(opts?: {
|
|
43
|
+
forceRefresh?: boolean;
|
|
44
|
+
freshMs?: number;
|
|
45
|
+
}, deps?: DoctorOverviewCacheDeps): Promise<OverviewGate>;
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Singleflight + short-TTL disk cache for the `agents doctor --json` OVERVIEW
|
|
3
|
+
* payload (the bare, no-target form the menu-bar helper and other pollers read).
|
|
4
|
+
*
|
|
5
|
+
* Why this exists (RUSH-2153): the bare `doctor --json` overview is expensive —
|
|
6
|
+
* it probes every host CLI, spawns every installed agent CLI for its sign-in,
|
|
7
|
+
* and diffs every agent×version against its source. On an idle box that is a few
|
|
8
|
+
* seconds; on a loaded one it is minutes. The menu-bar helper polls it on a 60s
|
|
9
|
+
* timer with a per-*process* in-flight guard, so nothing coalesces ACROSS
|
|
10
|
+
* processes: a helper relaunch (or any second poller) each launches its own
|
|
11
|
+
* live compute, and a helper killed mid-run orphans its `doctor --json` child,
|
|
12
|
+
* which keeps spinning. In steady state this stacked to dozens of concurrent
|
|
13
|
+
* `doctor --json` processes pinning ~14 cores and driving load to ~300.
|
|
14
|
+
*
|
|
15
|
+
* The fix mirrors the {@link readStatsCache}/`writeStatsCache` mirror-file
|
|
16
|
+
* convention: reads are cache-first, and when a live compute IS needed exactly
|
|
17
|
+
* ONE runs at a time. The singleflight is the shared `proper-lockfile` lock (via
|
|
18
|
+
* `ensureLockTarget` + `lockfile.lock`) — the SAME battle-tested lock the rest of
|
|
19
|
+
* the CLI uses (fs-atomic.ts) — which owns two things a hand-rolled lock got
|
|
20
|
+
* wrong: (1) it auto-refreshes the lock's mtime on a timer while held, so a live
|
|
21
|
+
* computer whose compute runs for minutes is never mistaken for a crashed one
|
|
22
|
+
* and stolen; (2) `release()` only ever releases the lock THIS caller acquired,
|
|
23
|
+
* so a slow computer can't delete a successor's lock. Waiters block on the lock
|
|
24
|
+
* up to a bounded budget, then serve the last snapshot rather than pile on.
|
|
25
|
+
*
|
|
26
|
+
* The cache write is tmp+rename so a concurrent reader never sees a partial file.
|
|
27
|
+
* All IO is best-effort: a failure degrades to a live compute, never a throw.
|
|
28
|
+
*/
|
|
29
|
+
import * as fs from 'fs';
|
|
30
|
+
import * as path from 'path';
|
|
31
|
+
import lockfile from 'proper-lockfile';
|
|
32
|
+
import { getCacheDir } from '../state.js';
|
|
33
|
+
import { ensureLockTarget } from '../fs-atomic.js';
|
|
34
|
+
const CACHE_FILE = '.doctor-overview.json';
|
|
35
|
+
const LOCK_TARGET_FILE = '.doctor-overview.lock-target';
|
|
36
|
+
/** Serve a cached snapshot without recomputing while it is younger than this. */
|
|
37
|
+
export const DOCTOR_OVERVIEW_FRESH_MS = 90_000;
|
|
38
|
+
/**
|
|
39
|
+
* A held lock older than this is treated as a crashed computer and broken. The
|
|
40
|
+
* lock's mtime is auto-refreshed by proper-lockfile every `stale/2` while a live
|
|
41
|
+
* computer holds it (the event loop turns during the compute's `await`ed
|
|
42
|
+
* subprocess spawns), so this only ever breaks a genuinely dead holder.
|
|
43
|
+
*/
|
|
44
|
+
const LOCK_STALE_MS = 60_000;
|
|
45
|
+
/**
|
|
46
|
+
* How long a waiter blocks on the lock before giving up and serving the last
|
|
47
|
+
* snapshot. Sized to comfortably exceed a slow (multi-second-to-minutes) compute
|
|
48
|
+
* so a waiter normally gets the winner's fresh write; capped so a truly wedged
|
|
49
|
+
* holder never hangs the CLI (it serves stale instead).
|
|
50
|
+
*/
|
|
51
|
+
const LOCK_RETRIES = { retries: 240, factor: 1, minTimeout: 500, maxTimeout: 500 };
|
|
52
|
+
function cachePath(dir) {
|
|
53
|
+
return path.join(dir, CACHE_FILE);
|
|
54
|
+
}
|
|
55
|
+
/** Read the last snapshot (best-effort; missing/corrupt/wrong-version → null). */
|
|
56
|
+
export function readDoctorOverviewCache(deps = {}) {
|
|
57
|
+
const dir = deps.dir ?? getCacheDir();
|
|
58
|
+
try {
|
|
59
|
+
const parsed = JSON.parse(fs.readFileSync(cachePath(dir), 'utf-8'));
|
|
60
|
+
if (parsed && parsed.version === 1 && typeof parsed.fetchedAt === 'number') {
|
|
61
|
+
return { fetchedAt: parsed.fetchedAt, payload: parsed.payload };
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
// missing or corrupt — treat as no snapshot
|
|
66
|
+
}
|
|
67
|
+
return null;
|
|
68
|
+
}
|
|
69
|
+
/** Persist a fresh overview payload (best-effort; tmp+rename so reads are atomic). */
|
|
70
|
+
export function writeDoctorOverviewCache(payload, deps = {}) {
|
|
71
|
+
const dir = deps.dir ?? getCacheDir();
|
|
72
|
+
const now = deps.now ?? Date.now;
|
|
73
|
+
try {
|
|
74
|
+
if (!fs.existsSync(dir))
|
|
75
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
76
|
+
const body = { version: 1, fetchedAt: now(), payload };
|
|
77
|
+
const tmp = `${cachePath(dir)}.tmp.${process.pid}`;
|
|
78
|
+
fs.writeFileSync(tmp, JSON.stringify(body, null, 2));
|
|
79
|
+
fs.renameSync(tmp, cachePath(dir));
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
// best-effort; a failed write just means the next read falls back to live
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Enter the doctor-overview singleflight gate. Returns a cached string to print,
|
|
87
|
+
* or a lock token telling the caller to compute (and then write + release).
|
|
88
|
+
*
|
|
89
|
+
* Contract:
|
|
90
|
+
* - Fresh snapshot present (and not `forceRefresh`) → `{ cached }`, no lock.
|
|
91
|
+
* - Otherwise exactly one caller holds the lock and gets `{ cached: null,
|
|
92
|
+
* release }`; everyone else blocks on the lock, then (on acquiring it)
|
|
93
|
+
* double-checks and serves the winner's fresh write — or, if the winner runs
|
|
94
|
+
* past the wait budget, serves the last snapshot — rather than recomputing.
|
|
95
|
+
* - Never throws: any IO/lock failure degrades to a compute token or a served
|
|
96
|
+
* snapshot.
|
|
97
|
+
*/
|
|
98
|
+
export async function enterDoctorOverviewGate(opts = {}, deps = {}) {
|
|
99
|
+
const dir = deps.dir ?? getCacheDir();
|
|
100
|
+
const now = deps.now ?? Date.now;
|
|
101
|
+
const freshMs = opts.freshMs ?? DOCTOR_OVERVIEW_FRESH_MS;
|
|
102
|
+
const serveFresh = () => {
|
|
103
|
+
if (opts.forceRefresh)
|
|
104
|
+
return null;
|
|
105
|
+
const c = readDoctorOverviewCache({ dir });
|
|
106
|
+
if (c && now() - c.fetchedAt < freshMs)
|
|
107
|
+
return JSON.stringify(c.payload, null, 2);
|
|
108
|
+
return null;
|
|
109
|
+
};
|
|
110
|
+
// 1. Fast path: a fresh snapshot serves without any compute or lock.
|
|
111
|
+
const fast = serveFresh();
|
|
112
|
+
if (fast !== null)
|
|
113
|
+
return { cached: fast };
|
|
114
|
+
try {
|
|
115
|
+
if (!fs.existsSync(dir))
|
|
116
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
117
|
+
}
|
|
118
|
+
catch {
|
|
119
|
+
// Can't even make the cache dir — fall back to an unguarded compute.
|
|
120
|
+
return { cached: null, release: () => { } };
|
|
121
|
+
}
|
|
122
|
+
const lockTarget = path.join(dir, LOCK_TARGET_FILE);
|
|
123
|
+
ensureLockTarget(lockTarget);
|
|
124
|
+
// 2. Singleflight via proper-lockfile: one caller holds the lock and computes;
|
|
125
|
+
// the rest block here until it releases.
|
|
126
|
+
let release = null;
|
|
127
|
+
try {
|
|
128
|
+
release = await lockfile.lock(lockTarget, {
|
|
129
|
+
stale: LOCK_STALE_MS,
|
|
130
|
+
retries: LOCK_RETRIES,
|
|
131
|
+
// A peer broke our lock (only possible if we somehow went stale). Don't
|
|
132
|
+
// crash on the async callback; we re-check the cache and serve/recompute.
|
|
133
|
+
onCompromised: () => { },
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
catch {
|
|
137
|
+
// 3. Winner held the lock past our wait budget. Serve the last snapshot
|
|
138
|
+
// (even if stale) rather than pile on; only if there is genuinely none do
|
|
139
|
+
// we compute unguarded (rare cold-start under sustained load).
|
|
140
|
+
const c = readDoctorOverviewCache({ dir });
|
|
141
|
+
if (c)
|
|
142
|
+
return { cached: JSON.stringify(c.payload, null, 2) };
|
|
143
|
+
return { cached: null, release: () => { } };
|
|
144
|
+
}
|
|
145
|
+
// 4. Acquired. The winner may have written a fresh snapshot while we waited —
|
|
146
|
+
// serve it and release, instead of recomputing.
|
|
147
|
+
const afterWait = serveFresh();
|
|
148
|
+
if (afterWait !== null) {
|
|
149
|
+
// AWAIT, don't fire-and-forget: returning while the lockfile is still on
|
|
150
|
+
// disk makes the next caller retry against a lock that is logically free —
|
|
151
|
+
// the same pile-up this gate exists to prevent, just narrowed to the window
|
|
152
|
+
// between return and unlink. We are already in an async function, so the
|
|
153
|
+
// wait costs one unlink.
|
|
154
|
+
await release().catch(() => { });
|
|
155
|
+
return { cached: afterWait };
|
|
156
|
+
}
|
|
157
|
+
const rel = release;
|
|
158
|
+
let released = false;
|
|
159
|
+
return {
|
|
160
|
+
cached: null,
|
|
161
|
+
release: () => {
|
|
162
|
+
if (released)
|
|
163
|
+
return;
|
|
164
|
+
released = true;
|
|
165
|
+
void rel();
|
|
166
|
+
},
|
|
167
|
+
};
|
|
168
|
+
}
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import { spawnSync } from 'child_process';
|
|
11
11
|
import { isControlDevice } from './registry.js';
|
|
12
|
+
import { isSelfHost } from './self-host.js';
|
|
12
13
|
import { buildSshInvocation, sshTargetFor, writeAskpassShim } from './connect.js';
|
|
13
14
|
/** npm dist-tags / semver pins only — rejects shell metacharacters. */
|
|
14
15
|
export const FLEET_VERSION_RE = /^[A-Za-z0-9._-]+$/;
|
|
@@ -52,7 +53,11 @@ export function planFleetTargets(reg) {
|
|
|
52
53
|
* surface — so their `skip` reason still flows through as an `unreachable` row.
|
|
53
54
|
*/
|
|
54
55
|
export function remoteFleetTargets(planned, self) {
|
|
55
|
-
|
|
56
|
+
// Exclude self by name AND by full identity (tailscale dnsName, loopback): a
|
|
57
|
+
// device referenced by its dnsName slipped past the bare name check and got a
|
|
58
|
+
// remote version+doctor dial back to THIS box, which orphaned on timeout and
|
|
59
|
+
// piled up (RUSH-2114). `isSelfHost` matches every alias the box answers to.
|
|
60
|
+
return planned.filter((t) => t.device.name !== self && !isSelfHost(t.device.name) && t.skip !== 'control');
|
|
56
61
|
}
|
|
57
62
|
/**
|
|
58
63
|
* Decide whether a fleet-health target should skip the expensive version+doctor
|
|
@@ -189,7 +194,7 @@ export function runFleet(targets, cmd, opts = {}) {
|
|
|
189
194
|
continue;
|
|
190
195
|
}
|
|
191
196
|
try {
|
|
192
|
-
const isSelf = opts.self !== undefined && t.device.name === opts.self;
|
|
197
|
+
const isSelf = (opts.self !== undefined && t.device.name === opts.self) || isSelfHost(t.device.name);
|
|
193
198
|
const res = isSelf ? localRunner(cmd) : runner(t.device, cmd);
|
|
194
199
|
const ok = res.code === 0;
|
|
195
200
|
const detail = (res.stderr || res.stdout).trim().slice(0, 200);
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* True when `name` refers to the local machine. Case-insensitive and
|
|
3
|
+
* trailing-dot-tolerant. Use this everywhere a `--host`/fleet target is compared
|
|
4
|
+
* against "self" so a tailscale-name reference short-circuits to a local run
|
|
5
|
+
* instead of self-SSHing.
|
|
6
|
+
*/
|
|
7
|
+
export declare function isSelfHost(name: string | undefined | null): boolean;
|
|
8
|
+
/** Test hook: drop the memoized alias set so a fresh registry/env is re-read. */
|
|
9
|
+
export declare function resetSelfHostCache(): void;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* "Is this hostname the local machine?" — matched against every identity the box
|
|
3
|
+
* answers to, not just its short id.
|
|
4
|
+
*
|
|
5
|
+
* The self-checks that gate `--host` dispatch and the fleet-health fan-out used to
|
|
6
|
+
* compare only against {@link machineId} (the lowercased short hostname, e.g.
|
|
7
|
+
* `zion`). A caller that referenced the box by its **tailscale MagicDNS name**
|
|
8
|
+
* (`zion.tail1a85a1.ts.net`) — which is exactly what `fleetDialTarget` and the
|
|
9
|
+
* Factory floor's `--host` probes use — slipped past the check and SSH'd to the
|
|
10
|
+
* LOCAL box over its own tailscale name. On a loaded machine that self-SSH'd
|
|
11
|
+
* `doctor --json` orphaned on timeout and piled up until the host was crushed
|
|
12
|
+
* (RUSH-2114). Matching the full identity set closes that gap at the source.
|
|
13
|
+
*/
|
|
14
|
+
import { machineId } from '../machine-id.js';
|
|
15
|
+
import { loadDevicesSync } from './registry.js';
|
|
16
|
+
const LOOPBACK = ['localhost', '127.0.0.1', '::1'];
|
|
17
|
+
/** Lowercase + strip a trailing dot (FQDNs are equivalent with or without it). */
|
|
18
|
+
function normalize(name) {
|
|
19
|
+
return name.trim().toLowerCase().replace(/\.$/, '');
|
|
20
|
+
}
|
|
21
|
+
let cached = null;
|
|
22
|
+
/**
|
|
23
|
+
* Every name that resolves to THIS machine: the short id, loopback, and the self
|
|
24
|
+
* device's tailscale dnsName plus its short form. Computed once per process — the
|
|
25
|
+
* self identity does not change under a running CLI — and reads the registry
|
|
26
|
+
* best-effort (an unreadable registry still leaves the short id + loopback).
|
|
27
|
+
*/
|
|
28
|
+
function selfAliases() {
|
|
29
|
+
if (cached)
|
|
30
|
+
return cached;
|
|
31
|
+
const aliases = new Set([machineId(), ...LOOPBACK]);
|
|
32
|
+
try {
|
|
33
|
+
const dns = loadDevicesSync()[machineId()]?.address?.dnsName;
|
|
34
|
+
if (dns) {
|
|
35
|
+
const d = normalize(dns);
|
|
36
|
+
aliases.add(d);
|
|
37
|
+
aliases.add(d.split('.')[0]); // the short form of the FQDN
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
/* registry unreadable — the short id + loopback aliases still hold */
|
|
42
|
+
}
|
|
43
|
+
cached = aliases;
|
|
44
|
+
return cached;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* True when `name` refers to the local machine. Case-insensitive and
|
|
48
|
+
* trailing-dot-tolerant. Use this everywhere a `--host`/fleet target is compared
|
|
49
|
+
* against "self" so a tailscale-name reference short-circuits to a local run
|
|
50
|
+
* instead of self-SSHing.
|
|
51
|
+
*/
|
|
52
|
+
export function isSelfHost(name) {
|
|
53
|
+
if (!name)
|
|
54
|
+
return false;
|
|
55
|
+
const n = normalize(name);
|
|
56
|
+
return n.length > 0 && selfAliases().has(n);
|
|
57
|
+
}
|
|
58
|
+
/** Test hook: drop the memoized alias set so a fresh registry/env is re-read. */
|
|
59
|
+
export function resetSelfHostCache() {
|
|
60
|
+
cached = null;
|
|
61
|
+
}
|
|
@@ -25,6 +25,7 @@ import { stripRoutingFlags, buildRemoteAgentsInvocation, HOST_ROUTING_SPECS, } f
|
|
|
25
25
|
import { resolveRemoteOsSync } from './remote-os.js';
|
|
26
26
|
import { machineId } from '../session/sync/config.js';
|
|
27
27
|
import { loadDevices } from '../devices/registry.js';
|
|
28
|
+
import { isSelfHost } from '../devices/self-host.js';
|
|
28
29
|
import { fanOutDevices, planFleetTargets, runLocalCommand, runOnDevice, } from '../devices/fleet.js';
|
|
29
30
|
import { platformGroupLabel } from '../devices/health-report.js';
|
|
30
31
|
/**
|
|
@@ -334,7 +335,7 @@ export async function runFleetPassthrough(command, allArgs, spec, opts = {}) {
|
|
|
334
335
|
const localRunner = opts.localRunner ?? runLocalCommand;
|
|
335
336
|
const results = await fanOutDevices(targets, async (target) => {
|
|
336
337
|
const cmd = ['agents', ...forwarded];
|
|
337
|
-
const isSelf = target.device.name.toLowerCase() === self.toLowerCase();
|
|
338
|
+
const isSelf = target.device.name.toLowerCase() === self.toLowerCase() || isSelfHost(target.device.name);
|
|
338
339
|
const res = isSelf ? localRunner(cmd) : runner(target.device, cmd);
|
|
339
340
|
if (res.code !== 0) {
|
|
340
341
|
const detail = (res.stderr || res.stdout || 'unreachable').trim().slice(0, 200);
|
|
@@ -443,11 +444,12 @@ export async function maybeRunOnHost(command, allArgs, opts) {
|
|
|
443
444
|
if (!hostName)
|
|
444
445
|
return false;
|
|
445
446
|
// Running against your own machine is just a local run — skip the SSH round-trip.
|
|
446
|
-
//
|
|
447
|
-
//
|
|
448
|
-
//
|
|
449
|
-
//
|
|
450
|
-
|
|
447
|
+
// Match EVERY identity the box answers to (short id, loopback, tailscale
|
|
448
|
+
// dnsName), not just machineId() — a `--host <self-dnsName>` used to slip past a
|
|
449
|
+
// short-hostname-only check and self-SSH (RUSH-2114). Strip the routing flags
|
|
450
|
+
// from process.argv so the local command never sees an unregistered
|
|
451
|
+
// `--host`/`--device` and dies with "unknown option".
|
|
452
|
+
if (isSelfHost(hostName)) {
|
|
451
453
|
const stripped = stripRoutingFlags(allArgs, STRIP_SPECS);
|
|
452
454
|
process.argv = [process.argv[0], process.argv[1], ...stripped];
|
|
453
455
|
return false;
|
|
Binary file
|
|
Binary file
|