@phnx-labs/agents-cli 1.22.29 → 1.22.31

package/dist/lib/daemon-health.js

@@ -0,0 +1,84 @@
+/**
+ * Per-subsystem health record for the always-on daemon.
+ *
+ * Today a subsystem failure inside `runDaemon()` (daemon.ts) is a single
+ * `log('ERROR', ...)` line that scrolls out of the log file and is never
+ * surfaced anywhere else — `agents daemon status` has no way to answer "is the
+ * secrets broker actually healthy right now?" beyond "the daemon process is
+ * alive". This module gives every subsystem a small persisted record —
+ * {@link SubsystemHealth} — so `agents daemon status` / `agents daemon
+ * services` can report health, not just liveness (RUSH-2354).
+ *
+ * Scheduled routines get this for free once migrated onto `agents routines`
+ * (their run history already carries success/failure — `agents routines
+ * stats`). This module exists for the two subsystems that predate routines
+ * and have no run history of their own: the secrets broker and the browser
+ * IPC server.
+ *
+ * File-backed (one JSON object keyed by subsystem name) rather than in-memory
+ * because `agents daemon status` runs as a SEPARATE process from the daemon —
+ * it must read what the daemon last recorded, not maintain its own state.
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import { getDaemonDir } from './state.js';
+const HEALTH_FILE = 'health.json';
+/** Stable subsystem identifiers shared by the daemon (writer) and `agents daemon` (reader). */
+export const SUBSYSTEM_SECRETS_BROKER = 'secrets-broker';
+export const SUBSYSTEM_BROWSER_IPC = 'browser-ipc';
+function getHealthPath() {
+    return path.join(getDaemonDir(), HEALTH_FILE);
+}
+function readAll() {
+    try {
+        const raw = fs.readFileSync(getHealthPath(), 'utf-8');
+        const parsed = JSON.parse(raw);
+        if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+            return parsed;
+        }
+        return {};
+    }
+    catch {
+        return {};
+    }
+}
+function writeAll(records) {
+    const healthPath = getHealthPath();
+    fs.mkdirSync(path.dirname(healthPath), { recursive: true });
+    fs.writeFileSync(healthPath, JSON.stringify(records), 'utf-8');
+    try {
+        fs.chmodSync(healthPath, 0o600);
+    }
+    catch { /* best effort */ }
+}
+function blankRecord(subsystem) {
+    return { subsystem, lastError: null, lastErrorAt: null, consecutiveFailures: 0, lastOkAt: null };
+}
+/** Record a successful subsystem check-in — clears the failure streak. */
+export function recordSubsystemOk(subsystem, at = new Date().toISOString()) {
+    const all = readAll();
+    const existing = all[subsystem] ?? blankRecord(subsystem);
+    all[subsystem] = { ...existing, subsystem, consecutiveFailures: 0, lastOkAt: at };
+    writeAll(all);
+}
+/** Record a subsystem failure — bumps the consecutive-failure streak. */
+export function recordSubsystemError(subsystem, error, at = new Date().toISOString()) {
+    const all = readAll();
+    const existing = all[subsystem] ?? blankRecord(subsystem);
+    all[subsystem] = {
+        ...existing,
+        subsystem,
+        lastError: error,
+        lastErrorAt: at,
+        consecutiveFailures: existing.consecutiveFailures + 1,
+    };
+    writeAll(all);
+}
+/** Read one subsystem's health record, or null if it has never reported in. */
+export function readSubsystemHealth(subsystem) {
+    return readAll()[subsystem] ?? null;
+}
+/** Read every subsystem's health record, sorted by subsystem name. */
+export function readAllSubsystemHealth() {
+    return Object.values(readAll()).sort((a, b) => a.subsystem.localeCompare(b.subsystem));
+}

package/dist/lib/daemon-ticks.d.ts

@@ -0,0 +1,81 @@
+/**
+ * Daemon housekeeping ticks — one-shot bodies invoked by system routines.
+ *
+ * RUSH-2353: these were ~15 hardcoded `setInterval` timers inside
+ * `runDaemon()` (daemon.ts) — a parallel, inferior reimplementation of the
+ * routines system (no declaration, no run history, no pause/disable, no
+ * device pin). Each function here is one migrated tick's *body*, unchanged in
+ * behavior, now invoked as a detached one-shot process by a system routine's
+ * `command:` (via the `agents __daemon-tick <name>` entrypoint in index.ts)
+ * instead of an in-process interval closure.
+ *
+ * Overlap protection that used to live in daemon.ts as a per-tick boolean
+ * flag (`watchdogInFlight`, `probingDevices`, ...) is now provided by the
+ * routine runner's launch claim (`withRoutineLaunchClaim` in runner.ts) — two
+ * fires of the same routine name never run concurrently.
+ *
+ * Output goes to `console.log`/`console.error`: each tick runs as a spawned
+ * child of `executeCommandJobDetached`, which redirects the child's stdout to
+ * the run's `stdout.log` — so this doubles as the run's log, readable via
+ * `agents routines runs <name>`.
+ */
+/** ~every 3 min. Mirrors the old WATCHDOG_TICK_MS. */
+export declare function runWatchdogTick(): Promise<void>;
+/**
+ * Device probe: refresh registered devices' reachability and detect newly
+ * appeared tailnet nodes, dropping a sentinel per pending device so the
+ * menu-bar helper can surface "NEW DEVICES -> Register / Ignore". Refresh
+ * mode never auto-registers a newcomer. A machine without tailscale is a
+ * clean no-op. ~every 3 min.
+ */
+export declare function runDeviceProbeTick(): Promise<void>;
+/**
+ * tmux hook reconcile: retrofit the guarded `pane-died` hook onto managed
+ * `agents run` sessions a pre-fix binary left with the old unconditional
+ * hook. Non-destructive: set-hook only, never a kill or detach. ~every 5 min.
+ */
+export declare function runTmuxReconcileTick(): Promise<void>;
+/**
+ * Launch-health self-heal: probe that each agent's DEFAULT version actually
+ * LAUNCHES, and repair a gutted install. Never repoints the global default
+ * (allowDefaultSwitch: false) — a background default switch would be a
+ * silent logout. ~every 6h.
+ */
+export declare function runLaunchHealthTick(): Promise<void>;
+/**
+ * Fleet cache warm: publish THIS host's row for the caches `agents fleet
+ * status` / `agents devices list` read (PUBLISH-OWN / READ-UNION, RUSH-2061).
+ * ~every 3 min.
+ */
+export declare function runFleetCacheWarmTick(): Promise<void>;
+/**
+ * Session-status cache warm (RUSH-2062): publish THIS host's local active
+ * sessions so menubar / Factory / watchdog / CLI share one warm snapshot.
+ * Publish-own only (no cross-host SSH). ~every 3 min.
+ */
+export declare function runSessionCacheWarmTick(): Promise<void>;
+/**
+ * Usage refresh: keep the usage cache the `agents run` router reads
+ * (RUSH-2061, readOnly hot path) fresh, WITHOUT the hot path ever fetching.
+ * This host is the sole writer for its own local accounts. ~every 60s
+ * (USAGE_REFRESH_TICK_MS in usage-refresh.ts — keep in sync).
+ */
+export declare function runUsageRefreshTick(): Promise<void>;
+/**
+ * Auto-dispatch: for any managed project that has opted in (autoDispatch:true
+ * + maxAgents>0 in ~/.agents/factory/projects.json), pick up Linear tickets
+ * delegated to an agent and still in Todo, and dispatch each through
+ * agents-cli's own cloud-provider layer. OFF unless a project opts in; no
+ * opted-in project or no LINEAR_API_KEY is a clean no-op. ~every 3 min.
+ *
+ * Migrated to a routine (RUSH-2353) so it inherits the `devices:` allowlist —
+ * pin with `agents routines devices auto-dispatch --set <one>` to fix the
+ * shared-input double-fire problem this job had hardcoded (every daemon on
+ * the fleet polled the same Linear queue with no coordination).
+ */
+export declare function runAutoDispatchTick(): Promise<void>;
+/** Registry: routine-facing name -> tick body. Keys match the shipped routine YAML names. */
+export declare const DAEMON_TICKS: Record<string, () => Promise<void>>;
+export declare const DAEMON_TICK_ROUTINE_NAMES: readonly string[];
+/** Run one named tick, or throw for an unknown name (fails the routine run loud). */
+export declare function runDaemonTick(name: string): Promise<void>;

package/dist/lib/daemon-ticks.js

@@ -0,0 +1,190 @@
+/**
+ * Daemon housekeeping ticks — one-shot bodies invoked by system routines.
+ *
+ * RUSH-2353: these were ~15 hardcoded `setInterval` timers inside
+ * `runDaemon()` (daemon.ts) — a parallel, inferior reimplementation of the
+ * routines system (no declaration, no run history, no pause/disable, no
+ * device pin). Each function here is one migrated tick's *body*, unchanged in
+ * behavior, now invoked as a detached one-shot process by a system routine's
+ * `command:` (via the `agents __daemon-tick <name>` entrypoint in index.ts)
+ * instead of an in-process interval closure.
+ *
+ * Overlap protection that used to live in daemon.ts as a per-tick boolean
+ * flag (`watchdogInFlight`, `probingDevices`, ...) is now provided by the
+ * routine runner's launch claim (`withRoutineLaunchClaim` in runner.ts) — two
+ * fires of the same routine name never run concurrently.
+ *
+ * Output goes to `console.log`/`console.error`: each tick runs as a spawned
+ * child of `executeCommandJobDetached`, which redirects the child's stdout to
+ * the run's `stdout.log` — so this doubles as the run's log, readable via
+ * `agents routines runs <name>`.
+ */
+import { getConfigValue } from './device-config.js';
+/** ~every 3 min. Mirrors the old WATCHDOG_TICK_MS. */
+export async function runWatchdogTick() {
+    if (getConfigValue('watchdog.enabled').value !== true) {
+        console.log('watchdog: disabled (watchdog.enabled != true) — skipping');
+        return;
+    }
+    const { runWatchdogPass } = await import('./watchdog/service.js');
+    const result = await runWatchdogPass({ nudge: true });
+    console.log(`watchdog: ${result.counts.total} live, ${result.counts.stalled} stalled, ${result.counts.nudged} nudged`);
+}
+/**
+ * Device probe: refresh registered devices' reachability and detect newly
+ * appeared tailnet nodes, dropping a sentinel per pending device so the
+ * menu-bar helper can surface "NEW DEVICES -> Register / Ignore". Refresh
+ * mode never auto-registers a newcomer. A machine without tailscale is a
+ * clean no-op. ~every 3 min.
+ */
+export async function runDeviceProbeTick() {
+    const { runDeviceSync } = await import('./devices/sync.js');
+    const { reconcilePendingSentinels } = await import('./devices/pending.js');
+    const dev = await runDeviceSync({ soft: true, mode: 'refresh' });
+    if (!dev.ok) {
+        console.log('device probe: sync not ok — skipping sentinel reconcile');
+        return;
+    }
+    reconcilePendingSentinels(dev.pending);
+    if (dev.pending.length) {
+        console.log(`devices: ${dev.pending.length} new pending (${dev.pending.map((p) => p.name).join(', ')})`);
+    }
+    else {
+        console.log('device probe: no new pending devices');
+    }
+}
+/**
+ * tmux hook reconcile: retrofit the guarded `pane-died` hook onto managed
+ * `agents run` sessions a pre-fix binary left with the old unconditional
+ * hook. Non-destructive: set-hook only, never a kill or detach. ~every 5 min.
+ */
+export async function runTmuxReconcileTick() {
+    const { isTmuxInstalled } = await import('./tmux/binary.js');
+    if (!isTmuxInstalled()) {
+        console.log('tmux reconcile: tmux not installed — skipping');
+        return;
+    }
+    const { reconcileSessionHooks } = await import('./tmux/session.js');
+    const r = await reconcileSessionHooks();
+    console.log(`tmux: retrofitted pane-died hook on ${r.reconciled} session(s)`);
+}
+/**
+ * Launch-health self-heal: probe that each agent's DEFAULT version actually
+ * LAUNCHES, and repair a gutted install. Never repoints the global default
+ * (allowDefaultSwitch: false) — a background default switch would be a
+ * silent logout. ~every 6h.
+ */
+export async function runLaunchHealthTick() {
+    const { healBrokenDefaultLaunches } = await import('./versions.js');
+    const { repaired, unhealed } = await healBrokenDefaultLaunches((m) => console.log(`launch-health: ${m}`), { allowDefaultSwitch: false });
+    if (repaired.length)
+        console.log(`launch-health: repaired ${repaired.join(', ')}`);
+    if (unhealed.length) {
+        console.log(`launch-health: ${unhealed.join(', ')} won't launch and will not be auto-switched — choose a version with \`agents use <agent> <version>\` or \`agents add <agent>@latest\``);
+    }
+    if (!repaired.length && !unhealed.length)
+        console.log('launch-health: all default launches healthy');
+}
+/**
+ * Fleet cache warm: publish THIS host's row for the caches `agents fleet
+ * status` / `agents devices list` read (PUBLISH-OWN / READ-UNION, RUSH-2061).
+ * ~every 3 min.
+ */
+export async function runFleetCacheWarmTick() {
+    const { machineId } = await import('./machine-id.js');
+    const self = machineId();
+    const { probeLocalFleetAuth, writeFleetAuthRows } = await import('./auth-health.js');
+    const { getCliVersion } = await import('./version.js');
+    const authRows = await probeLocalFleetAuth({ cliVersion: getCliVersion() });
+    writeFleetAuthRows(self, authRows);
+    const { publishLocalFleetStatus } = await import('./fleet-status.js');
+    const row = await publishLocalFleetStatus(self);
+    console.log(`fleet cache warm: ${authRows.length} auth row(s), ${row.agents.running} running agent(s) on ${self}`);
+}
+/**
+ * Session-status cache warm (RUSH-2062): publish THIS host's local active
+ * sessions so menubar / Factory / watchdog / CLI share one warm snapshot.
+ * Publish-own only (no cross-host SSH). ~every 3 min.
+ */
+export async function runSessionCacheWarmTick() {
+    const { publishLocalActiveSessions } = await import('./session/session-cache.js');
+    const r = await publishLocalActiveSessions();
+    console.log(`session cache warm: ${r.sessions.length} local session(s)`);
+}
+/**
+ * Usage refresh: keep the usage cache the `agents run` router reads
+ * (RUSH-2061, readOnly hot path) fresh, WITHOUT the hot path ever fetching.
+ * This host is the sole writer for its own local accounts. ~every 60s
+ * (USAGE_REFRESH_TICK_MS in usage-refresh.ts — keep in sync).
+ */
+export async function runUsageRefreshTick() {
+    const { runUsageRefresh, buildLocalUsageAccounts } = await import('./usage-refresh.js');
+    const { writeClaudeUsageCache } = await import('./usage.js');
+    const { usageRateLimitedUntil } = await import('./usage-backoff.js');
+    const r = await runUsageRefresh({
+        listAccounts: buildLocalUsageAccounts,
+        writeUsageCache: writeClaudeUsageCache,
+        backoffUntil: usageRateLimitedUntil,
+    });
+    console.log(`usage refresh: ${r.refreshed} refreshed, ${r.failed} failed, ${r.skippedNotDue} not-due, ${r.skippedBackoff} backed-off, ${r.skippedCap} capped`);
+}
+/**
+ * Auto-dispatch: for any managed project that has opted in (autoDispatch:true
+ * + maxAgents>0 in ~/.agents/factory/projects.json), pick up Linear tickets
+ * delegated to an agent and still in Todo, and dispatch each through
+ * agents-cli's own cloud-provider layer. OFF unless a project opts in; no
+ * opted-in project or no LINEAR_API_KEY is a clean no-op. ~every 3 min.
+ *
+ * Migrated to a routine (RUSH-2353) so it inherits the `devices:` allowlist —
+ * pin with `agents routines devices auto-dispatch --set <one>` to fix the
+ * shared-input double-fire problem this job had hardcoded (every daemon on
+ * the fleet polled the same Linear queue with no coordination).
+ */
+export async function runAutoDispatchTick() {
+    const { readAutoDispatchProjects, isEligible, autoDispatchTick } = await import('./auto-dispatch.js');
+    const projects = readAutoDispatchProjects();
+    if (!projects.some(isEligible)) {
+        console.log('auto-dispatch: no opted-in project — skipping');
+        return;
+    }
+    const { createLinearGateway } = await import('./auto-dispatch-linear.js');
+    const linear = createLinearGateway();
+    if (!linear) {
+        console.log('auto-dispatch: no LINEAR_API_KEY configured — skipping');
+        return;
+    }
+    const { createProviderDispatcher } = await import('./auto-dispatch-provider.js');
+    const dispatcher = createProviderDispatcher();
+    const dispatched = await autoDispatchTick({
+        projects,
+        linear,
+        dispatcher,
+        log: (lvl, m) => (lvl === 'ERROR' ? console.error(m) : console.log(m)),
+    });
+    if (dispatched.length) {
+        console.log(`auto-dispatch: started ${dispatched.length} delegated ticket(s): ${dispatched.map((d) => d.identifier).join(', ')}`);
+    }
+    else {
+        console.log('auto-dispatch: no delegated tickets to dispatch');
+    }
+}
+/** Registry: routine-facing name -> tick body. Keys match the shipped routine YAML names. */
+export const DAEMON_TICKS = {
+    watchdog: runWatchdogTick,
+    'device-probe': runDeviceProbeTick,
+    'tmux-reconcile': runTmuxReconcileTick,
+    'launch-health': runLaunchHealthTick,
+    'fleet-cache-warm': runFleetCacheWarmTick,
+    'session-cache-warm': runSessionCacheWarmTick,
+    'usage-refresh': runUsageRefreshTick,
+    'auto-dispatch': runAutoDispatchTick,
+};
+export const DAEMON_TICK_ROUTINE_NAMES = Object.freeze(Object.keys(DAEMON_TICKS));
+/** Run one named tick, or throw for an unknown name (fails the routine run loud). */
+export async function runDaemonTick(name) {
+    const fn = DAEMON_TICKS[name];
+    if (!fn) {
+        throw new Error(`Unknown daemon tick '${name}'. Known: ${Object.keys(DAEMON_TICKS).join(', ')}`);
+    }
+    await fn();
+}

package/dist/lib/daemon.d.ts

@@ -58,25 +58,45 @@ export declare function isDaemonRunning(): boolean;
  * writeDaemonPid() unconditionally, clobber a live daemon's recorded PID, and
  * run a second JobScheduler concurrently, so every cron routine fires twice.
  *
- * Returns true and records our PID when no other live daemon owns the pid file;
- * returns false when a live daemon already holds it (the caller must exit
- * without touching any further state). The read-decide-write is serialized
- * behind the same O_EXCL start lock startDaemon() uses, so two _run processes
- * can't both claim in the window between the liveness check and the write.
+ * LAST-WINS takeover (SING-11, RUSH-2352): when a live daemon already owns the
+ * pid file, this does NOT defer to it — it evicts the incumbent and becomes the
+ * survivor, so a second install can never leave two daemons running. Returns true
+ * and records our PID once the incumbent is provably dead (its resources
+ * released). Returns false ONLY when another `__daemon-run` currently holds the
+ * O_EXCL start lock — i.e. a concurrent claimer is mid-takeover and will be the
+ * singleton — in which case the caller must exit without touching further state.
+ * The read-evict-write is serialized behind the same start lock startDaemon()
+ * uses, so two `_run` processes can't both claim in the window between the
+ * liveness check and the write.
  */
 export declare function claimDaemonInstance(): boolean;
 /**
- * Reap stray duplicate daemon processes — a `__daemon-run` of THIS install that
- * isn't this process and isn't the pid-file owner. Mirrors the browser orphan
- * reaper (below): a predecessor that was SIGKILLed/OOM-ed without cleaning up,
- * or a duplicate that lost the pid-file write race, would otherwise keep a
- * second scheduler alive and double-fire jobs even after claimDaemonInstance()
- * hands the pid file to the survivor.
- *
- * Scoped to our own launch entry (process.argv[1]) so it only ever targets
- * daemons of the same installation — a daemon from a different install / home
- * (e.g. a side-by-side dev build, or a test fixture) is a legitimately separate
- * instance and is left untouched. POSIX-only (uses `ps`); a no-op on Windows.
+ * Record this daemon in the device's instance registry — a marker file named by
+ * pid under `<daemonDir>/instances/`. The registry, not a process scan, is how
+ * the reaper enumerates the device singleton: because the dir lives INSIDE the
+ * daemon dir (`AGENTS_DAEMON_DIR` ?? `<HOME>/.agents/.cache/helpers/daemon`), every
+ * daemon of one device — however it was launched — registers in the same place,
+ * while a genuinely separate install/home or a test fixture registers under its
+ * own daemon dir and is invisible here. This is what fixes the two-entry pile-up:
+ * the compiled `dist/bin/agents` binary and the `node <shim>` JS entry have
+ * different `process.argv[1]`, so the old launch-entry-scoped `ps` match never
+ * reaped across them and duplicates accumulated (78 observed on one box), every
+ * routine double-firing. Best-effort — the reaper self-heals a missing/stale
+ * marker, and reading another process's ENV to key on the daemon dir directly is
+ * not portable (hardened macOS hides it from `ps`), so identity rides the shared
+ * on-disk registry instead. No-op on Windows (POSIX-only reaper).
+ */
+export declare function registerDaemonInstance(pid?: number): void;
+/** Remove this daemon's registry marker on graceful shutdown. */
+export declare function unregisterDaemonInstance(pid?: number): void;
+/**
+ * Reap stray duplicate daemons of THIS device — every registrant in the instance
+ * registry that is a live `agents __daemon-run` and is neither this process nor
+ * the current pid-file owner. A predecessor SIGKILLed/OOM-ed without cleanup, or a
+ * duplicate that lost the pid-file write race, would otherwise keep a second
+ * scheduler alive and double-fire jobs even after claimDaemonInstance() hands the
+ * pid file to the survivor. Also garbage-collects markers whose pid is dead or was
+ * reused by an unrelated process. No-op on Windows (POSIX-only).
  */
 export declare function reapStrayDaemons(keepPid?: number): {
     reaped: number;
@@ -236,8 +256,38 @@ export declare function startDetached(opts?: StartDetachedOptions): {
     pid: number | null;
     method: string;
 };
-/** Stop the daemon, unloading it from launchd/systemd if applicable. */
-export declare function stopDaemon(): boolean;
+/**
+ * Structured outcome of {@link stopDaemon} (SING-12, RUSH-2355). `stopDaemon`
+ * asserts its postcondition instead of assuming it: `ok` is true only when every
+ * resource the daemon held is provably released. `surviving` names anything that
+ * did not release (a still-live daemon, or a stale socket that could not be
+ * cleared) and is what drives a non-zero exit; `detachedChildren` are the
+ * in-flight routine children that survive deliberately (SING-11a) and are
+ * reported, never killed.
+ */
+export interface DaemonStopResult {
+    ok: boolean;
+    stoppedPid: number | null;
+    escalated: boolean;
+    released: string[];
+    surviving: string[];
+    detachedChildren: number[];
+}
+/**
+ * Stop the daemon and ASSERT its postcondition (SING-12, RUSH-2355), unloading it
+ * from launchd/systemd if applicable.
+ *
+ * The SIGTERM → grace → killTree sequence is unchanged; what it adds is
+ * verification. After the daemon is gone it checks that the secrets broker socket
+ * and browser IPC binding actually released — a stale socket present on disk but
+ * with no live owner is the orphan that keeps clients holding unlocked bundles
+ * hanging (`daemon.ts` broker-hosting; the two-brokers-on-one-socket bug) — and
+ * that no `__daemon-run` for THIS state dir survives. A killTree escalation exits
+ * without running the daemon's graceful handleShutdown, so those sockets can be
+ * left stale; this reclaims each (the owner is provably dead) and reports it. It
+ * never reports success on an unverified stop.
+ */
+export declare function stopDaemon(): DaemonStopResult;
 /** Get current daemon status including running state, PID, and enabled job count. */
 export declare function getDaemonStatus(): {
     state: 'running' | 'wedged' | 'stopped';