@phnx-labs/agents-cli 1.20.18 → 1.20.20

package/dist/lib/secrets/agent.js

@@ -24,6 +24,7 @@
  */
 import * as net from 'net';
 import * as fs from 'fs';
+import * as os from 'os';
 import * as path from 'path';
 import { spawn, spawnSync, execFileSync } from 'child_process';
 import { getHelpersDir, readMeta } from '../state.js';
@@ -61,24 +62,134 @@ function pidPath() {
     return path.join(agentDir(), 'agent.pid');
 }
 /**
- * Argv for re-invoking THIS cli to run the broker, so a side-by-side dev build
- * spawns its own broker rather than the registry-installed one. We always go
- * through `process.execPath` (the node binary) with the JS entrypoint as the
- * first arg — the entrypoint isn't reliably executable in dev builds (invoked
- * as `node dist/index.js`, no +x), so spawning it directly EACCES'd.
+ * Argv for re-invoking THIS cli with a hidden subcommand, so a side-by-side dev
+ * build spawns its own helpers rather than the registry-installed one. We always
+ * go through `process.execPath` (the node binary) with the JS entrypoint as the
+ * first arg — the entrypoint isn't reliably executable in dev builds (invoked as
+ * `node dist/index.js`, no +x), so spawning it directly EACCES'd.
  */
-function brokerSpawn() {
+function cliSpawn(sub) {
     const argv1 = process.argv[1];
     const entry = argv1 && fs.existsSync(argv1) ? argv1 : null;
     if (entry)
-        return { cmd: process.execPath, args: [entry, 'secrets', '_agent-run'] };
+        return { cmd: process.execPath, args: [entry, ...sub] };
     // No resolvable entrypoint (unusual) — fall back to the PATH shim.
     let bin = 'agents';
     try {
         bin = execFileSync('which', ['agents'], { encoding: 'utf-8' }).trim();
     }
     catch { /* default */ }
-    return { cmd: bin, args: ['secrets', '_agent-run'] };
+    return { cmd: bin, args: sub };
+}
+function brokerSpawn() {
+    return cliSpawn(['secrets', '_agent-run']);
+}
+// ─── Persistent launchd service ──────────────────────────────────────────────
+// On a heavily-loaded machine a freshly-spawned broker (a full CLI cold start)
+// can't get scheduled enough CPU to finish booting and bind its socket — so the
+// on-demand model fails exactly when there are many agents (the case we care
+// about). The fix is to run the broker as a launchd user service: started once
+// with RunAtLoad + KeepAlive, it stays up, and every read just connects. The
+// cold start happens once (and launchd retries until it wins), never per-read.
+const SERVICE_LABEL = 'com.phnx-labs.agents-secrets-agent';
+function servicePlistPath() {
+    return path.join(os.homedir(), 'Library', 'LaunchAgents', `${SERVICE_LABEL}.plist`);
+}
+/** True if the launchd plist for the persistent broker is installed. */
+export function secretsAgentServiceInstalled() {
+    return onDarwin() && fs.existsSync(servicePlistPath());
+}
+function generateServicePlist() {
+    const { cmd, args } = cliSpawn(['secrets', '_agent-run', '--service']);
+    const progArgs = [cmd, ...args]
+        .map((a) => `    <string>${a.replace(/&/g, '&amp;').replace(/</g, '&lt;')}</string>`)
+        .join('\n');
+    const logPath = path.join(agentDir(), 'service.log');
+    const home = os.homedir();
+    return `<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>Label</key>
+  <string>${SERVICE_LABEL}</string>
+  <key>ProgramArguments</key>
+  <array>
+${progArgs}
+  </array>
+  <key>RunAtLoad</key>
+  <true/>
+  <key>KeepAlive</key>
+  <true/>
+  <key>ProcessType</key>
+  <string>Interactive</string>
+  <key>StandardOutPath</key>
+  <string>${logPath}</string>
+  <key>StandardErrorPath</key>
+  <string>${logPath}</string>
+  <key>EnvironmentVariables</key>
+  <dict>
+    <key>PATH</key>
+    <string>/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin:${home}/.bun/bin</string>
+  </dict>
+</dict>
+</plist>`;
+}
+/**
+ * Install + start the persistent broker as a launchd user service (idempotent).
+ * Writes the plist, bootstraps it into the GUI domain, and waits for the socket.
+ * `ProcessType: Interactive` asks launchd to schedule it at foreground priority
+ * so it can boot even when the machine is loaded. Returns true once reachable.
+ */
+export async function installSecretsAgentService(timeoutMs = 30000) {
+    if (!onDarwin())
+        return false;
+    const plist = servicePlistPath();
+    fs.mkdirSync(path.dirname(plist), { recursive: true });
+    fs.writeFileSync(plist, generateServicePlist());
+    const uid = process.getuid?.() ?? 0;
+    // bootstrap is the modern API; fall back to legacy load. Both idempotent-ish.
+    try {
+        execFileSync('launchctl', ['bootstrap', `gui/${uid}`, plist], { stdio: ['ignore', 'ignore', 'ignore'] });
+    }
+    catch {
+        try {
+            execFileSync('launchctl', ['load', '-w', plist], { stdio: ['ignore', 'ignore', 'ignore'] });
+        }
+        catch { /* may already be loaded */ }
+    }
+    // kickstart to force an immediate start even if already bootstrapped.
+    try {
+        execFileSync('launchctl', ['kickstart', '-k', `gui/${uid}/${SERVICE_LABEL}`], { stdio: ['ignore', 'ignore', 'ignore'] });
+    }
+    catch { /* best effort */ }
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+        if (await agentPing())
+            return true;
+        await new Promise((r) => setTimeout(r, 200));
+    }
+    return false;
+}
+/** Stop + remove the persistent broker service, and wipe whatever it held. */
+export async function uninstallSecretsAgentService() {
+    if (!onDarwin())
+        return;
+    await agentLock(); // wipe the in-memory store before tearing down
+    const plist = servicePlistPath();
+    const uid = process.getuid?.() ?? 0;
+    try {
+        execFileSync('launchctl', ['bootout', `gui/${uid}/${SERVICE_LABEL}`], { stdio: ['ignore', 'ignore', 'ignore'] });
+    }
+    catch {
+        try {
+            execFileSync('launchctl', ['unload', '-w', plist], { stdio: ['ignore', 'ignore', 'ignore'] });
+        }
+        catch { /* not loaded */ }
+    }
+    try {
+        fs.unlinkSync(plist);
+    }
+    catch { /* already gone */ }
 }
 // ─── Broker server (runs in the detached `secrets _agent-run` process) ───────
 /**
@@ -127,9 +238,13 @@ export function handleAgentRequest(store, req, now = Date.now()) {
  * `agents secrets _agent-run`. Holds the store in memory, serves the socket,
  * sweeps expired entries, wipes on screen-lock/sleep, and self-exits when idle.
  */
-export async function runSecretsAgent() {
+export async function runSecretsAgent(opts = {}) {
     if (!onDarwin())
         return; // nothing to broker without biometry prompts
+    // When launchd keeps us alive as a persistent service, never idle-exit:
+    // exiting would just make launchd cold-start us again, reintroducing the
+    // startup-under-load fragility the service exists to avoid.
+    const persistent = opts.service === true;
     // Single-instance guard: O_EXCL pid file. If a live broker already holds it,
     // exit quietly — the existing one keeps serving.
     const pidFile = pidPath();
@@ -169,7 +284,7 @@ export async function runSecretsAgent() {
             if (now >= e.expiresAt)
                 store.delete(name);
         if (store.size === 0) {
-            if (now - emptySince >= IDLE_EXIT_MS)
+            if (!persistent && now - emptySince >= IDLE_EXIT_MS)
                 shutdown(0);
         }
         else {
@@ -371,64 +486,60 @@ export function secretsAgentAutoEnabled() {
         return false;
     }
 }
-/**
- * Inline node program that loads one bundle into the broker, started detached
- * from the hot path. Reads the JSON payload from stdin (so secret values never
- * appear in argv / `ps`), retries the socket for a few seconds to absorb a
- * cold-started agent, sends the load, and exits. argv after -e: [execPath, <socket>].
- */
-const DETACHED_LOAD_PROGRAM = `
-const net = require('net');
-const sock = process.argv[1];
-let input = '';
-process.stdin.setEncoding('utf-8');
-process.stdin.on('data', (d) => { input += d; });
-process.stdin.on('end', () => {
-  let payload; try { payload = JSON.parse(input); } catch (e) { process.exit(1); }
-  let attempts = 0;
-  const tryConnect = () => {
-    const c = net.createConnection(sock);
-    c.on('connect', () => {
-      c.write(JSON.stringify({ cmd: 'load', name: payload.name, bundle: payload.bundle, env: payload.env, ttlMs: payload.ttlMs }) + '\\n');
-    });
-    c.setEncoding('utf-8');
-    c.on('data', () => { try { c.destroy(); } catch (e) {} process.exit(0); });
-    c.on('error', () => {
-      try { c.destroy(); } catch (e) {}
-      if (++attempts >= 30) process.exit(1);
-      setTimeout(tryConnect, 100);
-    });
-  };
-  tryConnect();
-});
-`;
 /**
  * Fire-and-forget: populate the broker with a freshly-resolved bundle so the
  * NEXT process reads it without a prompt. Used by the auto-cache path after a
  * real keychain read of a `session`-tier bundle. Adds no latency to the caller
- * — it spawns the agent (if needed) and a detached loader, both unref'd, then
- * returns immediately. Entirely best-effort; never throws. macOS only.
+ * — it spawns a detached `secrets _agent-load` worker (passing the resolved env
+ * over stdin, never argv) and returns immediately.
+ *
+ * The worker reuses the robust `ensureAgentRunning` path (spawn-then-ping with a
+ * generous budget) rather than a tight inline retry loop: under heavy load the
+ * broker is itself a cold-starting full CLI and can take several seconds to bind
+ * the socket, so a short fixed budget would give up before it's ready and the
+ * cache would silently never populate. Best-effort; never throws. macOS only.
  */
 export function agentAutoLoadSync(name, bundle, env, ttlMs) {
     if (!onDarwin())
         return;
     try {
-        if (!agentSocketExists()) {
-            const { cmd, args } = brokerSpawn();
-            spawn(cmd, args, { stdio: 'ignore', detached: true }).unref();
-        }
-        const loader = spawn(process.execPath, ['-e', DETACHED_LOAD_PROGRAM, socketPath()], {
-            stdio: ['pipe', 'ignore', 'ignore'],
-            detached: true,
-        });
-        loader.stdin?.write(JSON.stringify({ name, bundle, env, ttlMs }));
-        loader.stdin?.end();
-        loader.unref();
+        const { cmd, args } = cliSpawn(['secrets', '_agent-load']);
+        const worker = spawn(cmd, args, { stdio: ['pipe', 'ignore', 'ignore'], detached: true });
+        worker.stdin?.write(JSON.stringify({ name, bundle, env, ttlMs }));
+        worker.stdin?.end();
+        worker.unref();
     }
     catch {
         // best-effort: the next read just pops Touch ID as it would today
     }
 }
+/**
+ * Body of the hidden `secrets _agent-load` worker. Reads one `{name, bundle,
+ * env, ttlMs}` payload from stdin, ensures the broker is up (robust, generous
+ * budget), and loads the bundle into it. Detached from the originating read, so
+ * its latency is invisible — which is why it can afford a long ensure budget.
+ */
+export async function runAgentLoadFromStdin() {
+    if (!onDarwin())
+        return;
+    const chunks = [];
+    for await (const chunk of process.stdin)
+        chunks.push(chunk);
+    let payload;
+    try {
+        payload = JSON.parse(Buffer.concat(chunks).toString('utf-8'));
+    }
+    catch {
+        return; // malformed payload — nothing to load
+    }
+    if (!payload || !payload.name || !payload.bundle || !payload.env)
+        return;
+    // Generous budget: the broker is a cold-starting full CLI; under load it can
+    // take several seconds to bind. We're detached, so waiting costs nothing.
+    if (!(await ensureAgentRunning(20000)))
+        return;
+    await agentLoad(payload.name, payload.bundle, payload.env, payload.ttlMs ?? DEFAULT_TTL_MS);
+}
 /** Store a resolved bundle in the broker. Returns false on transport failure. */
 export async function agentLoad(name, bundle, env, ttlMs) {
     const r = await request({ cmd: 'load', name, bundle, env, ttlMs });
@@ -453,16 +564,43 @@ async function agentPing() {
     return r?.ok === true && r.cmd === 'ping' && r.version === PROTOCOL_VERSION;
 }
 /**
- * Ensure a broker is running and reachable, spawning one detached if not.
- * Returns true once the socket answers a ping. On protocol-version skew, kills
- * the stale broker and respawns. macOS only.
+ * Ensure a broker is running and reachable. Returns true once the socket answers
+ * a ping. macOS only.
+ *
+ * Prefers the persistent launchd service: if it isn't installed we install it
+ * (which makes the broker survive for the whole login session, so subsequent
+ * reads never cold-start); if it's installed but unreachable we kickstart it.
+ * Only when the service path can't be used do we fall back to a one-off detached
+ * broker — that's the model that gets starved under heavy load, so it's last.
  */
 export async function ensureAgentRunning(timeoutMs = 5000) {
     if (!onDarwin())
         return false;
     if (await agentPing())
         return true;
-    // Socket exists but ping failed → stale/old broker. Kill it before respawn.
+    // Path 1: the persistent service. installSecretsAgentService is idempotent and
+    // waits for the socket; for an already-installed service we kickstart and wait.
+    try {
+        if (!secretsAgentServiceInstalled()) {
+            if (await installSecretsAgentService(Math.max(timeoutMs, 20000)))
+                return true;
+        }
+        else {
+            const uid = process.getuid?.() ?? 0;
+            try {
+                execFileSync('launchctl', ['kickstart', '-k', `gui/${uid}/${SERVICE_LABEL}`], { stdio: ['ignore', 'ignore', 'ignore'] });
+            }
+            catch { /* may already be running */ }
+            const d = Date.now() + timeoutMs;
+            while (Date.now() < d) {
+                if (await agentPing())
+                    return true;
+                await new Promise((r) => setTimeout(r, 150));
+            }
+        }
+    }
+    catch { /* fall through to the one-off spawn */ }
+    // Path 2 (fallback): one-off detached broker. Clear a stale socket/pid first.
     const stalePid = (() => {
         try {
             return parseInt(fs.readFileSync(pidPath(), 'utf-8').trim(), 10);
@@ -486,11 +624,7 @@ export async function ensureAgentRunning(timeoutMs = 5000) {
     }
     catch { /* gone */ }
     const { cmd, args } = brokerSpawn();
-    const child = spawn(cmd, args, {
-        stdio: 'ignore',
-        detached: true,
-    });
-    child.unref();
+    spawn(cmd, args, { stdio: 'ignore', detached: true }).unref();
     const deadline = Date.now() + timeoutMs;
     while (Date.now() < deadline) {
         if (await agentPing())

package/dist/lib/secrets/bundles.d.ts

@@ -1,17 +1,32 @@
 /**
- * Secret bundles — named sets of keychain-backed environment variables.
+ * Secret bundles — named sets of environment variables backed by a secret store.
  *
- * Bundle metadata (name, description, vars map) is stored in the macOS
- * Keychain as a JSON blob under `agents-cli.bundles.<name>`. Secret values
- * live one per keychain item under `agents-cli.secrets.<bundle>.<key>`.
- * Every item is device-local and gated by Touch ID / device passcode — see
- * src/lib/secrets/index.ts for the access-control story. Nothing about
- * secrets ever lives in plaintext on disk.
+ * Bundle metadata (name, description, vars map) is stored as a JSON blob under
+ * `agents-cli.bundles.<name>`; secret values live one per item under
+ * `agents-cli.secrets.<bundle>.<key>`. Two backends carry those items:
+ *
+ *  - `keychain` (default): the macOS Keychain (device-local, Touch ID / device
+ *    passcode gated) or Linux libsecret — see src/lib/secrets/index.ts.
+ *  - `file`: an AES-256-GCM encrypted-file store keyed by a passphrase
+ *    (src/lib/secrets/filestore.ts). Opt-in, for headless / remote runs where
+ *    no biometry prompt can be satisfied (e.g. a release on a remote Mac over
+ *    SSH). The item-name scheme is identical, so the only difference is where
+ *    bytes land. A file-backed bundle is discovered by the presence of its
+ *    metadata item in the file store.
  *
  * Cross-machine sync is handled by src/lib/secrets/sync.ts via an explicit
  * encrypted export/import flow; the bundle layer is sync-agnostic.
  */
 import { type BundleValue, type SecretRef } from './index.js';
+/** Which store carries a bundle's items. */
+export type SecretsBackend = 'keychain' | 'file';
+/**
+ * Discover a bundle's backend by location: a file-backed bundle's metadata
+ * item exists in the encrypted-file store. This is a plain file-existence
+ * check — no passphrase, no Touch ID — so it sidesteps the chicken-and-egg of
+ * "read metadata to learn where metadata lives." Absent ⇒ keychain.
+ */
+export declare function bundleBackend(name: string): SecretsBackend;
 /** Allowed values for a secret's `type` metadata field. */
 export declare const SECRET_TYPES: readonly ["api-key", "token", "password", "url", "database-url", "ssh-key", "certificate", "webhook", "note"];
 export type SecretType = typeof SECRET_TYPES[number];
@@ -38,6 +53,8 @@ export interface SecretsBundle {
     name: string;
     description?: string;
     allow_exec?: boolean;
+    /** Which store carries this bundle's items. Absent ⇒ `keychain` (the default). */
+    backend?: SecretsBackend;
     /** Secrets-agent interaction tier. Absent ⇒ `biometry` (the safe default). */
     tier?: SecretsTier;
     /** ISO 8601 UTC timestamp. Set once on the first writeBundle() for a bundle. */
@@ -156,6 +173,19 @@ export interface RenameOptions {
  * a safe no-op for the source items.
  */
 export declare function renameBundle(oldName: string, newName: string, opts?: RenameOptions): void;
+/**
+ * The store (keychain or encrypted file) that carries a bundle's items. The
+ * CLI uses this to read/write/delete per-key items (built with
+ * secretsKeychainItem) in the same store as the bundle's metadata, for `add` /
+ * `import` / `remove` / `delete`. Pass the bundle's resolved backend
+ * (`bundle.backend ?? 'keychain'`).
+ */
+export declare function bundleItemStore(backend: SecretsBackend | undefined): {
+    set(item: string, value: string): void;
+    delete(item: string): boolean;
+    get(item: string): string;
+    has(item: string): boolean;
+};
 export declare function keychainItemsForBundle(bundle: SecretsBundle): Array<{
     key: string;
     item: string;