@phnx-labs/agents-cli 1.22.25 → 1.22.26

package/dist/lib/teams/placement-probe.js

@@ -0,0 +1,135 @@
+/**
+ * Live pool probing for the teams placement scheduler (RUSH-2002).
+ *
+ * Gathers one {@link DevicePlacementSignal} per pool device so the pure pick in
+ * {@link ./scheduler} can filter unreachable / overloaded / not-installed
+ * devices and rank the rest. Two concerns, both best-effort:
+ *
+ *   - reachability + headroom + load  ← {@link probeFleetStats} (one parallel
+ *     SSH fan-out over the pool; the local box is measured directly).
+ *   - requested agent installed (+ signed in, locally)  ← a one-shot readiness
+ *     probe per remote device ({@link buildReadyProbeCommand} → `agents view`),
+ *     the local box via {@link checkCliAvailable}/{@link checkCliSignedIn}.
+ *
+ * The result is cached briefly per (pool, agent) so a `teams start` wave that
+ * places N teammates probes the pool ONCE, not N times — the roster-count part
+ * of the rank stays live (the pure pick recounts the roster each call), only the
+ * SSH-measured load/harness snapshot is reused within the TTL.
+ *
+ * All SSH here is via `execFile` (async, bounded) so the whole pool is probed in
+ * parallel; a slow or wedged box degrades to "no signal" instead of blocking the
+ * launch. Remote sign-in is deliberately left unknown — sign-in detection is
+ * unreliable over SSH (see {@link checkCliSignedIn}); it ranks the local box
+ * only, and the install gate (not sign-in) is what drives the fail-loud.
+ */
+import { execFile } from 'child_process';
+import { probeFleetStats, headroom } from '../devices/health.js';
+import { loadDevicesSync } from '../devices/registry.js';
+import { buildSshInvocation, writeAskpassShim } from '../devices/connect.js';
+import { buildReadyProbeCommand, parseReadyProbe, viewHasAgent } from '../hosts/ready.js';
+import { localMachineId } from '../session/origin-machine.js';
+import { normalizeHost } from '../machine-id.js';
+import { checkCliAvailable, checkCliSignedIn } from './agents.js';
+/** Per-remote readiness probe budget — matches the health probe's short window
+ * (`agents view` on a warm box is sub-second; a wedged one degrades to unknown). */
+export const READY_PROBE_TIMEOUT_MS = 8_000;
+/** How long a probed pool snapshot is reused within a `teams start` wave. Short
+ * enough that a device coming online / going overloaded is seen next wave, long
+ * enough that placing a wave of teammates does not re-fan-out per teammate. */
+export const SIGNAL_TTL_MS = 15_000;
+const cache = new Map();
+function cacheKey(pool, agent) {
+    return `${agent}::${[...pool].map(normalizeHost).sort().join(',')}`;
+}
+/** Clear the probe cache — for tests and after a device-registry change. */
+export function clearPlacementSignalCache() {
+    cache.clear();
+}
+/**
+ * Whether the requested agent is installed on a REMOTE device, via one SSH
+ * readiness probe. `undefined` when the probe could not answer (ssh/login
+ * failure) — reachability is then left to {@link probeFleetStats}; `false` when
+ * the box answered but agents-cli or the agent is absent (a genuine can't-run).
+ */
+function probeRemoteInstalled(device, agent) {
+    let args;
+    let env;
+    try {
+        const shim = writeAskpassShim();
+        const cmd = buildReadyProbeCommand(device.shell === 'powershell' ? 'windows' : undefined);
+        // agentOnly: a read-only probe must never force a foreground Touch ID sheet
+        // on a password-auth device (mirrors probeDeviceStats in devices/health).
+        ({ args, env } = buildSshInvocation(device, [cmd], shim, {}, { agentOnly: true }));
+    }
+    catch {
+        return Promise.resolve(undefined);
+    }
+    return new Promise((resolve) => {
+        execFile('ssh', args, { encoding: 'utf-8', env: { ...process.env, ...env }, timeout: READY_PROBE_TIMEOUT_MS }, (err, stdout) => {
+            if (err || !stdout)
+                return resolve(undefined);
+            const probe = parseReadyProbe(stdout);
+            if (!probe.reachable)
+                return resolve(undefined); // ssh/login issue → unknown
+            if (!probe.version)
+                return resolve(false); // agents-cli missing → cannot run
+            resolve(viewHasAgent(probe.view, agent));
+        });
+    });
+}
+/**
+ * Probe every device in the team pool and return a name→signal map for the pure
+ * placement pick. Devices with no data at all are omitted (the pick then neither
+ * excludes nor prefers them). Never throws — a probe failure degrades to a
+ * missing/partial signal.
+ */
+export async function probePoolSignals(pool, agent, opts = {}) {
+    const now = opts.now ?? Date.now();
+    const key = cacheKey(pool, agent);
+    const cached = cache.get(key);
+    if (!opts.force && cached && now - cached.at < SIGNAL_TTL_MS)
+        return cached.signals;
+    const reg = loadDevicesSync();
+    const self = normalizeHost(localMachineId());
+    const lookup = (name) => reg[name] ?? reg[normalizeHost(name)];
+    const profiles = [];
+    const seen = new Set();
+    for (const name of pool) {
+        const d = lookup(name);
+        if (d && !seen.has(d.name)) {
+            profiles.push(d);
+            seen.add(d.name);
+        }
+    }
+    const selfProfile = profiles.find((d) => normalizeHost(d.name) === self);
+    const stats = await probeFleetStats(profiles, { selfName: selfProfile?.name });
+    const installed = new Map(await Promise.all(profiles.map(async (d) => {
+        const isSelf = normalizeHost(d.name) === self;
+        if (isSelf) {
+            const inst = checkCliAvailable(agent)[0];
+            const signedIn = inst ? await checkCliSignedIn(agent) : undefined;
+            return [d.name, { installed: inst, signedIn }];
+        }
+        // Remote sign-in stays unknown (see the module docblock); the install
+        // gate is what drives the fail-loud.
+        return [d.name, { installed: await probeRemoteInstalled(d, agent), signedIn: undefined }];
+    })));
+    const signals = new Map();
+    for (const name of pool) {
+        const d = lookup(name);
+        const s = d ? stats.get(d.name) : undefined;
+        const inst = d ? installed.get(d.name) : undefined;
+        if (!s && !inst)
+            continue; // fully unknown device — leave it out of the map
+        signals.set(name, {
+            reachable: s?.reachable,
+            headroom: s ? headroom(s) : undefined,
+            loadPercent: s?.loadPercent,
+            memPercent: s?.memPercent,
+            installed: inst?.installed,
+            signedIn: inst?.signedIn,
+        });
+    }
+    cache.set(key, { at: now, signals });
+    return signals;
+}

package/dist/lib/teams/scheduler.d.ts

@@ -1,3 +1,26 @@
+import type { Headroom } from '../devices/health.js';
+/**
+ * Live signal for one pool device, gathered by the caller (SSH probes) and
+ * passed into the pure pick. Every field is optional/best-effort: a device with
+ * no signal is neither excluded nor preferred — it degrades to the roster-count
+ * path. `reachable === false` and `installed === false` are the only two
+ * exclusion signals here; `headroom` drives the overload exclusion + the load
+ * tier of the rank. See {@link DeviceStats}/{@link headroom} in devices/health.
+ */
+export interface DevicePlacementSignal {
+    /** SSH probe answered. `false` → excluded (unreachable). undefined → unknown. */
+    reachable?: boolean;
+    /** Headroom bucket from load+memory. `'loaded'` → excluded (overloaded). */
+    headroom?: Headroom;
+    /** Normalized CPU load percent (finer rank tiebreak within a headroom tier). */
+    loadPercent?: number;
+    /** Memory pressure percent (used with loadPercent for the load cost). */
+    memPercent?: number;
+    /** Requested agent installed on the device. `false` → excluded (can't run). */
+    installed?: boolean;
+    /** Requested agent signed in (best-effort). Ranks a signed-in device first. */
+    signedIn?: boolean;
+}
 /** Team fields the placement cascade reads (a subset of TeamMeta). */
 export interface PlacementTeam {
     devices?: string[];
@@ -14,7 +37,37 @@ export interface PlacementTeam {
  */
 export interface PlacementOptions {
     maxConcurrent?: Record<string, number>;
+    /**
+     * Live per-device signals (RUSH-2002). When present, the many-device pick
+     * filters unreachable/overloaded/not-installed devices and ranks the rest by
+     * health + harness + load. Absent → the pick stays a pure roster count.
+     */
+    signals?: Map<string, DevicePlacementSignal>;
+    /** Human label of the requested agent (e.g. `claude@2.1.112`) for the
+     * fail-loud message. */
+    agentLabel?: string;
+}
+/** Why a device was excluded from the viable set, for the fail-loud message. */
+export type ExclusionReason = 'unreachable' | 'overloaded' | 'capped' | 'not-installed';
+/** A pool device dropped from the auto-pick, with the reason + live detail. */
+export interface ExcludedDevice {
+    device: string;
+    reason: ExclusionReason;
+    /** Present for `capped`: `running/cap`. */
+    detail?: string;
 }
+/**
+ * No device in the pool can host the teammate. Carries the per-device reasons so
+ * the caller can print exactly why the pool is unusable instead of silently
+ * degrading to a local run the user did not ask for.
+ */
+export declare class NoViableDeviceError extends Error {
+    readonly excluded: ExcludedDevice[];
+    readonly agentLabel?: string;
+    constructor(excluded: ExcludedDevice[], agentLabel?: string);
+}
+/** Build the user-facing fail-loud message from the per-device reasons. */
+export declare function formatNoViableMessage(excluded: ExcludedDevice[], agentLabel?: string): string;
 /**
  * A roster entry the load counter reads — the shape any teammate satisfies
  * (AgentProcess included). `status` is compared against `'running'` (the
@@ -45,11 +98,31 @@ export declare function cappedDevices(devices: string[], roster: RosterEntry[],
  * teammate silently landing on a machine its operator capped.
  */
 export declare function pickLeastLoaded(devices: string[], roster: RosterEntry[], maxConcurrent?: Record<string, number>): string;
+/**
+ * Split the pool into the viable set and the excluded set (with reasons), from
+ * the caps + live signals. Pure — the caller decides what to do with an empty
+ * viable set. Reasons are checked in a stable order (unreachable → not-installed
+ * → overloaded → capped) so a device reports its most fundamental blocker first.
+ */
+export declare function classifyExclusions(devices: string[], roster: RosterEntry[], opts?: PlacementOptions): {
+    eligible: string[];
+    excluded: ExcludedDevice[];
+};
+/**
+ * Health-, harness-, and load-aware pick over a pool (RUSH-2002). Filters
+ * unreachable / overloaded / capped / not-installed devices, then ranks the
+ * survivors by: (a) agent signed in, (b) lower load (headroom tier, then raw
+ * load cost), (c) fewer running teammates, ties broken by pool order. Throws
+ * {@link NoViableDeviceError} when nothing survives — never returns a local
+ * fallback. Pure: all I/O is pre-resolved into `opts.signals`.
+ */
+export declare function pickBestDevice(devices: string[], roster: RosterEntry[], opts?: PlacementOptions): string;
 /**
  * Resolve where a teammate runs. Returns `{ device: null }` for a local run
  * (no pin, no pool, or the chosen device is the local machine) and
  * `{ device: <name> }` for a remote placement. See the cascade in the module
- * header.
+ * header. Throws {@link NoViableDeviceError} when a required pool has no viable
+ * host — the caller surfaces it at `teams start` rather than falling back local.
  */
 export declare function resolvePlacement(team: PlacementTeam, explicitDevice: string | null, roster: RosterEntry[], opts?: PlacementOptions): {
     device: string | null;

package/dist/lib/teams/scheduler.js

@@ -8,19 +8,68 @@
  *
  *   1. teammate has an explicit `--device` pin      → that device
  *   2. else the team pool has exactly one device     → that device (whole team)
- *   3. else the team pool has many devices           → least-loaded pick
+ *   3. else the team pool has many devices           → best-viable pick
  *   4. else (no pin, no pool)                         → null == run local
  *
- * Step 3 is cap-aware: a device at its `agents.max-concurrent` cap (from the
- * device doc, passed in via PlacementOptions) is excluded from the auto-pick,
- * and an all-capped pool fails loud. Pins and pools of one are the user's own
- * choice and are never second-guessed.
+ * Step 3 is cap-, health-, and harness-aware (RUSH-2002). When the caller
+ * supplies live {@link DevicePlacementSignal}s (probed reachability, headroom,
+ * load/mem, and whether the requested agent is installed + signed in), the pick:
+ *   - EXCLUDES an unreachable device, an overloaded one (headroom `loaded`), a
+ *     device at its `agents.max-concurrent` cap, and one the requested agent is
+ *     not installed on;
+ *   - RANKS the survivors by (a) agent installed + signed in, (b) lower load /
+ *     memory (by headroom tier, then raw), (c) fewer running teammates, ties
+ *     broken by pool order.
+ * With no signals it degrades to the original cap-aware least-loaded pick, so
+ * the pure roster-count path (and its tests) are unchanged.
+ *
+ * When every pool device is excluded, the pick FAILS LOUD ({@link
+ * NoViableDeviceError}) naming each device's reason — never a silent fall back
+ * to local, since the user asked for a remote pool. Pins are the user's own
+ * choice and are never second-guessed; a pool of one is respected for
+ * load/cap/reachability but still fails loud when the agent is provably not
+ * installed there (running it would be futile).
  *
  * A device whose name equals the local machine id is treated as "local" — it
  * resolves to a null placement so the existing local spawn path runs unchanged,
  * letting the local machine participate in a pool as just another member.
  */
 import { machineId } from '../session/sync/config.js';
+/**
+ * No device in the pool can host the teammate. Carries the per-device reasons so
+ * the caller can print exactly why the pool is unusable instead of silently
+ * degrading to a local run the user did not ask for.
+ */
+export class NoViableDeviceError extends Error {
+    excluded;
+    agentLabel;
+    constructor(excluded, agentLabel) {
+        super(formatNoViableMessage(excluded, agentLabel));
+        this.name = 'NoViableDeviceError';
+        this.excluded = excluded;
+        this.agentLabel = agentLabel;
+    }
+}
+/** Build the user-facing fail-loud message from the per-device reasons. */
+export function formatNoViableMessage(excluded, agentLabel) {
+    const anyNotInstalled = excluded.some((e) => e.reason === 'not-installed');
+    const agent = agentLabel ?? 'the requested agent';
+    const perDevice = excluded
+        .map((e) => (e.detail ? `${e.device} (${e.reason} ${e.detail})` : `${e.device} (${e.reason})`))
+        .join(', ');
+    // When the whole pool lacks the agent, lead with the harness message the
+    // ticket specifies; otherwise summarize the mixed reasons.
+    const head = anyNotInstalled
+        ? `No device in the team pool can run ${agent}.`
+        : `No viable device in the team pool for ${agent}.`;
+    const anyUnreachable = excluded.some((e) => e.reason === 'unreachable');
+    const hint = anyNotInstalled
+        ? " Run 'agents devices ping' to see which devices have the agent installed + signed in, or add the agent to a pool device."
+        : anyUnreachable
+            ? " Add a device to the pool, raise a cap, or wait for the pool to free up / come back online ('agents devices ping')."
+            : ' Add a device to the pool, raise a cap, or bring an overloaded box under load.';
+    return `${head} ${perDevice}.${hint}`;
+}
 /** True when `device` names the local machine (case-insensitive). */
 function isLocalDevice(device) {
     return device.toLowerCase() === machineId();
@@ -97,14 +146,134 @@ export function pickLeastLoaded(devices, roster, maxConcurrent) {
     }
     return best;
 }
+/** Load cost for a signal: worst of load% and mem%, or undefined when neither
+ * was probed. The "how full is this box" number the finer rank tiebreak reads. */
+function loadCost(s) {
+    if (!s)
+        return undefined;
+    const vals = [s.loadPercent, s.memPercent].filter((v) => typeof v === 'number');
+    return vals.length ? Math.max(...vals) : undefined;
+}
+/** Rank a headroom bucket into a load TIER (lower is less loaded). An unprobed
+ * device sits between `light` and `busy` — preferred over a known-busy box but
+ * not over a known-idle one. `loaded` is excluded before ranking, so it never
+ * reaches here. */
+function headroomTier(h) {
+    switch (h) {
+        case 'idle': return 0;
+        case 'light': return 1;
+        case 'busy': return 3;
+        default: return 2; // 'unknown' / undefined
+    }
+}
+/**
+ * Split the pool into the viable set and the excluded set (with reasons), from
+ * the caps + live signals. Pure — the caller decides what to do with an empty
+ * viable set. Reasons are checked in a stable order (unreachable → not-installed
+ * → overloaded → capped) so a device reports its most fundamental blocker first.
+ */
+export function classifyExclusions(devices, roster, opts) {
+    const load = loadByDevice(devices, roster);
+    const caps = opts?.maxConcurrent ?? {};
+    const signals = opts?.signals;
+    const eligible = [];
+    const excluded = [];
+    for (const d of devices) {
+        const s = signals?.get(d);
+        if (s?.reachable === false) {
+            excluded.push({ device: d, reason: 'unreachable' });
+            continue;
+        }
+        if (s?.installed === false) {
+            excluded.push({ device: d, reason: 'not-installed' });
+            continue;
+        }
+        if (s?.headroom === 'loaded') {
+            excluded.push({ device: d, reason: 'overloaded' });
+            continue;
+        }
+        const cap = caps[d];
+        if (cap !== undefined && (load.get(d) ?? 0) >= cap) {
+            excluded.push({ device: d, reason: 'capped', detail: `${load.get(d) ?? 0}/${cap}` });
+            continue;
+        }
+        eligible.push(d);
+    }
+    return { eligible, excluded };
+}
+/**
+ * Health-, harness-, and load-aware pick over a pool (RUSH-2002). Filters
+ * unreachable / overloaded / capped / not-installed devices, then ranks the
+ * survivors by: (a) agent signed in, (b) lower load (headroom tier, then raw
+ * load cost), (c) fewer running teammates, ties broken by pool order. Throws
+ * {@link NoViableDeviceError} when nothing survives — never returns a local
+ * fallback. Pure: all I/O is pre-resolved into `opts.signals`.
+ */
+export function pickBestDevice(devices, roster, opts) {
+    if (devices.length === 0) {
+        throw new Error('pickBestDevice called with an empty device pool');
+    }
+    const load = loadByDevice(devices, roster);
+    const { eligible, excluded } = classifyExclusions(devices, roster, opts);
+    if (eligible.length === 0) {
+        // A pool excluded ONLY for unreachability is a probe MISS / transient infra
+        // blip (a worker went briefly unresponsive during `teams start`), not proof
+        // the agent can't RUN there. Per the fail-loud contract — fire only on
+        // positive evidence (not-installed / saturated), never on a bare
+        // reachability miss — degrade to a best-effort roster-count pick so the wave
+        // retries and the real error surfaces at SSH dispatch, instead of a hard
+        // exit-1 that strands an all-remote pool on a transient blip.
+        const onlyUnreachable = excluded.every((e) => e.reason === 'unreachable');
+        if (!onlyUnreachable) {
+            throw new NoViableDeviceError(excluded, opts?.agentLabel);
+        }
+        let fallback = devices[0];
+        let fallbackLoad = load.get(fallback) ?? 0;
+        for (const d of devices) {
+            const l = load.get(d) ?? 0;
+            if (l < fallbackLoad) {
+                fallback = d;
+                fallbackLoad = l;
+            }
+        }
+        return fallback;
+    }
+    const signals = opts?.signals;
+    const order = new Map(devices.map((d, i) => [d, i]));
+    // Ascending composite sort — the first element is the best placement.
+    return [...eligible].sort((a, b) => {
+        const sa = signals?.get(a);
+        const sb = signals?.get(b);
+        // (a) signed-in agent first.
+        const signedIn = (sa?.signedIn === true ? 0 : 1) - (sb?.signedIn === true ? 0 : 1);
+        if (signedIn !== 0)
+            return signedIn;
+        // (b) lower load — coarse headroom tier, then raw load cost.
+        const tier = headroomTier(sa?.headroom) - headroomTier(sb?.headroom);
+        if (tier !== 0)
+            return tier;
+        // (c) fewer running teammates.
+        const teammates = (load.get(a) ?? 0) - (load.get(b) ?? 0);
+        if (teammates !== 0)
+            return teammates;
+        // finer load tiebreak: a known-lower load wins; unknown sits mid (50).
+        const cost = (loadCost(sa) ?? 50) - (loadCost(sb) ?? 50);
+        if (cost !== 0)
+            return cost;
+        // stable: first declared in the pool wins.
+        return (order.get(a) ?? 0) - (order.get(b) ?? 0);
+    })[0];
+}
 /**
  * Resolve where a teammate runs. Returns `{ device: null }` for a local run
  * (no pin, no pool, or the chosen device is the local machine) and
  * `{ device: <name> }` for a remote placement. See the cascade in the module
- * header.
+ * header. Throws {@link NoViableDeviceError} when a required pool has no viable
+ * host — the caller surfaces it at `teams start` rather than falling back local.
  */
 export function resolvePlacement(team, explicitDevice, roster, opts) {
-    // 1. Explicit pin wins — even without a pool.
+    // 1. Explicit pin wins — even without a pool — and is never second-guessed
+    //    (the remote `agents run` surfaces any install/health issue there).
     if (explicitDevice) {
         return { device: isLocalDevice(explicitDevice) ? null : explicitDevice };
     }
@@ -112,11 +281,19 @@ export function resolvePlacement(team, explicitDevice, roster, opts) {
     // 4. No pool → local, exactly like today.
     if (pool.length === 0)
         return { device: null };
-    // 2. Pool of one → the whole team runs there.
+    // 2. Pool of one → the whole team runs there. Respect the choice for
+    //    load/cap/reachability, but still fail loud when the agent is provably
+    //    NOT installed there — running it would be futile.
     if (pool.length === 1) {
+        if (opts?.signals?.get(pool[0])?.installed === false) {
+            throw new NoViableDeviceError([{ device: pool[0], reason: 'not-installed' }], opts.agentLabel);
+        }
         return { device: isLocalDevice(pool[0]) ? null : pool[0] };
     }
-    // 3. Many → least-loaded across the pool (cap-aware when caps are provided).
-    const picked = pickLeastLoaded(pool, roster, opts?.maxConcurrent);
+    // 3. Many → best viable pick. With live signals, this is health-, harness-,
+    //    and load-aware; without them it degrades to the cap-aware roster count.
+    const picked = opts?.signals
+        ? pickBestDevice(pool, roster, opts)
+        : pickLeastLoaded(pool, roster, opts?.maxConcurrent);
     return { device: isLocalDevice(picked) ? null : picked };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@phnx-labs/agents-cli",
-  "version": "1.22.25",
+  "version": "1.22.26",
   "description": "One CLI for all your AI coding agents - versions, config, cloud dispatch, sessions, and teams (now with first-class Grok Build CLI support)",
   "type": "module",
   "main": "dist/index.js",