@phnx-labs/agents-cli 1.22.49 → 1.22.51

package/dist/lib/teams/agents.js

@@ -35,9 +35,10 @@ import { pullRemoteLogDelta, REMOTE_MIRROR_MAX_BYTES } from '../hosts/progress.j
 import { createRemoteWorktree, ensureRemoteRepo } from './remoteWorktree.js';
 import { getTeam, isTeamDisbanded } from './registry.js';
 import { atomicWriteJsonSync } from '../fs-atomic.js';
-import { resolvePlacement, classifyExclusions, NoViableDeviceError } from './scheduler.js';
+import { resolvePlacement, classifyExclusions, isTransientPlacementBlock, NoViableDeviceError, } from './scheduler.js';
 import { probePoolSignals } from './placement-probe.js';
 import { readMaxConcurrentCaps } from '../device-config.js';
+import { redactSecrets, sanitizeForTerminal } from '../redact.js';
 import chalk from 'chalk';
 let lastMemoryWarnAt = 0;
 // On macOS, os.freemem() returns only the truly-free pool and ignores the
@@ -148,6 +149,9 @@ export const TERMINAL_STATUSES = new Set([
 export function isTerminalStatus(status) {
     return TERMINAL_STATUSES.has(status);
 }
+function safeFailureMessage(message) {
+    return redactSecrets(sanitizeForTerminal(message)).replace(/\s+/g, ' ').trim().slice(0, 500);
+}
 /**
  * The per-teammate shell that emits `<id> <ALIVE|EXITED|GONE> <codeOrEmpty>`.
  * Shared by the batched prefetch (many teammates, one round-trip) and the
@@ -546,6 +550,7 @@ export class AgentProcess {
     remotePid = null;
     remoteLog = null;
     remoteExit = null;
+    failure = null;
     // Offset-tail cursor into the REMOTE log (bytes already pulled). Distinct from
     // lastReadPos, which tracks the LOCAL mirror the parser consumes.
     remoteLogOffset = 0;
@@ -621,6 +626,7 @@ export class AgentProcess {
             cloud_session_id: this.cloudSessionId,
             cloud_repo: this.cloudRepo,
             cloud_branch: this.cloudBranch,
+            failure: this.failure,
             agent_dir: await this.getAgentDir(),
             cwd: this.cwd,
         };
@@ -668,6 +674,7 @@ export class AgentProcess {
             task_type: this.taskType,
             cloud_repo: this.cloudRepo,
             cloud_branch: this.cloudBranch,
+            failure: this.failure,
         };
     }
     duration() {
@@ -767,6 +774,13 @@ export class AgentProcess {
             const code = Number.parseInt(snap.exit.trim(), 10);
             if (Number.isFinite(code)) {
                 this.status = code === 0 ? AgentStatus.COMPLETED : AgentStatus.FAILED;
+                if (code !== 0) {
+                    this.failure = {
+                        stage: 'execution', code: 'remote-process-exit-nonzero',
+                        message: `Remote teammate process exited with code ${code}.`, exit_code: code,
+                        retryable: false, observed_at: new Date().toISOString(),
+                    };
+                }
                 if (!this.completedAt)
                     this.completedAt = new Date();
                 return;
@@ -780,6 +794,11 @@ export class AgentProcess {
         // RUNNING above precisely so this branch does not misfire on that race.
         if (this.status === AgentStatus.RUNNING && !snap.alive && !snap.exitFilePresent) {
             this.status = AgentStatus.FAILED;
+            this.failure = {
+                stage: 'execution', code: 'remote-process-gone',
+                message: 'Remote teammate process disappeared before recording an exit code.', exit_code: null,
+                retryable: true, observed_at: new Date().toISOString(),
+            };
             if (!this.completedAt)
                 this.completedAt = this.getLatestEventTime() || this.startedAt || new Date();
         }
@@ -870,6 +889,11 @@ export class AgentProcess {
                             }
                             else if (event.status === 'error') {
                                 this.status = AgentStatus.FAILED;
+                                this.failure = {
+                                    stage: 'execution', code: 'harness-reported-error',
+                                    message: 'The harness reported a terminal error.', exit_code: null,
+                                    retryable: false, observed_at: new Date().toISOString(),
+                                };
                                 this.completedAt = event.timestamp ? new Date(event.timestamp) : new Date();
                             }
                         }
@@ -992,6 +1016,7 @@ export class AgentProcess {
             remote_log: this.remoteLog,
             remote_exit: this.remoteExit,
             remote_log_offset: this.remoteLogOffset,
+            failure: this.failure,
         };
         const metaPath = await this.getMetaPath();
         atomicWriteJsonSync(metaPath, meta);
@@ -1088,6 +1113,16 @@ export class AgentProcess {
             agent.remoteLog = meta.remote_log || null;
             agent.remoteExit = meta.remote_exit || null;
             agent.remoteLogOffset = typeof meta.remote_log_offset === 'number' ? meta.remote_log_offset : 0;
+            agent.failure = meta.failure && typeof meta.failure === 'object'
+                ? {
+                    stage: meta.failure.stage,
+                    code: String(meta.failure.code),
+                    message: safeFailureMessage(String(meta.failure.message ?? '')),
+                    exit_code: typeof meta.failure.exit_code === 'number' ? meta.failure.exit_code : null,
+                    retryable: Boolean(meta.failure.retryable),
+                    observed_at: String(meta.failure.observed_at),
+                }
+                : null;
             // The team's project. Absent on every teammate added before `--project`.
             agent.project = typeof meta.project === 'string' ? meta.project : null;
             return agent;
@@ -1258,6 +1293,11 @@ export class AgentProcess {
                 const fallbackCompletion = this.getLatestEventTime() || this.startedAt || new Date();
                 if (this.status === AgentStatus.RUNNING) {
                     this.status = AgentStatus.FAILED;
+                    this.failure = {
+                        stage: 'spawn', code: 'process-identity-missing',
+                        message: 'The teammate was marked running without a process identity.', exit_code: null,
+                        retryable: true, observed_at: new Date().toISOString(),
+                    };
                     this.completedAt = fallbackCompletion;
                 }
                 await this.saveMeta();
@@ -1275,12 +1315,22 @@ export class AgentProcess {
             return;
         }
         if (this.status === AgentStatus.RUNNING) {
-            const exitCode = await this.reapProcess();
+            const exit = await this.reapProcess();
             await this.readNewEvents();
             if (this.status === AgentStatus.RUNNING) {
                 const fallbackCompletion = this.getLatestEventTime() || this.startedAt || new Date();
-                if (exitCode !== null && exitCode !== 0) {
+                if (exit !== null && exit.code !== 0) {
                     this.status = AgentStatus.FAILED;
+                    this.failure = {
+                        stage: 'execution',
+                        code: exit.sentinelPresent ? 'process-exit-nonzero' : 'process-exit-unrecorded',
+                        message: exit.sentinelPresent
+                            ? `Teammate process exited with code ${exit.code}.`
+                            : 'Teammate process disappeared before recording an exit code.',
+                        exit_code: exit.sentinelPresent ? exit.code : null,
+                        retryable: !exit.sentinelPresent,
+                        observed_at: new Date().toISOString(),
+                    };
                 }
                 else {
                     this.status = AgentStatus.COMPLETED;
@@ -1322,11 +1372,11 @@ export class AgentProcess {
         try {
             const raw = (await fs.readFile(await this.getExitCodePath(), 'utf-8')).trim();
             const code = Number.parseInt(raw, 10);
-            return Number.isNaN(code) ? 1 : code;
+            return { code: Number.isNaN(code) ? 1 : code, sentinelPresent: true };
         }
         catch {
             // No sentinel: the shell died before recording $? (killed mid-run).
-            return 1;
+            return { code: 1, sentinelPresent: false };
         }
     }
 }
@@ -1485,6 +1535,26 @@ export class AgentManager {
     registerAgent(agent) {
         this.agents.set(agent.agentId, agent);
     }
+    async failAgent(agent, failure) {
+        agent.failure = {
+            ...failure,
+            message: safeFailureMessage(failure.message),
+            observed_at: new Date().toISOString(),
+        };
+        agent.status = AgentStatus.FAILED;
+        agent.completedAt = new Date();
+        await agent.saveMeta();
+    }
+    async deferAgent(agent, failure) {
+        agent.failure = {
+            ...failure,
+            message: safeFailureMessage(failure.message),
+            observed_at: new Date().toISOString(),
+        };
+        agent.status = AgentStatus.PENDING;
+        agent.completedAt = null;
+        await agent.saveMeta();
+    }
     /**
      * Scan the agents dir for meta.json files not already in the in-memory
      * cache and load them. Needed when another process (e.g. a Planner
@@ -1761,7 +1831,7 @@ export class AgentManager {
         // --session-id so it's also Claude's session id (unified identity).
         const agentId = randomUUID();
         const isStaged = cleanAfter.length > 0;
-        const initialStatus = isStaged || !isCloudBacked
+        const initialStatus = isStaged || !isCloudBacked || !cloudSessionId
             ? AgentStatus.PENDING
             : AgentStatus.RUNNING;
         const agent = new AgentProcess(agentId, taskName, agentType, prompt, resolvedCwd, resolvedMode, null, initialStatus, new Date(), null, this.agentsDir, parentSessionId, workspaceDir, cloudSessionId, cloudProvider, null, version, null, name, cleanAfter, effort, model, envOverrides && Object.keys(envOverrides).length > 0 ? envOverrides : null, taskType, cloudRepo, cloudBranch, worktreeName, worktreePath, profileName);
@@ -1796,10 +1866,29 @@ export class AgentManager {
             debug(`Staged ${agentType} teammate '${name}' in team '${taskName}' (after: ${cleanAfter.join(', ')})`);
         }
         else if (isCloudBacked) {
-            // Cloud-backed teammate: the provider already dispatched a remote task.
-            // No local process to launch; status polling walks the provider instead.
-            await agent.saveMeta();
-            debug(`Cloud-backed ${agentType} teammate via ${cloudProvider} (session=${cloudSessionId})`);
+            if (cloudSessionId) {
+                // Compatibility path for API callers that already dispatched remotely.
+                await agent.saveMeta();
+                debug(`Cloud-backed ${agentType} teammate via ${cloudProvider} (session=${cloudSessionId})`);
+            }
+            else {
+                try {
+                    if (!this.cloudDispatcher)
+                        throw new Error('No cloud dispatcher registered.');
+                    const dispatched = await this.cloudDispatcher(agent);
+                    agent.cloudSessionId = dispatched.cloudSessionId;
+                    agent.status = AgentStatus.RUNNING;
+                    agent.startedAt = new Date();
+                    await agent.saveMeta();
+                }
+                catch (err) {
+                    await this.failAgent(agent, {
+                        stage: 'cloud', code: 'cloud-dispatch-failed', message: err.message,
+                        exit_code: null, retryable: true,
+                    });
+                    throw err;
+                }
+            }
         }
         else if (isRemoteBacked) {
             // Distributed teammate that can run now (no unmet --after deps): dispatch
@@ -1810,7 +1899,27 @@ export class AgentManager {
             // Unpinned + launching now: consult the pool scheduler before defaulting to
             // local, so an unpinned teammate on a --devices team auto-schedules even when
             // added without --after (it wouldn't pass through startReady otherwise).
-            await this.maybeSchedulePlacement(agent, taskName);
+            try {
+                await this.maybeSchedulePlacement(agent, taskName);
+            }
+            catch (err) {
+                if (isTransientPlacementBlock(err)) {
+                    await this.deferAgent(agent, {
+                        stage: 'placement', code: 'placement-capacity-wait', message: err.message,
+                        exit_code: null, retryable: true,
+                    });
+                    await this.cleanupOldAgents();
+                    return agent;
+                }
+                await this.failAgent(agent, {
+                    stage: 'placement',
+                    code: err instanceof NoViableDeviceError ? 'no-viable-device' : 'placement-failed',
+                    message: err.message,
+                    exit_code: null,
+                    retryable: !(err instanceof NoViableDeviceError),
+                });
+                throw err;
+            }
             if (agent.hostName)
                 await this.launchRemoteProcess(agent);
             else
@@ -1878,6 +1987,7 @@ export class AgentManager {
         const resume = { id: agent.remoteSessionId ?? agent.agentId, message };
         const priorRuntime = {
             status: agent.status,
+            failure: agent.failure,
             completedAt: agent.completedAt,
             pid: agent.pid,
             startTime: agent.startTime,
@@ -1894,6 +2004,13 @@ export class AgentManager {
         // that happens, restore the stopped lifecycle state and keep its existing
         // metadata/log directory intact so the user can retry.
         agent.status = AgentStatus.RUNNING;
+        // Failure evidence describes the CURRENT attempt. Once a resume has
+        // successfully launched, retaining the prior terminal attempt's failure on
+        // a RUNNING (and eventually COMPLETED) teammate is false state. Clear it
+        // before either launcher persists RUNNING; priorRuntime restores it if the
+        // replacement launch fails, so a failed resume never erases the evidence
+        // needed to diagnose and retry the original attempt.
+        agent.failure = null;
         agent.completedAt = null;
         try {
             if (agent.hostName) {
@@ -1992,11 +2109,15 @@ export class AgentManager {
                 await terminateSpawnedProcess(childProcess.pid);
             if (resumeLog)
                 await rollbackResumeLogTransaction(resumeLog);
-            // Fresh spawns own a newly-created directory, so a failed launch removes
-            // that partial record. A resume reuses an existing teammate: its caller
-            // restores the prior terminal state and preserves the directory for retry.
-            if (!resume)
-                await this.cleanupPartialAgent(agent);
+            if (!resume) {
+                await this.failAgent(agent, {
+                    stage: 'spawn',
+                    code: 'local-spawn-failed',
+                    message: err.message,
+                    exit_code: null,
+                    retryable: true,
+                });
+            }
             console.error(`Failed to spawn agent ${agent.agentId}:`, err);
             throw new Error(`Failed to spawn agent: ${err.message}`);
         }
@@ -2118,6 +2239,15 @@ export class AgentManager {
                 }
             }
             console.error(`Failed to launch remote teammate ${agent.agentId} on ${agent.hostName}:`, err);
+            if (!resume) {
+                await this.failAgent(agent, {
+                    stage: 'spawn',
+                    code: 'remote-launch-failed',
+                    message: err.message,
+                    exit_code: null,
+                    retryable: true,
+                });
+            }
             if (cleanupError) {
                 throw new Error(`Failed to launch remote teammate: ${err.message}; cleanup failed: ${cleanupError.message}`, { cause: err });
             }
@@ -2200,49 +2330,6 @@ export class AgentManager {
     placementAgentLabel(agent) {
         return agent.version ? `${agent.agentType}@${agent.version}` : String(agent.agentType);
     }
-    /**
-     * Pre-flight the team pool at `teams start` (RUSH-2002): probe the pool and
-     * confirm each distinct agent among the pending, unpinned, pooled teammates
-     * has at least one viable device. Returns the first {@link NoViableDeviceError}
-     * so the command can fail loud BEFORE launching a wave, rather than silently
-     * leaving teammates stranded pending. A poolless team (no `--devices`) or a
-     * probe that could not gather positive evidence returns null — fail-loud fires
-     * only on proof that no device can run the agent, never on a probe miss.
-     */
-    async preflightPlacement(taskName) {
-        await this.initialize();
-        const teamMeta = await getTeam(taskName);
-        const pool = teamMeta?.devices ?? [];
-        if (!teamMeta || pool.length === 0)
-            return null;
-        const roster = await this.listByTask(taskName);
-        const pending = roster.filter((a) => a.status === AgentStatus.PENDING && !a.hostName && !a.cloudProvider);
-        if (pending.length === 0)
-            return null;
-        const maxConcurrent = pool.length > 1 ? readMaxConcurrentCaps(pool) : undefined;
-        // One representative teammate per distinct agent type — the signal + gate is
-        // per harness, not per teammate.
-        const byAgent = new Map();
-        for (const a of pending)
-            if (!byAgent.has(a.agentType))
-                byAgent.set(a.agentType, a);
-        for (const [, rep] of byAgent) {
-            const signals = await probePoolSignals(pool, rep.agentType, { now: Date.now() });
-            try {
-                resolvePlacement(teamMeta, null, roster, {
-                    maxConcurrent,
-                    signals,
-                    agentLabel: this.placementAgentLabel(rep),
-                });
-            }
-            catch (err) {
-                if (err instanceof NoViableDeviceError)
-                    return err;
-                throw err;
-            }
-        }
-        return null;
-    }
     /**
      * One-ssh-per-host batched liveness/exit pre-pass for a team's remote teammates.
      * The supervisor calls this each wave BEFORE listByTask() so the per-teammate
@@ -2319,6 +2406,20 @@ export class AgentManager {
         for (const agent of teammates) {
             if (agent.status !== AgentStatus.PENDING)
                 continue;
+            const blockers = agent.after
+                .map((depName) => ({ depName, dep: byName.get(depName) }))
+                .filter(({ dep }) => !dep || (isTerminalStatus(dep.status) && dep.status !== AgentStatus.COMPLETED));
+            if (blockers.length > 0) {
+                const names = blockers.map(({ depName, dep }) => `${depName} (${dep?.status ?? 'missing'})`);
+                await this.failAgent(agent, {
+                    stage: 'dependency',
+                    code: 'dependency-failed',
+                    message: `Blocked by dependency: ${names.join(', ')}.`,
+                    exit_code: null,
+                    retryable: false,
+                });
+                continue;
+            }
             const depsReady = agent.after.every((depName) => {
                 const dep = byName.get(depName);
                 return dep && dep.status === AgentStatus.COMPLETED;
@@ -2333,9 +2434,21 @@ export class AgentManager {
                 await this.maybeSchedulePlacement(agent, taskName, { probe: true });
             }
             catch (err) {
-                // A NoViableDeviceError means the pool cannot host this teammate right
-                // now — leave it PENDING (never a silent local fallback) and surface the
-                // reason; a transient device loss is retried next wave.
+                if (isTransientPlacementBlock(err)) {
+                    await this.deferAgent(agent, {
+                        stage: 'placement', code: 'placement-capacity-wait', message: err.message,
+                        exit_code: null, retryable: true,
+                    });
+                    console.error(`Placement deferred for ${agent.agentId}; the pool may free up on a later wave:`, err);
+                    continue;
+                }
+                await this.failAgent(agent, {
+                    stage: 'placement',
+                    code: err instanceof NoViableDeviceError ? 'no-viable-device' : 'placement-failed',
+                    message: err.message,
+                    exit_code: null,
+                    retryable: !(err instanceof NoViableDeviceError),
+                });
                 console.error(`Could not schedule ${agent.agentId} onto the team pool:`, err);
                 continue;
             }
@@ -2347,7 +2460,12 @@ export class AgentManager {
                 }
                 else if (agent.cloudProvider) {
                     if (!this.cloudDispatcher) {
-                        console.error(`Cannot start cloud-backed teammate ${agent.agentId}: no dispatcher registered.`);
+                        const message = `Cannot start cloud-backed teammate ${agent.agentId}: no dispatcher registered.`;
+                        await this.failAgent(agent, {
+                            stage: 'cloud', code: 'cloud-dispatcher-missing', message,
+                            exit_code: null, retryable: false,
+                        });
+                        console.error(message);
                         continue;
                     }
                     const { cloudSessionId } = await this.cloudDispatcher(agent);
@@ -2363,6 +2481,13 @@ export class AgentManager {
                 }
             }
             catch (err) {
+                await this.failAgent(agent, {
+                    stage: agent.cloudProvider ? 'cloud' : 'spawn',
+                    code: agent.cloudProvider ? 'cloud-dispatch-failed' : agent.hostName ? 'remote-launch-failed' : 'local-spawn-failed',
+                    message: err.message,
+                    exit_code: null,
+                    retryable: true,
+                });
                 console.error(`Could not launch ${agent.agentId}:`, err);
             }
         }
@@ -2657,16 +2782,6 @@ export class AgentManager {
         }
         return false;
     }
-    async cleanupPartialAgent(agent) {
-        this.agents.delete(agent.agentId);
-        try {
-            const agentDir = await agent.getAgentDir();
-            await fs.rm(agentDir, { recursive: true });
-        }
-        catch (err) {
-            console.warn(`Failed to clean up agent directory:`, err);
-        }
-    }
     async cleanupOldAgents() {
         // listCompleted() is terminal-only (isTerminalStatus), so a pending or
         // running teammate is never a reap candidate — retention can only delete a

package/dist/lib/teams/api.d.ts

@@ -1,4 +1,4 @@
-import { AgentManager, type TaskType } from './agents.js';
+import { AgentManager, type TaskType, type TeammateFailure } from './agents.js';
 import { AgentType } from './parsers.js';
 export interface ToolCallDetail {
     tool: string;
@@ -22,6 +22,8 @@ export interface SpawnResult {
     cloud_session_id?: string | null;
     /** Device name the teammate runs on for a distributed (--on) teammate; null for local. */
     host?: string | null;
+    /** Sanitized evidence observed at the lifecycle boundary that failed. */
+    failure?: TeammateFailure | null;
 }
 /** Detailed status of a single teammate, including file ops, commands, and a cursor for delta polling. */
 export interface AgentStatusDetail {
@@ -60,6 +62,8 @@ export interface AgentStatusDetail {
     task_type?: TaskType | null;
     /** Device name the teammate runs on for a distributed (--on) teammate; null for local. */
     host?: string | null;
+    /** Sanitized evidence observed at the lifecycle boundary that failed. */
+    failure?: TeammateFailure | null;
 }
 /** Aggregated status of all teammates in a task, with per-status counts and a global cursor. */
 export interface TaskStatusResult {
@@ -121,6 +125,7 @@ export interface AgentStatusSummary {
     last_messages: string[];
     /** Device name for a distributed (--on) teammate; null for local. */
     host: string | null;
+    failure: TeammateFailure | null;
     /** ISO timestamp — feed back via --since for delta polling. */
     cursor: string;
 }

package/dist/lib/teams/api.js

@@ -125,6 +125,7 @@ export function toAgentStatusSummary(detail) {
             .slice(-SUMMARY_MAX_MESSAGES)
             .map((m) => trimMessage(m)),
         host: detail.host ?? null,
+        failure: detail.failure ?? null,
         cursor: detail.cursor,
     };
 }
@@ -253,6 +254,7 @@ parentSessionId) {
             after: agent.after,
             task_type: agent.taskType,
             host: agent.hostName,
+            failure: agent.failure,
             mode: agent.mode,
             cloud_session_id: agent.cloudSessionId,
             cloud_provider: agent.cloudProvider,

package/dist/lib/teams/scheduler.d.ts

@@ -68,6 +68,13 @@ export declare class NoViableDeviceError extends Error {
     readonly agentLabel?: string;
     constructor(excluded: ExcludedDevice[], agentLabel?: string);
 }
+/**
+ * Whether a failed placement can become viable without changing the team.
+ * Capacity and load are time-varying; a pool with at least one such exclusion
+ * must remain schedulable so a later supervisor wave can retry it. A pool that
+ * is only missing the requested harness is a durable configuration failure.
+ */
+export declare function isTransientPlacementBlock(error: unknown): error is NoViableDeviceError;
 /** Build the user-facing fail-loud message from the per-device reasons. */
 export declare function formatNoViableMessage(excluded: ExcludedDevice[], agentLabel?: string): string;
 /**

package/dist/lib/teams/scheduler.js

@@ -50,9 +50,20 @@ export class NoViableDeviceError extends Error {
         this.agentLabel = agentLabel;
     }
 }
+/**
+ * Whether a failed placement can become viable without changing the team.
+ * Capacity and load are time-varying; a pool with at least one such exclusion
+ * must remain schedulable so a later supervisor wave can retry it. A pool that
+ * is only missing the requested harness is a durable configuration failure.
+ */
+export function isTransientPlacementBlock(error) {
+    return error instanceof NoViableDeviceError
+        && error.excluded.some((entry) => entry.reason === 'capped' || entry.reason === 'overloaded');
+}
 /** Build the user-facing fail-loud message from the per-device reasons. */
 export function formatNoViableMessage(excluded, agentLabel) {
     const anyNotInstalled = excluded.some((e) => e.reason === 'not-installed');
+    const allCapped = excluded.length > 0 && excluded.every((e) => e.reason === 'capped');
     const agent = agentLabel ?? 'the requested agent';
     const perDevice = excluded
         .map((e) => (e.detail ? `${e.device} (${e.reason} ${e.detail})` : `${e.device} (${e.reason})`))
@@ -61,14 +72,21 @@ export function formatNoViableMessage(excluded, agentLabel) {
     // ticket specifies; otherwise summarize the mixed reasons.
     const head = anyNotInstalled
         ? `No device in the team pool can run ${agent}.`
-        : `No viable device in the team pool for ${agent}.`;
+        : allCapped
+            ? `Every device in the pool is at its agents.max-concurrent cap:`
+            : `No viable device in the team pool for ${agent}.`;
     const anyUnreachable = excluded.some((e) => e.reason === 'unreachable');
     const hint = anyNotInstalled
         ? " Run 'agents devices ping' to see which devices have the agent installed + signed in, or add the agent to a pool device."
         : anyUnreachable
             ? " Add a device to the pool, raise a cap, or wait for the pool to free up / come back online ('agents devices ping')."
-            : ' Add a device to the pool, raise a cap, or bring an overloaded box under load.';
-    return `${head} ${perDevice}.${hint}`;
+            : allCapped
+                ? " Raise a cap with 'agents devices config <name> agents.max-concurrent N' or add a device to the pool."
+                : ' Add a device to the pool, raise a cap, or bring an overloaded box under load.';
+    const reasons = allCapped
+        ? excluded.map((e) => `${e.device} (${e.detail})`).join(', ')
+        : perDevice;
+    return `${head} ${reasons}.${hint}`;
 }
 /** True when `device` names the local machine (case-insensitive). */
 function isLocalDevice(device) {
@@ -128,11 +146,11 @@ export function pickLeastLoaded(devices, roster, maxConcurrent) {
     const capped = new Set(maxConcurrent ? cappedDevices(devices, roster, maxConcurrent).map((c) => c.device) : []);
     const eligible = devices.filter((d) => !capped.has(d));
     if (eligible.length === 0) {
-        const detail = devices
-            .map((d) => `${d} (${load.get(d) ?? 0}/${maxConcurrent[d]})`)
-            .join(', ');
-        throw new Error(`Every device in the pool is at its agents.max-concurrent cap: ${detail}. ` +
-            `Raise a cap with 'agents devices config <name> agents.max-concurrent N' or add a device to the pool.`);
+        throw new NoViableDeviceError(devices.map((device) => ({
+            device,
+            reason: 'capped',
+            detail: `${load.get(device) ?? 0}/${maxConcurrent[device]}`,
+        })));
     }
     // Iterate the pool in declared order so the first device wins ties.
     let best = eligible[0];

package/dist/lib/traces/sync.d.ts

@@ -9,6 +9,15 @@
  * Sync gate: `sessions.db` `file_mtime_ms` is the source of truth.  A ledger
  * at `getRuntimeStateDir()/traces-sync.json` records the last sync timestamp
  * per device so re-runs skip unchanged sessions.
+ *
+ * Retry ledger: the mtime watermark alone cannot retry failures — a later-mtime
+ * success advances it past an earlier failed row, which the next run's
+ * `file_mtime_ms > watermark` filter then skips forever. So the same ledger also
+ * records the identity + typed evidence of each failed session (`failures`), and
+ * the row query unions those retry-worthy ids back in regardless of the watermark
+ * (PHNX-3267). Failures are typed so a gone transcript (`transcript-unavailable`,
+ * expected history, aged out after a TTL and never re-read) is distinguished from a
+ * real parse/upload failure (retried until it resolves).
  */
 import { type SessionTrajectory } from '../session/trajectory.js';
 import { type BucketStats, type DriftSignal, type TraceFailureCause, type TraceTopicGroup } from './classify.js';
@@ -30,7 +39,14 @@ export interface SyncOpts {
 export interface SyncResult {
     uploaded: number;
     skipped: number;
+    /** Total failures = transcriptUnavailable + parseFailed + uploadFailed. Kept for callers. */
     errors: number;
+    /** The transcript file the row points at is gone/unreadable — expected history, not retried each run. */
+    transcriptUnavailable: number;
+    /** The transcript exists but could not be parsed into a trajectory — retried until it parses. */
+    parseFailed: number;
+    /** Parsed fine but the upload PUT failed (network/5xx) — retried on the next sync. */
+    uploadFailed: number;
 }
 /** Push derived, redacted trajectories for this device to the traces store. */
 export declare function syncTraces(opts?: SyncOpts): Promise<SyncResult>;
@@ -107,8 +123,22 @@ export interface SessionDetail {
  * written to R2. `repo` is the cwd basename only.
  */
 export declare function buildSessionDetail(traj: SessionTrajectory): SessionDetail;
+/** How a session failed to sync. Only parse/upload failures are re-queried every run. */
+export type SyncFailureKind = 'transcript-unavailable' | 'parse-failed' | 'upload-failed';
+export interface SyncFailure {
+    id: string;
+    mtimeMs: number;
+    kind: SyncFailureKind;
+    /** Bounded, redacted first line of the underlying error — actionable evidence. */
+    detail: string;
+    /** Epoch ms first recorded, so a permanently-unavailable transcript can be aged out. */
+    firstSeen: number;
+    attempts: number;
+}
 interface SyncLedger {
     lastSyncMtime?: number;
+    /** Failed session identities, keyed by id, that survive across syncs. */
+    failures?: SyncFailure[];
 }
 export declare function readSyncLedger(): SyncLedger;
 export declare function writeSyncLedger(ledger: SyncLedger): void;