@phnx-labs/agents-cli 1.22.39 → 1.22.41

package/dist/lib/runner.js

@@ -18,15 +18,15 @@ import * as fs from 'fs';
 import * as path from 'path';
 import * as os from 'os';
 import { getCliLaunch, getAgentsBinDir } from './cli-entry.js';
-import { resolveJobPrompt, parseTimeout, writeRunMeta, listRuns, getJobRunsDir, getRunDir, checkJobDeviceEligibility, finalizeRunMeta, resolveJobExecutionContext, slotRunId, claimRunSlot, readRunMeta, resolveHostStrategy, } from './routines.js';
-import { getRunsDir, getUserAgentsDir, readMeta } from './state.js';
+import { resolveJobPrompt, parseTimeout, writeRunMeta, listRuns, getJobRunsDir, getRunDir, checkJobDeviceEligibility, finalizeRunMeta, resolveJobExecutionContext, slotRunId, claimRunSlot, readRunMeta, resolveHostStrategy, } from './scheduling/routines.js';
+import { getRunsDir, getUserAgentsDir, readMeta, getDaemonDir } from './state.js';
 import { shortCodexHome } from './codex-home.js';
 import { prepareJobHome, buildSpawnEnv, getJobHomePath } from './sandbox.js';
 import { resolveModel, buildReasoningFlags } from './models.js';
-import { createTimer, redactPrompt, emitRoutineEnd } from './events.js';
+import { createTimer, redactPrompt, emitRoutineEnd } from './feed/events.js';
 import { codexEditWritableRoots, codexPolicyArgs } from './codex-policy.js';
 import { applyAddDirs } from './add-dir.js';
-import { normalizeMode, resolveHeadlessMode, buildExecEnv, detectRateLimit, detectAuthFailure, isAuthFailureFromLog, authFailureReason, } from './exec.js';
+import { normalizeMode, resolveHeadlessMode, buildExecEnv, detectRateLimit, detectAuthFailure, isAuthFailureFromLog, authFailureReason, AGENT_COMMANDS, } from './exec.js';
 import { resolveActor } from './actor.js';
 import { loadTask as loadHostTask } from './hosts/tasks.js';
 import { reconcileTask as reconcileHostTask } from './hosts/reconcile.js';
@@ -34,12 +34,12 @@ import { backgroundSpawnOptions, killTree } from './platform/process.js';
 import lockfile from 'proper-lockfile';
 import { ensureLockTarget } from './fs-atomic.js';
 import { walkForFiles } from './fs-walk.js';
-import { getBinaryPath, isVersionInstalled, resolveVersion, getVersionHomePath } from './versions.js';
+import { getBinaryPath, isVersionInstalled, resolveVersion, getVersionHomePath } from './installations/versions.js';
 import { resolveClaudeSetupToken } from './claude-account-token.js';
-import { getConfiguredRunStrategy, resolveRunVersion, resolveAccountVersion, rotationFailoverChain, readinessFromCandidate, formatNoHealthyAccountError, } from './rotate.js';
+import { getConfiguredRunStrategy, resolveRunVersion, resolveAccountVersion, rotationFailoverChain, readinessFromCandidate, formatNoHealthyAccountError, } from './accounting/rotate.js';
 import { readAuthHealth, isDeadVerdict } from './auth-health.js';
 import { machineId } from './machine-id.js';
-import { isSelfUpdatingAgent, ROUTINE_AGENT_COMMANDS as AGENT_COMMANDS, ROUTINE_AGENT_IDS, isAgentHardDeprecated, hardDeprecationError } from './agents.js';
+import { isSelfUpdatingAgent, ROUTINE_AGENT_COMMANDS, isAgentHardDeprecated, hardDeprecationError } from './agents.js';
 export class RoutineAlreadyRunningError extends Error {
     constructor(jobName, runId) {
         super(`Routine '${jobName}' already has a running execution (${runId})`);
@@ -48,24 +48,97 @@ export class RoutineAlreadyRunningError extends Error {
 }
 const ROUTINE_LAUNCH_LOCK_STALE_MS = 30_000;
 const ROUTINE_LAUNCH_LOCK_WAIT_MS = 10_000;
+/**
+ * The prior run of this routine that still holds the active-run slot, or null.
+ *
+ * ONLY a run still marked `running` can hold the slot. Every terminal status —
+ * completed / failed / timeout / missed / blocked / skipped — releases it the
+ * moment it is reached (RUSH-2640). A `failed`/`timeout` record used to keep the
+ * slot while its pid stayed alive, but that conflated two different operations:
+ * cleaning up a leftover process group (reapTerminalRoutineProcesses's job) with
+ * occupying the slot. A daemon-launched run that reached a terminal state while
+ * still carrying the daemon's own pid (a host run that threw `target_unreachable`
+ * before spawning a child) then held the slot forever, because the daemon never
+ * dies and `isPidOurs` never went false — every later scheduled slot was refused
+ * with `already has an active run` while the routine was silently dead.
+ *
+ * A `running` record is also aged out past its own timeout: a run cannot
+ * legitimately outlive its configured deadline, so past that window it is a
+ * wedged record (a daemon that died mid-run, a missed child-exit event, a reused
+ * pid an old record without `spawnedAt` reads as "ours"), not a live run — and it
+ * never holds the slot regardless of what its recorded pid now points at.
+ */
 function activeRoutineRun(config) {
     const timeoutMs = parseTimeout(config.timeout) || 10 * 60 * 1000;
     const now = Date.now();
     const runs = listRuns(config.name);
     for (let i = runs.length - 1; i >= 0; i--) {
         const run = runs[i];
-        if (!['running', 'failed', 'timeout'].includes(run.status))
+        if (run.status !== 'running')
+            continue;
+        const startedAt = Date.parse(run.startedAt);
+        const limit = run.timeoutMs ?? timeoutMs;
+        if (Number.isFinite(startedAt) && now - startedAt >= limit)
             continue;
-        if (run.pid && isPidOurs(run.pid, run.spawnedAt))
+        // A provisional launcher claim carries no child pid yet — trust it until the
+        // timeout window above elapses. A record with a child pid is active only
+        // while that pid is genuinely ours (alive with the same birth time).
+        if (!run.pid)
+            return run;
+        if (isPidOurs(run.pid, run.spawnedAt))
             return run;
-        if (!run.pid) {
-            const startedAt = Date.parse(run.startedAt);
-            if (Number.isFinite(startedAt) && now - startedAt < (run.timeoutMs ?? timeoutMs))
-                return run;
-        }
     }
     return null;
 }
+/**
+ * Consecutive trailing `skipped`/`active_run` records — a routine that keeps
+ * being refused because a prior run "still owns" the slot. After the slot is
+ * released correctly on terminal states (see {@link activeRoutineRun}) this can
+ * only accumulate for a genuine long-overlap or a new wedge; either way it means
+ * the routine has stopped firing on schedule and should be surfaced, not left to
+ * pile up silently. `runs` is oldest→newest (listRuns sorts by run id).
+ */
+export function activeRunSkipStreak(runs) {
+    let streak = 0;
+    for (let i = runs.length - 1; i >= 0; i--) {
+        const run = runs[i];
+        if (run.status === 'skipped' && run.skipReason === 'active_run') {
+            streak++;
+            continue;
+        }
+        break;
+    }
+    return streak;
+}
+/** How many consecutive active-run skips before a routine's stall is surfaced. */
+const SKIP_STREAK_ALERT_THRESHOLD = 3;
+/**
+ * The pid to stamp on a launcher's provisional active claim, or null when the
+ * launcher is this box's routines daemon.
+ *
+ * The claim pid exists so a launcher that dies between claiming the slot and
+ * spawning the child releases the slot quickly (`isPidOurs` goes false) instead
+ * of wedging it for the full timeout. That fast recovery only helps a SHORT-LIVED
+ * foreground launcher (`agents routines run`): the daemon never dies between
+ * claim and spawn, and recording ITS pid is actively harmful — the daemon
+ * outlives every run, so `isPidOurs` never goes false and a claim finalized to a
+ * terminal state before its child spawned kept the slot and drew
+ * `reapTerminalRoutineProcesses` at the daemon's own process group on every tick
+ * (RUSH-2640, requirement: never record the daemon's own pid as a run's pid). So
+ * the daemon records no claim pid; the timeout window bounds a daemon that
+ * crashes mid-claim instead.
+ */
+export function launcherClaimPid() {
+    try {
+        // Mirrors daemon.ts readDaemonPid() (PID_FILE = 'daemon.pid'); read directly
+        // rather than importing it to avoid a runner<->daemon import cycle.
+        const daemonPid = parseInt(fs.readFileSync(path.join(getDaemonDir(), 'daemon.pid'), 'utf-8').trim(), 10);
+        if (Number.isFinite(daemonPid) && daemonPid === process.pid)
+            return null;
+    }
+    catch { /* no daemon pid file — a foreground launcher; record its pid */ }
+    return process.pid;
+}
 /** Serialize concurrent launches of one routine on this box (the on-disk claim
  *  primitive `claimRunSlot` is what holds across processes; this lock only makes
  *  the allocate→spawn window atomic within this process's launch paths). */
@@ -150,10 +223,13 @@ function writeActiveClaim(config, attempt) {
         ...attempt.stamp,
         ...(config.workflow ? { workflow: config.workflow } : config.command ? { command: config.command } : config.agent ? { agent: config.agent } : {}),
         // The launcher owns this provisional claim until the command/agent child
-        // replaces it with its own pid. Recording the owner makes a crash between
-        // lock release and child spawn recoverable instead of wedging the routine
-        // for its full configured timeout.
-        pid: process.pid,
+        // replaces it with its own pid. Recording a SHORT-LIVED foreground launcher's
+        // pid makes a crash between lock release and child spawn recoverable instead
+        // of wedging the routine for its full configured timeout. The long-lived
+        // daemon records no pid here (see launcherClaimPid): its pid never dies, so
+        // recording it wedged the slot forever and mis-aimed the process reaper at the
+        // daemon itself (RUSH-2640).
+        pid: launcherClaimPid(),
         // `isPidOurs` compares this value with the OS process birth time. The
         // daemon may have been alive for days before claiming a routine, so the
         // claim must carry the launcher's birth, not the claim timestamp.
@@ -228,7 +304,11 @@ function allocateRoutineAttempt(config, trigger) {
             }),
         };
     }
-    if (!config.workflow && config.agent && ROUTINE_AGENT_IDS.includes(config.agent) && isAgentHardDeprecated(config.agent)) {
+    // Gate on deprecation alone, not ROUTINE_AGENT_IDS membership — a retired
+    // harness leaves the command table (gemini did), and the legacy routine must
+    // still land a visible 'blocked' record, never a generic buildJobCommand
+    // failure (RUSH-2202).
+    if (!config.workflow && config.agent && isAgentHardDeprecated(config.agent)) {
         const reason = hardDeprecationError(config.agent);
         return {
             proceed: false,
@@ -278,6 +358,24 @@ function allocateRoutineAttempt(config, trigger) {
     }
     return { proceed: true, attempt: { runId, stamp } };
 }
+/**
+ * Emit a loud, once-per-wedge warning when a routine has been refused its slot
+ * for {@link SKIP_STREAK_ALERT_THRESHOLD} consecutive scheduled runs. Fires only
+ * when the streak first reaches the threshold (not on every later skip), so a
+ * genuinely-stalled routine surfaces once instead of piling up silent `skipped`
+ * records the way RUSH-2640 did. Runs on the daemon's own stderr, so it lands in
+ * the daemon log the operator reads.
+ */
+function surfaceWedgedRoutine(config, terminal) {
+    if (terminal.status !== 'skipped' || terminal.skipReason !== 'active_run')
+        return;
+    const streak = activeRunSkipStreak(listRuns(config.name));
+    if (streak !== SKIP_STREAK_ALERT_THRESHOLD)
+        return;
+    const stuck = terminal.activeRunId ?? 'unknown';
+    process.stderr.write(`[agents] routine '${config.name}' skipped ${streak} consecutive scheduled runs — ` +
+        `its active-run slot is still held by ${stuck}; the routine is not firing on schedule.\n`);
+}
 /**
  * Claim one routine attempt under the short launch lock, then execute after
  * releasing it. The persisted `running` claim makes a concurrent entry point
@@ -293,8 +391,10 @@ async function runWithAttempt(config, trigger, run, wrapTerminal) {
         writeActiveClaim(config, alloc.attempt);
         return { attempt: alloc.attempt };
     });
-    if ('terminal' in claimed)
+    if ('terminal' in claimed) {
+        surfaceWedgedRoutine(config, claimed.terminal);
         return wrapTerminal(claimed.terminal);
+    }
     try {
         return await run(claimed.attempt);
     }
@@ -316,13 +416,31 @@ async function runWithAttempt(config, trigger, run, wrapTerminal) {
 function terminateRoutineTree(pid) {
     if (!pid)
         return;
+    // Never take THIS process down. Both kills below are unconditional SIGKILLs,
+    // so a RunMeta naming the reaper's own pid -- a reused pid, a record written
+    // by the process now doing the reaping, or a hand-written fixture -- makes the
+    // reaper SIGKILL itself, and via `-pid` its whole process group with it. There
+    // is no recovery from that: the run is never finalized, and on the daemon it
+    // takes the scheduler down mid-sweep. daemon.ts:457 already refuses to evict an
+    // incumbent whose pid is `process.pid` for exactly this reason; the reap path
+    // needs the same guard. Skipping the kill still lets the caller finalize the
+    // run record, which is the part that matters.
+    if (pid === process.pid)
+        return;
     if (process.platform === 'win32') {
         killTree(pid);
         return;
     }
+    // Detached spawn makes the child a new process-group leader; kill the group.
+    // If setsid has not landed yet (or the pid is not a leader), -pid fails with
+    // ESRCH and the child stays up — also kill the pid itself.
     try {
         process.kill(-pid, 'SIGKILL');
     }
+    catch { /* not a group leader, or already exited */ }
+    try {
+        process.kill(pid, 'SIGKILL');
+    }
     catch { /* already exited */ }
 }
 /**
@@ -396,7 +514,7 @@ export function buildJobCommand(config, resolvedPrompt, forwardAccount = true) {
             cmd.push('--account', config.account);
         return cmd;
     }
-    const template = AGENT_COMMANDS[agent];
+    const template = ROUTINE_AGENT_COMMANDS[agent];
     if (!template) {
         throw new Error(`Unsupported agent for daemon jobs: ${agent}`);
     }
@@ -433,15 +551,6 @@ export function buildJobCommand(config, resolvedPrompt, forwardAccount = true) {
         cmd.push(...codexPolicyArgs(policyMode, [...codexEditWritableRoots(), ...routineRoots]));
         appendModelAndReasoning(cmd, config);
     }
-    if (config.agent === 'gemini') {
-        if (mode === 'edit') {
-            cmd.push('--approval-mode', 'auto_edit');
-        }
-        else if (mode === 'skip') {
-            cmd.push('--yolo');
-        }
-        appendModelAndReasoning(cmd, config);
-    }
     if (config.agent === 'cursor') {
         if (mode === 'plan') {
             cmd.push('--plan');
@@ -512,7 +621,7 @@ export function buildJobCommand(config, resolvedPrompt, forwardAccount = true) {
     return cmd;
 }
 /**
- * Append --model and reasoning flags to a command being assembled.
+ * Append the agent's canonical model flag and reasoning flags to a command.
  *
  * Pass-through model resolution: validates against the installed (agent, version)
  * catalog when possible and writes a warning to stderr on miss, but never blocks.
@@ -523,15 +632,19 @@ function appendModelAndReasoning(cmd, config) {
     const agent = config.agent;
     const model = config.config?.model;
     if (model) {
+        const modelFlag = AGENT_COMMANDS[agent].modelFlag;
+        if (!modelFlag) {
+            throw new Error(`Agent ${agent} does not support routine model selection`);
+        }
         if (config.version) {
             const resolved = resolveModel(agent, config.version, model);
             if (resolved.warning) {
                 process.stderr.write(`[agents] ${resolved.warning}\n`);
             }
-            cmd.push('--model', resolved.forwarded);
+            cmd.push(modelFlag, resolved.forwarded);
         }
         else {
-            cmd.push('--model', model);
+            cmd.push(modelFlag, model);
         }
     }
     const reasoning = config.config?.reasoning;
@@ -772,18 +885,31 @@ export async function resolveRoutineLaunch(config, cwd = process.cwd(), deps = {
     // resolveRoutineLaunch is only called for agent jobs (workflow returns above;
     // command jobs branch out of execute*Job before reaching this).
     const agent = config.agent;
-    const { findAccount, resolveAccountSelection, resolveCredentialAccount } = await import('./account-registry.js');
+    const { findAccount, findUnifiedAccount, resolveAccountSelection, resolveCredentialAccount } = await import('./account-registry.js');
+    const meta = (deps.readMeta ?? readMeta)();
     const explicitCredential = config.account
         ? (deps.findCredentialAccount?.(config.account) ?? (deps.resolveCredentialAccount !== undefined || findAccount(config.account) !== null))
         : false;
     const selectedCredential = config.account
         ? (explicitCredential ? config.account : undefined)
-        : resolveAccountSelection(undefined, agent, (deps.readMeta ?? readMeta)());
+        : resolveAccountSelection(undefined, agent, meta);
     if (selectedCredential) {
-        (deps.resolveCredentialAccount ?? resolveCredentialAccount)(selectedCredential, agent);
+        // A default/binding may resolve to a native account — never route that
+        // through the provider credential path (it has no bundle to resolve).
+        const unified = findUnifiedAccount(selectedCredential, meta);
+        if (unified?.kind !== 'native')
+            (deps.resolveCredentialAccount ?? resolveCredentialAccount)(selectedCredential, agent);
     }
     if (config.account && !explicitCredential) {
-        const accountVersion = await (deps.resolveAccountVersion ?? resolveAccountVersion)(agent, config.account);
+        // A native routine account is named by its durable name; the version matcher
+        // keys on the identity (email/accountKey), so translate before resolving,
+        // and refuse a login that belongs to a different harness.
+        const unified = findUnifiedAccount(config.account, meta);
+        if (unified?.kind === 'native' && unified.agent !== agent) {
+            throw new Error(`Routine '${config.name}' account '${config.account}' is a ${unified.agent} login and cannot authenticate ${agent}.`);
+        }
+        const identity = unified?.kind === 'native' ? unified.identityKey : config.account;
+        const accountVersion = await (deps.resolveAccountVersion ?? resolveAccountVersion)(agent, identity);
         if (accountVersion) {
             if (config.version && config.version !== accountVersion) {
                 throw new Error(`Routine '${config.name}' account '${config.account}' is signed in at ${agent}@${accountVersion}, not pinned ${agent}@${config.version}.`);
@@ -809,7 +935,10 @@ export async function resolveRoutineLaunch(config, cwd = process.cwd(), deps = {
             ...(config.account ? { forwardAccount: explicitCredential } : {}),
         };
     }
-    const strategy = getConfiguredRunStrategy(agent, cwd);
+    // A per-routine strategy travels with the definition and beats the firing
+    // box's ambient run.<agent>.strategy — otherwise selection policy silently
+    // depends on whichever device fires the job (RUSH-2719).
+    const strategy = config.strategy ?? getConfiguredRunStrategy(agent, cwd);
     let version;
     let rotation = null;
     let exhausted;
@@ -892,6 +1021,70 @@ export function pinJobBinary(cmd, agent, version) {
  * Such commands must NOT be binary-pinned (pinning rewrites cmd[0] to the agent binary,
  * producing a broken `<binary> run …`) and must not receive a version-pinned spawn env.
  */
+/**
+ * Assert a routine's account can be dispatched to the resolved placement, BEFORE
+ * any off-box dispatch (placement is resolved before {@link resolveRoutineLaunch}),
+ * at the top of both the foreground and detached paths:
+ *
+ * - **native** account → rejected for host AND cloud: a native login is a
+ *   device-local harness credential that cannot be forwarded off-box.
+ * - **provider** account + **cloud** → rejected (fail loud): the cloud dispatch
+ *   has no secure way to inject a device-local provider bundle yet.
+ * - **provider** account + **host** → allowed: the host dispatch forwards the
+ *   account NAME (the remote resolves its own local bundle — no secret copied).
+ *
+ * `account` is injectable so the guard is unit-tested for both modes without a
+ * registry or a real dispatch.
+ */
+export async function assertRoutineAccountLocalForPlacement(config, mode, deps = {}) {
+    if (!config.account)
+        return;
+    let account = deps.account;
+    if (account === undefined) {
+        const { findUnifiedAccount } = await import('./account-registry.js');
+        account = findUnifiedAccount(config.account, (deps.readMeta ?? readMeta)());
+    }
+    if (!account) {
+        throw new Error(`Routine '${config.name}' account '${config.account}' is unknown.`);
+    }
+    if (account?.kind === 'native') {
+        throw new Error(`Routine '${config.name}' account '${config.account}' is a device-local ${account.agent} login and cannot run on a ${mode} placement. Use a provider account, or place this routine on the device that holds the login.`);
+    }
+    if (mode === 'cloud' && account?.kind === 'provider') {
+        // RUSH-2689: cloud+provider injection not yet implemented.
+        throw new Error(`Routine '${config.name}' account '${config.account}' is a provider credential; cloud placement cannot securely inject it (RUSH-2689). Run this routine locally or on a host that holds the bundle.`);
+    }
+}
+export async function dispatchPlacedJob(config, target, attempt, deps = {}) {
+    if (target.mode !== 'host' && target.mode !== 'cloud')
+        return undefined;
+    await assertRoutineAccountLocalForPlacement(config, target.mode, { account: deps.account });
+    if (target.mode === 'host') {
+        return (deps.host ?? executeJobOnHost)({ ...config, host: target.host }, { detached: false }, attempt);
+    }
+    return (deps.cloud ?? executeJobOnCloud)(config, { detached: false }, attempt);
+}
+/**
+ * Build the options passed to `dispatchPromptToHost` for a host routine. Split
+ * out so the execution boundary is unit-testable: it MUST forward the routine's
+ * `account` by name (the remote resolves its own local bundle; no secret copied)
+ * — dropping it silently ran the remote under the wrong identity.
+ */
+export function buildHostDispatchOptions(config, ctx) {
+    return {
+        agent: config.agent,
+        prompt: resolveJobPrompt(config),
+        mode: normalizeMode(config.mode),
+        effort: config.effort,
+        model: config.config?.model,
+        account: config.account,
+        timeout: config.timeout, // enforced by the REMOTE agents run
+        remoteCwd: ctx.remoteCwd,
+        name: config.name,
+        cwd: ctx.runDir,
+        follow: !ctx.detached,
+    };
+}
 export function dispatchesViaAgentsRun(config) {
     return Boolean(config.workflow || config.resume);
 }
@@ -1069,13 +1262,10 @@ async function executeJobPlaced(config, deps, attempt) {
     // remote run to completion when possible.
     {
         const { resolvePlacementTarget } = await import('./routines-placement.js');
-        const target = resolvePlacementTarget(config);
-        if (target.mode === 'host') {
-            return executeJobOnHost({ ...config, host: target.host }, { detached: false }, attempt);
-        }
-        if (target.mode === 'cloud') {
-            return executeJobOnCloud(config, { detached: false }, attempt);
-        }
+        const target = await resolvePlacementTarget(config);
+        const placed = await dispatchPlacedJob(config, target, attempt);
+        if (placed)
+            return placed;
     }
     // Command-mode: run a plain shell command directly (no agent, no rotation,
     // no pinning, no sandbox overlay). Reuses the run-record machinery so
@@ -1111,12 +1301,16 @@ async function executeJobPlaced(config, deps, attempt) {
     // (command jobs branched out earlier, so config.agent is set on the non-workflow path.)
     const effectiveAgent = config.workflow ? 'claude' : config.agent;
     if (!dispatchesViaAgentsRun(config)) {
-        const { findAccount, resolveAccountSelection, resolveCredentialAccount } = await import('./account-registry.js');
-        const selectedAccount = config.account && !findAccount(config.account)
-            ? undefined
-            : resolveAccountSelection(config.account, effectiveAgent, readMeta());
-        if (selectedAccount)
-            Object.assign(baseEnv, resolveCredentialAccount(selectedAccount, effectiveAgent).env);
+        const { findUnifiedAccount, resolveAccountSelection, resolveCredentialAccount } = await import('./account-registry.js');
+        const meta = readMeta();
+        const selectedAccount = resolveAccountSelection(config.account, effectiveAgent, meta);
+        if (selectedAccount) {
+            // Only a provider account injects env; a native account (explicit or via a
+            // device-scoped binding) is read from the harness home and forwards nothing.
+            const unified = findUnifiedAccount(selectedAccount, meta);
+            if (unified?.kind === 'provider')
+                Object.assign(baseEnv, resolveCredentialAccount(selectedAccount, effectiveAgent).env);
+        }
     }
     const meta = {
         jobName: config.name,
@@ -1443,18 +1637,7 @@ async function executeJobOnHost(config, opts, attempt) {
         host: host.name,
     };
     writeRunMeta(meta);
-    const { task, exitCode } = await dispatchPromptToHost(host, {
-        agent: config.agent,
-        prompt: resolveJobPrompt(config),
-        mode: normalizeMode(config.mode),
-        effort: config.effort,
-        model: config.config?.model,
-        timeout: config.timeout, // enforced by the REMOTE agents run
-        remoteCwd,
-        name: config.name,
-        cwd: runDir,
-        follow: !opts.detached,
-    });
+    const { task, exitCode } = await dispatchPromptToHost(host, buildHostDispatchOptions(config, { remoteCwd, runDir, detached: opts.detached }));
     meta.hostTaskId = task.id;
     // Sync path: a real exit code finalizes now. -1 (follow window closed) and
     // the detached path leave the meta `running` for the monitor to reconcile.
@@ -1572,7 +1755,10 @@ async function executeJobDetachedClaimed(config, attempt, hooks) {
     // sends the finish notification only for local detached runs below (RUSH-2030).
     {
         const { resolvePlacementTarget } = await import('./routines-placement.js');
-        const target = resolvePlacementTarget(config);
+        const target = await resolvePlacementTarget(config);
+        if (target.mode === 'host' || target.mode === 'cloud') {
+            await assertRoutineAccountLocalForPlacement(config, target.mode);
+        }
         if (target.mode === 'host') {
             const { meta } = await executeJobOnHost({ ...config, host: target.host }, { detached: true }, attempt);
             if (meta.status !== 'running')
@@ -2072,17 +2258,23 @@ export function monitorRunningJobs() {
                     finalizeHostRun(meta);
                     continue;
                 }
-                if (!meta.pid)
-                    continue;
                 const runDirPath = path.join(jobRunsPath, runDirEntry.name);
                 const stdoutPath = path.join(runDirPath, 'stdout.log');
                 // Command-mode records carry no agent; there is no stream-json report to
                 // parse or extract. Reap them on pid liveness alone.
                 const isCommandRun = Boolean(meta.command) || !meta.agent;
+                // Age-out runs FIRST, before the null-pid guard below, so a wedged record
+                // still marked `running` past its deadline is always finalized — a
+                // provisional launcher claim whose daemon crashed before spawning a child
+                // (pid null), or an old record whose recorded pid was reused, would
+                // otherwise linger as `running` forever (RUSH-2640). Only kill a process
+                // group that is genuinely still ours; terminating a reused pid would kill
+                // an unrelated process, and a null pid has nothing to kill.
                 const wallClockMs = Date.now() - Date.parse(meta.startedAt);
                 const timeoutMs = meta.timeoutMs ?? MAX_WALL_CLOCK_MS;
                 if (Number.isFinite(wallClockMs) && wallClockMs > timeoutMs) {
-                    terminateRoutineTree(meta.pid);
+                    if (meta.pid && isPidOurs(meta.pid, meta.spawnedAt))
+                        terminateRoutineTree(meta.pid);
                     finalizeRunMeta(meta, 'timeout', null, { errorMessage: 'exceeded configured timeout' });
                     writeRunMeta(meta);
                     emitRoutineEnd(meta);
@@ -2092,6 +2284,8 @@ export function monitorRunningJobs() {
                     }
                     continue;
                 }
+                if (!meta.pid)
+                    continue;
                 if (!isPidOurs(meta.pid, meta.spawnedAt)) {
                     if (isCommandRun) {
                         // Command routines normally record their own terminal status via

package/dist/lib/sandbox.d.ts

@@ -6,7 +6,7 @@
  * The spawned agent process sees only the overlay, limiting filesystem
  * access to explicitly allowed paths.
  */
-import type { JobConfig } from './routines.js';
+import type { JobConfig } from './scheduling/routines.js';
 /** Build a restricted environment for a sandboxed process, setting HOME to the overlay. */
 export declare function buildSpawnEnv(overlayHome: string, extraEnv?: Record<string, string>): Record<string, string>;
 /**

package/dist/lib/scheduler.d.ts

@@ -5,7 +5,7 @@
  * process creates a single JobScheduler instance that loads enabled jobs
  * on startup and reloads them on SIGHUP.
  */
-import type { JobConfig } from './routines.js';
+import type { JobConfig } from './scheduling/routines.js';
 /** How a fire was triggered, carrying the scheduler's intended UTC slot time. */
 export interface TriggerContext {
     /** The cron slot this callback fires for (croner `currentRun()`), for the

package/dist/lib/scheduler.js

@@ -6,7 +6,7 @@
  * on startup and reloads them on SIGHUP.
  */
 import { Cron } from 'croner';
-import { listJobs, deleteJob, isPastEndAt, isPastOneShotRoutine, isOneShotRoutine, setJobEnabled, shouldPurgeCompletedOneShotRoutine, jobRunsOnThisDevice, hasAmbiguousDevicePin, routineOwnerDevice, } from './routines.js';
+import { listJobs, deleteJob, isPastEndAt, isPastOneShotRoutine, isOneShotRoutine, setJobEnabled, shouldPurgeCompletedOneShotRoutine, jobRunsOnThisDevice, hasAmbiguousDevicePin, routineOwnerDevice, } from './scheduling/routines.js';
 /** In-memory cron scheduler that triggers a callback when jobs fire. */
 export class JobScheduler {
     jobs = new Map();