@sentropic/h2a-runtime 0.81.0

package/dist/delegate.js

@@ -0,0 +1,739 @@
+/**
+ * `remote delegate <type> "<task>"` — P1 of cross-type agent delegation.
+ *
+ * Spawns a LIVE, INTERACTIVE agent (claude / codex / agy) in a DETACHED tmux
+ * session, PRIMED with the task, with the h2a MCP server in a side window (the
+ * existing `remote run --h2a` path) so the parent/master feedback loop works.
+ * Returns a job id (slug). The job is enrolled in the SAME registry as sessions
+ * with `role: "job"` (no second jobs.json), so `listLive` + the liveness guards
+ * + the atomic write are reused as-is.
+ *
+ * Isolation: by default, if the cwd is a git repo, each job runs in its OWN git
+ * worktree under `<cwd>/.remote/jobs/<jobId>/wt` — the single-writer guard only
+ * protects the conversation `.jsonl`, not the file tree, so concurrent jobs in
+ * the shared cwd would clobber each other. `--cwd` overrides; a non-repo cwd is
+ * used as-is.
+ *
+ * `--headless` is the run-once-exit variant (NOT drop-to-shell): it runs
+ * `claude -p` / `codex exec`, redirects stdout+stderr to `<dir>/output.log`,
+ * writes `<dir>/result.json {state,exitCode}`, then ENDS the tmux session.
+ *
+ * SECURITY: the task is passed as a SINGLE argv token — never concatenated into
+ * a shell string. The jobId / `--name` pass `assertSafeName` before becoming a
+ * tmux slug, a directory, or a filename.
+ */
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { occupiesSlot } from "./registry.js";
+import { humanAge } from "./migrate-candidates.js";
+const DELEGATE_BIN = {
+    claude: "claude",
+    codex: "codex",
+    agy: "agy",
+};
+/** Job id / `--name`: becomes a tmux slug, a dir, a filename — keep it tame. */
+const SAFE_NAME = /^[A-Za-z0-9_-]+$/;
+export function assertSafeName(name) {
+    if (!SAFE_NAME.test(name)) {
+        throw new Error(`invalid job name "${name}" (allowed: letters, digits, "_", "-")`);
+    }
+}
+export function isDelegateType(value) {
+    return value === "claude" || value === "codex" || value === "agy";
+}
+/**
+ * The EXACT argv for the agent binary, by type and mode. Pure, exported for
+ * tests. The task is ALWAYS a single trailing token (no shell concat):
+ *  - interactive: the agent starts live, primed with the task as a positional
+ *    initial prompt (claude/codex/agy all accept a bare positional prompt);
+ *  - headless: a run-once-exit print mode — claude `-p`, codex `exec`. agy has
+ *    no confirmed headless mode (design R3), so headless agy throws.
+ *
+ * Returns `{ command, args }` where args ends with the task token.
+ */
+export function buildDelegateArgs(type, task, headless, model, effort) {
+    const command = DELEGATE_BIN[type];
+    // Flags prepended before the task/subcommand. Claude supports both
+    // --model and --effort; Codex exposes model via -m (global flag before
+    // subcommands); agy: ignore unknown flags (passed positional only).
+    const modelFlags = (() => {
+        if (!model && !effort)
+            return [];
+        switch (type) {
+            case "claude": {
+                const flags = [];
+                if (model)
+                    flags.push("--model", model);
+                if (effort)
+                    flags.push("--effort", effort);
+                return flags;
+            }
+            case "codex":
+                return model ? ["-m", model] : [];
+            case "agy":
+                return [];
+        }
+    })();
+    if (!headless) {
+        // Interactive + initial prompt — task is the last argv token.
+        return { command, args: [...modelFlags, task] };
+    }
+    switch (type) {
+        case "claude":
+            return { command, args: [...modelFlags, "-p", task] };
+        case "codex":
+            return { command, args: [...modelFlags, "exec", task] };
+        case "agy":
+            throw new Error("agy has no confirmed headless mode — run it interactively (drop --headless)");
+    }
+}
+// ---------------------------------------------------------------------------
+// Reliability slice 1 — rate-limit "throttled" RESUME (HEADLESS LOCAL only).
+//
+// When a HEADLESS LOCAL job finished on a transient provider rate-limit, the
+// conductor RELAUNCHES the SAME job in its `runCwd` with the tool's CONTINUE
+// flag so it picks up its prior conversation instead of starting over. The
+// continue flag is derived from the job's type; the task still rides as a single
+// trailing argv token (NEVER `bash -lc` string-concat — same safe channel as
+// `buildDelegateArgs`). PURE arg derivation, exported for tests.
+// ---------------------------------------------------------------------------
+/** Hard cap on auto-resume attempts before a throttled job is failed `rate-limited`. */
+export const THROTTLE_MAX_ATTEMPTS = 6;
+/** Backoff base (ms) for the resume schedule. */
+export const THROTTLE_BACKOFF_BASE_MS = 60_000; // 60s
+/** Backoff cap (ms) for the resume schedule. */
+export const THROTTLE_BACKOFF_CAP_MS = 30 * 60_000; // 30 min
+/**
+ * The EXACT argv to RESUME a throttled HEADLESS job, by type. The job continues
+ * its prior conversation (rather than re-running from scratch) and re-prints to
+ * the same log, so a fresh success settles it `done`:
+ *  - claude: `-p --continue <task>` (print mode, continue the last conversation).
+ *  - codex:  `exec resume --last <task>` (resume the most recent rollout).
+ *  - agy:    no confirmed headless mode → throws (same as buildDelegateArgs).
+ *
+ * The task stays a single trailing token (no shell concat). Pure, exported for
+ * tests.
+ *
+ * TODO(phase-2, interactive): an INTERACTIVE throttled tmux job would instead be
+ * nudged via `send-keys` (guarded by an attached-pane check so we never type
+ * into a pane the user is driving). Not in this slice.
+ */
+export function buildThrottleResumeArgs(type, task) {
+    const command = DELEGATE_BIN[type];
+    switch (type) {
+        case "claude":
+            return { command, args: ["-p", "--continue", task] };
+        case "codex":
+            return { command, args: ["exec", "resume", "--last", task] };
+        case "agy":
+            throw new Error("agy has no confirmed headless resume mode — interactive resume is phase 2");
+    }
+}
+/**
+ * Full-jitter exponential backoff delay for the throttle RESUME schedule —
+ * copied from the session-agent's `websocket-transport.ts` `jitteredDelay`
+ * (same formula, kept local so this module stays pure/dependency-free).
+ * Returns a random value in [0, min(cap, base * 2^attempt)]. `attempt` is
+ * 0-based (the FIRST throttle passes attempt 0). `rand` is injectable so tests
+ * pin the jitter. Pure, exported for tests.
+ */
+export function jitteredBackoffMs(attempt, baseMs = THROTTLE_BACKOFF_BASE_MS, capMs = THROTTLE_BACKOFF_CAP_MS, rand = Math.random) {
+    const ceiling = Math.min(capMs, baseMs * Math.pow(2, Math.max(0, attempt)));
+    return rand() * ceiling;
+}
+/**
+ * The CEILING of the full-jitter window for a given attempt: `min(cap, base *
+ * 2^attempt)`. Monotonic non-decreasing in `attempt`, capped at `capMs`. The
+ * jittered delay is always in `[0, ceiling]`. Pure, exported so tests can assert
+ * the schedule is monotonic + bounded without depending on the random draw.
+ */
+export function backoffCeilingMs(attempt, baseMs = THROTTLE_BACKOFF_BASE_MS, capMs = THROTTLE_BACKOFF_CAP_MS) {
+    return Math.min(capMs, baseMs * Math.pow(2, Math.max(0, attempt)));
+}
+// ---------------------------------------------------------------------------
+// Reliability slice 1 — AIMD circuit breaker for ADMITTING pending jobs.
+//
+// A provider rate-limit is ACCOUNT-WIDE: if jobs are already throttling, starting
+// MORE pending jobs just burns the same quota and they throttle too. So we
+// MULTIPLICATIVELY DECREASE the effective admission cap when a burst of throttles
+// is observed (≥2 jobs entered `throttled` within the last 10 min → halve), and
+// ADDITIVELY INCREASE it (+1) on every clean pass (no new throttle), restoring up
+// to the configured cap. Classic AIMD. PURE: given (configuredCap,
+// throttleHistory, now[, lastCap]) it returns the effective cap; the conductor
+// keeps `lastCap` across passes. Exported for tests.
+//
+// `throttled` jobs KEEP their slot (registry.occupiesSlot), so this only governs
+// admitting NEW `pending` work — it never evicts a throttled job.
+// ---------------------------------------------------------------------------
+/** Window over which throttle events are counted for the breaker (10 min). */
+export const AIMD_WINDOW_MS = 10 * 60_000;
+/** ≥ this many throttles in the window trips the multiplicative decrease. */
+export const AIMD_TRIP_THRESHOLD = 2;
+/**
+ * The effective admission cap under AIMD. Pure.
+ *
+ *  - Count throttle events with `firstAt`/`at` within `[now - AIMD_WINDOW_MS, now]`.
+ *  - If that count ≥ AIMD_TRIP_THRESHOLD → HALVE the previous effective cap
+ *    (`Math.floor`), floored at 1 (never admit zero — a single slot keeps the
+ *    queue making progress so the breaker can observe recovery).
+ *  - Else (a clean pass) → ADDITIVELY restore +1, capped at `configuredCap`.
+ *  - Always clamped to `[1, configuredCap]` (never below 1, never above the
+ *    configured cap). A `configuredCap <= 0` returns 0 (a degenerate config
+ *    admits nothing — mirrors planNextStarts).
+ *
+ * `lastCap` is the effective cap from the PREVIOUS pass (defaults to
+ * `configuredCap` on the first pass — we start fully open). `recentThrottles` is
+ * the list of throttle-event timestamps (ISO or ms) the conductor tracks.
+ */
+export function aimdEffectiveCap(configuredCap, recentThrottles, now = Date.now(), lastCap) {
+    if (configuredCap <= 0)
+        return 0;
+    const prev = lastCap === undefined
+        ? configuredCap
+        : Math.min(configuredCap, Math.max(1, Math.trunc(lastCap)));
+    const windowStart = now - AIMD_WINDOW_MS;
+    const inWindow = recentThrottles.filter((t) => {
+        const ms = typeof t === "number" ? t : Date.parse(t);
+        return Number.isFinite(ms) && ms >= windowStart && ms <= now;
+    }).length;
+    let next;
+    if (inWindow >= AIMD_TRIP_THRESHOLD) {
+        next = Math.max(1, Math.floor(prev / 2)); // multiplicative decrease
+    }
+    else {
+        next = prev + 1; // additive increase
+    }
+    // Clamp to [1, configuredCap].
+    if (next < 1)
+        next = 1;
+    if (next > configuredCap)
+        next = configuredCap;
+    return next;
+}
+/**
+ * Decide what to do with a HEADLESS LOCAL job whose latest finished run was
+ * classified as a transient rate-limit. PURE — no clock/registry/IO; the caller
+ * passes `now`, the prior `throttle` bookkeeping (undefined on the first hit),
+ * the matched `signature`, and a `delay` (a jittered backoff in ms, computed via
+ * `jitteredBackoffMs`).
+ *
+ *  - First/next throttle while `attempts < maxAttempts`: bump `attempts`,
+ *    preserve `firstAt`, schedule `nextRetryAt = now + delay`.
+ *  - When the NEXT attempt would exceed `maxAttempts` (i.e. prior attempts already
+ *    == maxAttempts): give up → `{action:"fail", reason:"rate-limited"}`.
+ *
+ * Exported for tests.
+ */
+export function planThrottleStep(opts) {
+    const max = opts.maxAttempts ?? THROTTLE_MAX_ATTEMPTS;
+    const priorAttempts = opts.prior?.attempts ?? 0;
+    if (priorAttempts >= max) {
+        return { action: "fail", reason: "rate-limited" };
+    }
+    const nowIso = new Date(opts.nowMs).toISOString();
+    const attempts = priorAttempts + 1;
+    return {
+        action: "throttle",
+        attempts,
+        firstAt: opts.prior?.firstAt ?? nowIso,
+        nextRetryAt: new Date(opts.nowMs + opts.delayMs).toISOString(),
+        ...(opts.signature !== undefined ? { signature: opts.signature } : {}),
+    };
+}
+/**
+ * Is a `throttled` job DUE for resume right now (now >= nextRetryAt)? Pure,
+ * tolerant of a missing/unparseable `nextRetryAt` (treated as due — better to
+ * retry than to strand a job forever). Exported for tests.
+ */
+export function isThrottleResumeDue(throttle, nowMs) {
+    const at = throttle?.nextRetryAt;
+    if (at === undefined)
+        return true;
+    const ms = Date.parse(at);
+    if (!Number.isFinite(ms))
+        return true;
+    return nowMs >= ms;
+}
+/**
+ * Minutes-until-retry label for a `throttled` row (`jobs ls`), e.g. "retry in
+ * 3m" / "retry now". Pure, exported for tests. Rounds UP so "<1 min away" still
+ * reads as 1m rather than 0.
+ */
+export function throttleRetryLabel(throttle, nowMs = Date.now()) {
+    const at = throttle?.nextRetryAt;
+    if (at === undefined)
+        return "retry now";
+    const ms = Date.parse(at);
+    if (!Number.isFinite(ms) || ms <= nowMs)
+        return "retry now";
+    const mins = Math.ceil((ms - nowMs) / 60_000);
+    return `retry in ${mins}m`;
+}
+/**
+ * P2 — REMOTE delegation. Map a (type, task) to the control-plane session body
+ * for a delegated agent running in a Pod. The agent CLI is the session PROFILE
+ * (profiles claude/codex/agy launch the matching binary on the Pod), and the
+ * task rides the ALREADY-SAFE argv channel `startupArgs` — exactly the args that
+ * follow the binary, which the session-agent reads from `SESSION_STARTUP_ARGS`
+ * (a JSON array, never a `bash -lc` string). We reuse `buildDelegateArgs` and
+ * DROP its `command` (the profile already provides the binary): `args` is what
+ * trails it (the bare task token interactive, `-p <task>` / `exec <task>`
+ * headless). Pure, exported for tests.
+ *
+ * SECURITY: the task stays a single argv element through `startupArgs` → JSON →
+ * the agent's argv; it is NEVER concatenated into a shell string.
+ */
+export function buildRemoteDelegate(type, task, headless, model, effort) {
+    const { args } = buildDelegateArgs(type, task, headless, model, effort);
+    return { profile: type, startupArgs: args };
+}
+// ---------------------------------------------------------------------------
+// P4 — concurrency cap + queue + spawn-depth (pure, conductor-driving)
+// ---------------------------------------------------------------------------
+/**
+ * Default concurrency cap (Hermes `max_concurrent_children`), applied to BOTH
+ * local AND remote jobs: the shared RWX volume is mounted RW across all nodes
+ * (subPath per job — no CSI packing limit, design F3 correction), so remote is
+ * not sequential. A new `delegate` beyond `running` jobs is enrolled `pending`.
+ */
+export const DEFAULT_MAX_CONCURRENT = 16;
+/** Default / min / max spawn depth (Hermes `max_spawn_depth`, clamp 1–3). */
+export const DEFAULT_MAX_DEPTH = 1;
+export const MIN_MAX_DEPTH = 1;
+export const MAX_MAX_DEPTH = 3;
+/** The env channel carrying the REMAINING delegation depth into a job's agent. */
+export const DEPTH_ENV = "REMOTE_DELEGATE_DEPTH";
+/**
+ * Env channel carrying a delegated job's id INTO its agent process (H1). The
+ * local-tmux spawn stamps `REMOTE_JOB_ID=<jobId>` so the claude SessionStart/End
+ * hooks — which only receive claude's conversation uuid (`session_id`), never the
+ * job slug — can resolve the job and complete it. Without this, an interactive
+ * tmux job never advances past `running`.
+ */
+export const JOB_ID_ENV = "REMOTE_JOB_ID";
+/**
+ * Clamp a requested `--max-depth` into [1, 3] (à la Hermes). A missing /
+ * non-finite value falls back to the default (1). Pure, exported for tests.
+ */
+export function clampDepth(requested) {
+    if (requested === undefined || !Number.isFinite(requested)) {
+        return DEFAULT_MAX_DEPTH;
+    }
+    const n = Math.trunc(requested);
+    if (n < MIN_MAX_DEPTH)
+        return MIN_MAX_DEPTH;
+    if (n > MAX_MAX_DEPTH)
+        return MAX_MAX_DEPTH;
+    return n;
+}
+/**
+ * The depth a job INHERITS from its parent, read off `REMOTE_DELEGATE_DEPTH`
+ * (decremented at each launch). When a `delegate` runs OUTSIDE a job (no env),
+ * the requested `--max-depth` is the budget. When it runs INSIDE a job, the
+ * inherited budget wins (a job cannot grant itself more than it was given).
+ * Pure, exported for tests; `env` is injectable.
+ *
+ * Returns the budget the CURRENT `delegate` may spend:
+ *  - no env (top-level): clamp(requestedMaxDepth).
+ *  - env present: the inherited value (already a remaining budget), clamped
+ *    to [0, 3] — 0 means "this job may not delegate further".
+ */
+export function inheritedDepthBudget(requestedMaxDepth, env = process.env) {
+    const raw = env[DEPTH_ENV];
+    if (raw === undefined)
+        return clampDepth(requestedMaxDepth);
+    const parsed = Number.parseInt(raw, 10);
+    if (!Number.isFinite(parsed))
+        return clampDepth(requestedMaxDepth);
+    if (parsed < 0)
+        return 0;
+    if (parsed > MAX_MAX_DEPTH)
+        return MAX_MAX_DEPTH;
+    return parsed;
+}
+/**
+ * May a `delegate` proceed given its current depth budget? A budget of 0 means
+ * the calling job has exhausted its delegation depth and must REFUSE. Pure.
+ */
+export function canDelegateAtDepth(budget) {
+    return budget > 0;
+}
+/**
+ * REMOTE depth clamp (YAGNI/safety): there is NO env channel carrying
+ * `REMOTE_DELEGATE_DEPTH` into a Pod (`createRemoteSession` only passes
+ * `startupArgs`, and the session-agent doesn't forward a depth budget), so a
+ * remote job could NOT enforce a budget even if we recorded one > 1. Until that
+ * channel exists we clamp a REMOTE job's recorded budget to AT MOST 1 — i.e. a
+ * job-in-a-Pod does not re-delegate. (Local jobs DO inherit the budget via the
+ * env, so they keep the full 1–3 range.) Pure, exported for tests.
+ */
+export function clampRemoteDepthBudget(budget) {
+    return Math.min(budget, 1);
+}
+/**
+ * The depth value to PROPAGATE to a freshly launched job: the parent's budget
+ * minus one (never below 0), so the child knows how much further it may go.
+ * Returned as the string the env channel carries. Pure, exported for tests.
+ */
+export function childDepthEnvValue(parentBudget) {
+    return String(Math.max(0, parentBudget - 1));
+}
+/**
+ * Decide which `pending` jobs to START this pass, given the cap. Pure, the heart
+ * of the queue: count jobs currently occupying a slot (`running`), and admit
+ * `pending` jobs (oldest first — FIFO by `enrolledAt`) until `running` reaches
+ * the cap. Terminal jobs (done/failed) free their slot. Returns the ids to
+ * start, in admission order. Exported for tests.
+ *
+ * `cap <= 0` admits nothing (a degenerate config never starts work).
+ */
+export function planNextStarts(jobs, cap) {
+    if (cap <= 0)
+        return [];
+    const onlyJobs = jobs.filter((j) => j.role === "job");
+    // `running` AND `throttled` jobs occupy a slot (a throttled job keeps its slot
+    // — the rate-limit is account-wide; see registry.occupiesSlot).
+    const running = onlyJobs.filter((j) => occupiesSlot(j.jobState ?? "pending")).length;
+    let free = cap - running;
+    if (free <= 0)
+        return [];
+    const pending = onlyJobs
+        .filter((j) => (j.jobState ?? "pending") === "pending")
+        .slice()
+        .sort((a, b) => Date.parse(a.enrolledAt) - Date.parse(b.enrolledAt));
+    const out = [];
+    for (const j of pending) {
+        if (free <= 0)
+            break;
+        out.push(j.id);
+        free -= 1;
+    }
+    return out;
+}
+/**
+ * Is there room to start a job RIGHT NOW (used by `delegate` to decide launch vs
+ * enqueue)? True when `running < cap`. Pure, exported for tests.
+ */
+export function hasFreeSlot(jobs, cap) {
+    if (cap <= 0)
+        return false;
+    const running = jobs.filter((j) => j.role === "job" && occupiesSlot(j.jobState ?? "pending")).length;
+    return running < cap;
+}
+// ---------------------------------------------------------------------------
+// P4 — track mirror (job graph = backlog). PURE arg derivation; best-effort.
+// ---------------------------------------------------------------------------
+/**
+ * The `track item new` argv that mirrors a delegated job as a child item under
+ * the workpackage `wpId`. PURE (no spawn): the caller resolves the track binary
+ * (realpath) and runs it best-effort. `--role item`, parented under the WP, with
+ * a stable title derived from the job. Exported for tests.
+ *
+ * SECURITY: jobId is assertSafeName-checked at delegate time; wpId/task ride
+ * structured argv (never a shell string).
+ */
+export function trackItemNewArgs(wpId, job) {
+    const title = job.task && job.task.length > 0 ? job.task : `job ${job.id}`;
+    return [
+        "item",
+        "new",
+        "--parent",
+        wpId,
+        "--role",
+        "item",
+        "--title",
+        title,
+        "--ref",
+        `job:${job.id}`,
+    ];
+}
+/**
+ * The `track item realize` argv to close the mirror item when a job reaches a
+ * terminal state. We address the item by the same `job:<id>` ref we created it
+ * with. PURE; the caller runs it best-effort. Exported for tests.
+ */
+export function trackItemRealizeArgs(job) {
+    return ["item", "realize", "--ref", `job:${job.id}`];
+}
+/**
+ * Resolve the `track` binary to its REALPATH (some track versions have an
+ * entrypoint guard that breaks through the npm-global bin symlink — same note as
+ * config.ts PluginMcp). Returns `node <realpath>` argv, or undefined when track
+ * is not installed. Best-effort, exported for tests (the `which`+realpath are
+ * injectable).
+ */
+export function resolveTrackBin(which = defaultWhich, realpath = (p) => p) {
+    const found = which("track");
+    if (!found)
+        return undefined;
+    try {
+        return { command: "node", prefix: [realpath(found)] };
+    }
+    catch {
+        return { command: found, prefix: [] };
+    }
+}
+function defaultWhich(bin) {
+    const r = spawnSync("sh", ["-lc", `command -v ${bin}`], {
+        encoding: "utf8",
+        stdio: ["ignore", "pipe", "ignore"],
+    });
+    const out = r.status === 0 ? r.stdout.trim() : "";
+    return out.length > 0 ? out : undefined;
+}
+/**
+ * Best-effort track-mirror call: resolve `track`, run `track <args>` in `cwd`,
+ * swallow ANY failure (track absent/erroring must NEVER couple delivery to
+ * track). Returns whether it ran. The runner is injectable for tests.
+ */
+export function runTrackMirror(args, cwd, run) {
+    try {
+        if (run) {
+            run(args);
+            return true;
+        }
+        const bin = resolveTrackBin();
+        if (!bin)
+            return false;
+        spawnSync(bin.command, [...bin.prefix, ...args], { cwd, stdio: "ignore" });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/** Per-job directory under the ORIGIN cwd (where worktree + logs live). */
+export function jobDir(originCwd, jobId) {
+    return join(originCwd, ".remote", "jobs", jobId);
+}
+/**
+ * The terminal result a HEADLESS job wrote to `result.json` (run-once-exit), or
+ * undefined when absent / unreadable. Lets supervision reconcile a finished
+ * headless job (whose tmux session has ended) to its REAL state instead of the
+ * default "session-gone → failed". Pure-ish (reads one file), injectable read.
+ */
+export function readJobResult(originCwd, jobId, read = (p) => readFileSync(p, "utf8")) {
+    const path = join(jobDir(originCwd, jobId), "result.json");
+    try {
+        const parsed = JSON.parse(read(path));
+        if ((parsed.state === "done" || parsed.state === "failed") &&
+            typeof parsed.exitCode === "number") {
+            return { state: parsed.state, exitCode: parsed.exitCode };
+        }
+    }
+    catch {
+        // missing / unreadable / malformed → no result yet
+    }
+    return undefined;
+}
+function defaultIsGitRepo(cwd) {
+    if (!existsSync(cwd))
+        return false;
+    const r = spawnSync("git", ["rev-parse", "--is-inside-work-tree"], {
+        cwd,
+        encoding: "utf8",
+        stdio: ["ignore", "pipe", "ignore"],
+    });
+    return r.status === 0 && r.stdout.trim() === "true";
+}
+function defaultRunGit(cmd, args, cwd) {
+    const r = spawnSync(cmd, [...args], { cwd, stdio: "inherit" });
+    return { status: r.status };
+}
+/**
+ * Decide WHERE the job runs and create isolation if warranted:
+ *  - `--cwd <path>` → that path, as-is (caller owns isolation).
+ *  - else if the origin cwd is a git repo → a dedicated worktree at
+ *    `<originCwd>/.remote/jobs/<jobId>/wt` (so concurrent jobs never clobber the
+ *    working tree; the single-writer guard only protects the `.jsonl`).
+ *  - else → the origin cwd, as-is (non-repo: nothing to isolate cheaply).
+ *
+ * Throws if the worktree creation fails (we must NOT silently run in the shared
+ * cwd, which would defeat isolation).
+ */
+export function resolveJobCwd(originCwd, jobId, opts = {}) {
+    if (opts.explicitCwd !== undefined) {
+        return { runCwd: opts.explicitCwd, isolated: false };
+    }
+    const isRepo = (opts.isGitRepo ?? defaultIsGitRepo)(originCwd);
+    if (!isRepo)
+        return { runCwd: originCwd, isolated: false };
+    const dir = jobDir(originCwd, jobId);
+    const wt = join(dir, "wt");
+    (opts.mkdir ?? ((p) => mkdirSync(p, { recursive: true })))(dir);
+    const run = opts.runGit ?? defaultRunGit;
+    // Detached worktree on current HEAD — a fresh, independent file tree.
+    const r = run("git", ["worktree", "add", "--detach", wt, "HEAD"], originCwd);
+    if (r.status !== 0) {
+        throw new Error(`git worktree add failed (exit ${r.status ?? "?"}) for job ${jobId} — ` +
+            `pass --cwd to run in an explicit directory instead`);
+    }
+    return { runCwd: wt, isolated: true };
+}
+/**
+ * The DISPLAY state of a job, reconciling its persisted `jobState` with live
+ * liveness: a job still marked `running`/`pending` whose tmux session is gone
+ * has crashed/finished → show `failed` (P1 has no headless-success poll in the
+ * pure layer; the headless result.json is surfaced by `jobs status`). A job
+ * already in a terminal state keeps it. Pure, exported for tests.
+ */
+export function reconcileJobState(entry, live) {
+    const persisted = entry.jobState ?? "pending";
+    if (persisted === "done" || persisted === "failed")
+        return persisted;
+    // A `throttled` job has DELIBERATELY finished its run (its tmux session is gone
+    // — not live) and is waiting for the conductor's backoff resume. Its
+    // non-liveness is expected, NOT a crash → keep `throttled` (the resume path /
+    // attempt cap drives it terminal). Reliability slice 1.
+    if (persisted === "throttled")
+        return "throttled";
+    if (entry.endedAt)
+        return "failed";
+    if (!live)
+        return "failed";
+    return persisted;
+}
+/**
+ * P2 — CLUSTER reconciliation for REMOTE jobs (`kind:"remote"`). The registry
+ * reports a remote entry as `isLive` ALWAYS (it can't probe the cluster —
+ * registry.ts), so a delegated Pod that died/finished would stay "running"
+ * forever. Given the set of session ids the control-plane still lists
+ * (`listRemoteSessions`), this decides the transitions to apply: a remote job
+ * still in a non-terminal state whose `remoteId` is NOT in the live set has
+ * ended → `done` when it left a `result.json` (success/known exit), else
+ * `failed`. Jobs whose Pod is still listed, or already terminal, are untouched.
+ * Pure (the result read is injected), exported for tests.
+ */
+export function reconcileRemoteJobs(jobs, liveRemoteIds, readResult) {
+    const out = [];
+    for (const job of jobs) {
+        if (job.kind !== "remote" || job.role !== "job")
+            continue;
+        const state = job.jobState ?? "pending";
+        if (state === "done" || state === "failed")
+            continue;
+        // The Pod is still listed by the control-plane → leave the job alone.
+        if (job.remoteId !== undefined && liveRemoteIds.has(job.remoteId))
+            continue;
+        const result = readResult(job);
+        out.push({ id: job.id, to: result?.state ?? "failed" });
+    }
+    return out;
+}
+/**
+ * M2 — SWEEP stale jobs. A job stuck in a non-terminal `running` /
+ * `awaiting-decision`-ish state (we only persist `running`; awaiting-decision is
+ * a DISPLAY overlay, so a job awaiting a decision is `running` on disk) whose
+ * runtime signal was lost — NOT live AND no `result.json` AND older than
+ * `maxAgeMs` — would otherwise occupy a concurrency slot forever. This returns
+ * the ids to transition to `failed`. Bounds BOTH plain running jobs and ones
+ * showing awaiting-decision (same persisted `running`). `pending` jobs are NOT
+ * swept here (they were never launched — the conductor owns them). Terminal jobs
+ * are skipped. Pure (liveness + result presence injected), exported for tests.
+ */
+export function sweepStaleJobs(jobs, opts) {
+    const now = opts.nowMs ?? Date.now();
+    const out = [];
+    for (const job of jobs) {
+        if (job.role !== "job")
+            continue;
+        const state = job.jobState ?? "pending";
+        if (state !== "running")
+            continue; // pending/terminal are not our concern
+        if (opts.isJobLive(job))
+            continue; // still alive → leave it
+        if (opts.hasResult(job))
+            continue; // headless finished → reconcile handles it
+        // Age from last activity (lastSeenAt), falling back to enrolledAt. Use
+        // Number.isFinite (NOT `||`) so a legitimate epoch-0 timestamp isn't treated
+        // as missing.
+        const seen = Date.parse(job.lastSeenAt);
+        const enrolled = Date.parse(job.enrolledAt);
+        const last = Number.isFinite(seen)
+            ? seen
+            : Number.isFinite(enrolled)
+                ? enrolled
+                : now;
+        if (now - last < opts.maxAgeMs)
+            continue; // not stale yet
+        out.push(job.id);
+    }
+    return out;
+}
+/**
+ * Build `jobs ls` rows from job entries + an injectable liveness probe. Pure,
+ * exported for tests. `nowMs` makes the age deterministic.
+ */
+export function buildJobRows(jobs, isJobLive, nowMs = Date.now(), resolveAccountLabel) {
+    return jobs.map((e) => {
+        const state = reconcileJobState(e, isJobLive(e));
+        // A `throttled` row shows its backoff "retry in Xm" (derived from
+        // nextRetryAt) plus, when present, the matched signature. Reliability slice 1.
+        const note = state === "throttled"
+            ? `${throttleRetryLabel(e.throttle, nowMs)}` +
+                (e.throttle?.attempts !== undefined ? ` (try ${e.throttle.attempts}/${THROTTLE_MAX_ATTEMPTS})` : "")
+            : "";
+        const row = {
+            id: e.id,
+            type: e.tool,
+            state,
+            age: humanAge(Date.parse(e.enrolledAt), nowMs),
+            cwd: e.cwd,
+            note,
+        };
+        if (e.remoteId)
+            row.session = e.remoteId;
+        if (resolveAccountLabel) {
+            const label = resolveAccountLabel(e.id);
+            if (label !== undefined)
+                row.account = label;
+        }
+        return row;
+    });
+}
+/**
+ * M3 — advisory line for `jobs ls`: when there are QUEUED (`pending`) jobs but
+ * NO conductor running to drain them, warn the user (we do NOT self-heal — a
+ * one-shot `jobs ls` can't start the queue; only `jobs conduct` does). Returns
+ * undefined when there's nothing to warn about. Pure (the conductor-liveness
+ * probe is passed in), exported for tests.
+ */
+export function conductorAdvisory(jobs, conductorRunning) {
+    if (conductorRunning)
+        return undefined;
+    const pending = jobs.filter((j) => j.role === "job" && (j.jobState ?? "pending") === "pending").length;
+    if (pending === 0)
+        return undefined;
+    return (`[remote] ${pending} pending job(s) and no active conductor — they will NOT start on their own. ` +
+        `Run: remote jobs conduct   (or  remote jobs conduct --watch <min>  to keep draining)`);
+}
+/** Render `jobs ls` rows as an aligned plain-text table. Pure. */
+export function renderJobsTable(rows) {
+    if (rows.length === 0)
+        return "(no delegated jobs)";
+    const cols = [
+        ["id", "ID"],
+        ["type", "TYPE"],
+        ["state", "STATE"],
+        ["age", "AGE"],
+        ["cwd", "CWD"],
+    ];
+    // Show SESSION for remote jobs so `remote attach <session>` is copy-paste-ready.
+    if (rows.some((r) => r.session !== undefined))
+        cols.push(["session", "SESSION"]);
+    // Show ACCOUNT when any row has a pool binding.
+    if (rows.some((r) => r.account !== undefined))
+        cols.push(["account", "ACCOUNT"]);
+    // Only show the NOTE column when something needs it (a throttled "retry in Xm"
+    // hint) — keeps the common all-running table unchanged. Reliability slice 1.
+    if (rows.some((r) => r.note.length > 0))
+        cols.push(["note", "NOTE"]);
+    const widths = cols.map(([key, title]) => Math.max(title.length, ...rows.map((r) => String(r[key]).length)));
+    const line = (vals) => vals.map((v, i) => v.padEnd(widths[i])).join("  ").trimEnd();
+    const out = [line(cols.map(([, t]) => t))];
+    for (const r of rows)
+        out.push(line(cols.map(([k]) => String(r[k]))));
+    return out.join("\n");
+}
+//# sourceMappingURL=delegate.js.map