@prohost/cli 0.8.3 → 0.9.0

package/dist/agent/run.js

@@ -10,9 +10,12 @@
  *
  * Two deliberate safety properties:
  *
- * * **Runs are serialized.** One local agent, one run at a time; concurrent
- *   frames queue. Agents hold working directories and rate limits, and running
- *   several at once is a good way to corrupt both.
+ * * **Runs are serialized by default.** One local agent, one run at a time;
+ *   concurrent frames queue. Agents hold working directories and rate limits,
+ *   and running several at once is a good way to corrupt both. `--concurrency`
+ *   raises the limit for runs in *different* conversations only — two turns of
+ *   one conversation never overlap (see `scheduler.ts`) — and `--repos` gives
+ *   each conversation its own git checkouts to do it in (see `worktrees.ts`).
  * * **Runs are deduped on ``run_id``.** Webhook deliveries retry, and each
  *   attempt is mirrored to this socket. Without dedupe a retry would execute
  *   the command a second time and post a second reply.
@@ -40,9 +43,11 @@ import { HeartbeatLoop } from './heartbeat.js';
 import { buildPrompt, replyFromStdout } from './prompt.js';
 import { NO_CAPABILITIES, capabilityProbe } from './capabilities.js';
 import { RunStore } from './runstore.js';
+import { MAX_CONCURRENCY, RunScheduler } from './scheduler.js';
 import { RUNTIME_STATUS_FEATURE, RuntimeStatusReporter, claudeQuotaProvider, codexConfiguredModel, codexQuotaProvider, modelFromExec, parseVersion, readCodexObservation, } from './runtime_status.js';
 import { SessionStore, sessionKeyFor } from './sessionstore.js';
 import { verifySignature } from './signature.js';
+import { RUN_REPOS_ENV_VAR, WorktreeManager } from './worktrees.js';
 import { isWebhookEvent } from '../forward.js';
 import { DEFAULT_WS_URL } from '../listen.js';
 import { ServerLiveness, enableTcpKeepAlive, seconds, terminateSocket } from '../liveness.js';
@@ -89,6 +94,15 @@ const RECONNECT_MAX_DELAY_MS = 30_000;
  * would hang with neither `open` nor `close` to move the loop along.
  */
 const HANDSHAKE_TIMEOUT_MS = 30_000;
+/**
+ * Set to `1` in the environment of every agent command this harness starts.
+ *
+ * Lets instructions the agent reads from disk tell a paired run — someone is
+ * waiting in a thread, and the run holds one of this agent's few slots — from
+ * the same file being read anywhere else. ProhostAI's own pull-request protocol
+ * uses it to stop a run from sitting on CI after the PR is open.
+ */
+export const PAIRED_RUN_ENV_VAR = 'PROHOST_PAIRED_RUN';
 /** Ack status codes reported back over the socket for each frame. */
 const ACK_ACCEPTED = 200;
 const ACK_IGNORED = 204;
@@ -228,6 +242,16 @@ function resolveWorkdir(options, log) {
         return undefined;
     }
 }
+/**
+ * The lane a run waits on: runs sharing one never overlap.
+ *
+ * The session key, because a session is exactly what two overlapping turns of
+ * one conversation would corrupt. A run with no surface resumes nothing and
+ * shares nothing, so it gets a lane of its own.
+ */
+function laneFor(run) {
+    return sessionKeyFor(run) ?? `run:${run.run_id}`;
+}
 /**
  * Assemble the per-process runtime: workspace, session ledger, and the
  * brain-specific MCP preparation used immediately before each run.
@@ -330,7 +354,11 @@ function buildRuntime(options, log) {
         if (!resolved.ok)
             return { mcpWired: false, error: resolved.error };
         const prepared = prepareMcp();
-        return { ...prepared, execEnv: { ...account.spawnEnv(), ...prepared.execEnv } };
+        return {
+            ...prepared,
+            execEnv: { ...account.spawnEnv(), ...prepared.execEnv },
+            accountLabel: resolved.record.label,
+        };
     };
     log(`${timestamp()} ⚙ account: ${account.label()}`);
     return {
@@ -579,7 +607,9 @@ async function runTurn(run, prompt, options, runtime, mcp, control, log,
  * Where the streaming brain's tool-call events go, if anywhere. Only the
  * Claude Code reader produces any; absent, they are simply never collected.
  */
-toolEvents) {
+toolEvents, 
+/** Harness-set variables for this run's command, over the brain's own. */
+runEnv) {
     // `--timeout` is a ceiling on the run, not on each attempt: the retry below
     // must not let one run take twice what the operator allowed. Both attempts
     // share one deadline, which costs the retry nothing in practice — a rejected
@@ -667,7 +697,7 @@ toolEvents) {
             timeoutMs: capped ? Math.max(1, remainingMs()) : 0,
             idleTimeoutMs: idleMs,
             cwd: runtime.workdir,
-            env: mcp.execEnv,
+            env: { ...mcp.execEnv, ...runEnv },
             spawnImpl: options.spawnImpl,
             cancelSignal: control.signal,
             onStdout: reader
@@ -842,11 +872,18 @@ toolEvents) {
  * Every path except a dry run — including every failure — calls the completion
  * endpoint before returning.
  */
-async function handleRun(run, options, runtime, api, log, sleep, control) {
+async function handleRun(run, options, runtime, api, log, sleep, control, isolation, 
+/** Prepared by the caller, which also hands the run's account back. */
+mcp) {
     const label = `run ${run.run_id.slice(0, 12)}`;
     const surface = run.conversation_id ?? run.kind ?? 'no conversation';
     log(`${timestamp()} ▶ ${label} started (${surface})`);
-    const mcp = runtime.prepareMcp();
+    // Before anything is spawned: the prompt names these paths. A dry run makes
+    // no writes, and a worktree is one.
+    const checkouts = isolation.worktrees.enabled && !options.dryRun ? await isolation.worktrees.prepare(isolation.lane) : undefined;
+    for (const failure of checkouts?.failures ?? []) {
+        log(`    ! no private checkout of ${failure.source} for ${label} (${failure.reason})`);
+    }
     // Only worth asking when the tools it describes are wired into this run.
     const capabilities = mcp.mcpWired ? await runtime.capabilities() : NO_CAPABILITIES;
     const prompt = buildPrompt(run, {
@@ -865,6 +902,9 @@ async function handleRun(run, options, runtime, api, log, sleep, control) {
         // The real numbers, not a vague "there is a limit" — see PromptContext.
         timeoutMs: options.timeoutMs ?? DEFAULT_EXEC_TIMEOUT_MS,
         idleTimeoutMs: options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS,
+        parallel: isolation.parallel,
+        checkouts: checkouts?.checkouts,
+        checkoutFailures: checkouts?.failures,
     });
     // Tool-call narration for the thread. A dry run makes no writes, so it gets
     // no path and every event is dropped; so does a server that advertised none.
@@ -874,7 +914,10 @@ async function handleRun(run, options, runtime, api, log, sleep, control) {
         log,
         label,
     });
-    const result = await runTurn(run, prompt, options, runtime, mcp, control, log, toolEvents);
+    const runEnv = { [PAIRED_RUN_ENV_VAR]: '1' };
+    if (checkouts && checkouts.checkouts.length > 0)
+        runEnv[RUN_REPOS_ENV_VAR] = checkouts.dir;
+    const result = await runTurn(run, prompt, options, runtime, mcp, control, log, toolEvents, runEnv);
     // Whatever the turn did, the timeline is whole BEFORE the reply is posted or
     // the run is closed — a completion that beats its own last tool row would
     // leave a `calling` row on a finished run. Never rejects, and after `stop`
@@ -1109,7 +1152,19 @@ export async function runAgentHarness(options) {
         }
         return true;
     };
-    let queue = Promise.resolve();
+    const concurrency = Math.min(MAX_CONCURRENCY, Math.max(1, Math.floor(options.concurrency ?? 1)));
+    const scheduler = new RunScheduler(concurrency);
+    const worktrees = new WorktreeManager({ repos: options.repos ?? [], env: options.env, log });
+    if (concurrency > 1) {
+        log(`${timestamp()} ⚙ up to ${concurrency} runs at once — runs in the same conversation still take turns` +
+            (worktrees.enabled
+                ? ''
+                : '. No --repos given: runs share every checkout on this machine, so name the ' +
+                    'repositories the agent edits to give each conversation its own'));
+    }
+    if (worktrees.enabled) {
+        log(`${timestamp()} ⚙ each conversation gets its own checkout of: ${(options.repos ?? []).join(', ')}`);
+    }
     let fatal;
     let attempt = 0;
     /** Stop tracking a run — it has reached a terminal state, one way or another. */
@@ -1195,7 +1250,7 @@ export async function runAgentHarness(options) {
     // Let an in-flight run finish reporting before we hand back metrics — and
     // any cancellation the heartbeat loop started and deliberately did not wait
     // for, so the harness never exits with a run half-closed.
-    await queue;
+    await scheduler.onIdle();
     status.detach();
     // Stopped first, so no further tick can start work after the wait below has
     // taken its snapshot of `background`.
@@ -1404,7 +1459,7 @@ export async function runAgentHarness(options) {
                     // completion. Retry only the completion: re-running the agent could
                     // repeat whatever it did with its credential the first time.
                     metrics.duplicates += 1;
-                    queue = queue.then(async () => {
+                    scheduler.submit(laneFor(run), async () => {
                         log(`${timestamp()} ↻ run ${run.run_id.slice(0, 12)} was started before but never reported; ` +
                             'reporting it now without re-running the agent');
                         const recovered = await completeWithRetry(api, previous.completion_path || run.completion_path, { ok: false, error: 'the harness restarted before this run was reported' }, log, sleep);
@@ -1434,9 +1489,11 @@ export async function runAgentHarness(options) {
                     },
                     onResolved: () => resolveAccepted(entry),
                 });
-                // Serialize execution but never block the socket reader: the ack and
-                // the heartbeat have to keep flowing while the agent thinks.
-                queue = queue.then(async () => {
+                // Queue execution but never block the socket reader: the ack and the
+                // heartbeat have to keep flowing while the agent thinks. The lane is
+                // what keeps two turns of one conversation from overlapping.
+                const lane = laneFor(run);
+                scheduler.submit(lane, async () => {
                     // Settled while it waited — cancelled, or closed server-side. Both
                     // were dealt with where they were decided; there is nothing to run
                     // and nothing left to report.
@@ -1465,17 +1522,32 @@ export async function runAgentHarness(options) {
                             entry.progress = activity;
                         },
                     };
-                    const outcome = await handleRun(run, options, runtime, api, log, sleep, control).catch((err) => {
+                    const isolation = { lane, parallel: concurrency > 1, worktrees };
+                    // Resolved here rather than inside `handleRun` so the account this
+                    // run is on can be handed back however the run ends.
+                    let mcp;
+                    try {
+                        mcp = runtime.prepareMcp();
+                    }
+                    catch (err) {
+                        // Fails the run closed, and — unlike a throw from here — still
+                        // reports it, so the server is not left waiting on it.
+                        mcp = { mcpWired: false, error: `could not prepare the run (${String(err)})` };
+                    }
+                    const outcome = await handleRun(run, options, runtime, api, log, sleep, control, isolation, mcp).catch((err) => {
                         log(`${timestamp()} ✗ run ${run.run_id} crashed the harness handler: ${String(err)}`);
                         return 'failed';
                     });
+                    if (mcp.accountLabel !== undefined)
+                        runtime.account?.finishSpawn(mcp.accountLabel);
                     // Held until here on purpose: heartbeats keep the row alive for the
                     // whole of `handleRun`, including the ten minutes of completion
                     // retries at the end of it.
                     release(run.run_id);
                     // An account picked mid-run was held back from the status frame
-                    // until this run (on the old account) was done; report it now.
-                    if (runtime.account?.changedSinceSpawn())
+                    // until this run (on the old account) was done; report it now. Asked
+                    // about *this* run's account: a newer run may already be on the pick.
+                    if (runtime.account?.changedSinceSpawn(mcp.accountLabel))
                         status.accountsChanged();
                     // Both terminal outcomes reached the completion endpoint; only then
                     // is the run settled and safe to skip on redelivery.

package/dist/agent/scheduler.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Which accepted runs execute now, and which wait.
+ *
+ * `agent run` used to chain every run onto one promise: one local agent, one
+ * run at a time. That is still the default, and with a limit of 1 this
+ * scheduler is exactly that chain — strict arrival order, nothing overlapping.
+ *
+ * `--concurrency N` lifts the limit without lifting the rule that actually
+ * protects an answer: **two runs on the same lane never overlap, and they start
+ * in the order they arrived.** A lane is a conversation (see `sessionKeyFor`).
+ * Each turn resumes the session the previous turn left, so running two turns of
+ * one thread side by side would have both resume the same session and the
+ * second answer without having seen the first. Runs on different lanes share
+ * nothing but the machine, and those are the ones allowed to overlap.
+ *
+ * Among the runs free to start, the oldest goes first. A lane that is busy does
+ * not hold up the lanes behind it — which is the whole point: a quick question
+ * in one thread no longer waits out a two-hour job in another.
+ */
+/** Most runs one harness will execute at once, whatever the flag says. */
+export declare const MAX_CONCURRENCY = 16;
+export declare class RunScheduler {
+    private readonly limit;
+    private readonly pending;
+    private readonly busy;
+    private waiters;
+    constructor(limit?: number);
+    /** How many tasks are executing right now. */
+    get active(): number;
+    /** How many tasks are waiting for a slot or for their lane. */
+    get queued(): number;
+    /**
+     * Queue a task on a lane. It starts as soon as a slot is free and no earlier
+     * task on the same lane is still running or waiting.
+     *
+     * The task must not reject: a rejection is swallowed so one broken run can
+     * never wedge its lane, but nothing reports it either — callers catch inside.
+     */
+    submit(lane: string, task: () => Promise<void>): void;
+    /** Resolves once nothing is executing and nothing is waiting. */
+    onIdle(): Promise<void>;
+    private pump;
+    private execute;
+}

package/dist/agent/scheduler.js

@@ -0,0 +1,93 @@
+/**
+ * Which accepted runs execute now, and which wait.
+ *
+ * `agent run` used to chain every run onto one promise: one local agent, one
+ * run at a time. That is still the default, and with a limit of 1 this
+ * scheduler is exactly that chain — strict arrival order, nothing overlapping.
+ *
+ * `--concurrency N` lifts the limit without lifting the rule that actually
+ * protects an answer: **two runs on the same lane never overlap, and they start
+ * in the order they arrived.** A lane is a conversation (see `sessionKeyFor`).
+ * Each turn resumes the session the previous turn left, so running two turns of
+ * one thread side by side would have both resume the same session and the
+ * second answer without having seen the first. Runs on different lanes share
+ * nothing but the machine, and those are the ones allowed to overlap.
+ *
+ * Among the runs free to start, the oldest goes first. A lane that is busy does
+ * not hold up the lanes behind it — which is the whole point: a quick question
+ * in one thread no longer waits out a two-hour job in another.
+ */
+/** Most runs one harness will execute at once, whatever the flag says. */
+export const MAX_CONCURRENCY = 16;
+export class RunScheduler {
+    limit;
+    pending = [];
+    busy = new Set();
+    waiters = [];
+    constructor(limit = 1) {
+        this.limit = limit;
+        if (!Number.isInteger(limit) || limit < 1) {
+            throw new RangeError(`concurrency must be a whole number of at least 1, got ${String(limit)}`);
+        }
+    }
+    /** How many tasks are executing right now. */
+    get active() {
+        return this.busy.size;
+    }
+    /** How many tasks are waiting for a slot or for their lane. */
+    get queued() {
+        return this.pending.length;
+    }
+    /**
+     * Queue a task on a lane. It starts as soon as a slot is free and no earlier
+     * task on the same lane is still running or waiting.
+     *
+     * The task must not reject: a rejection is swallowed so one broken run can
+     * never wedge its lane, but nothing reports it either — callers catch inside.
+     */
+    submit(lane, task) {
+        this.pending.push({ lane, task });
+        this.pump();
+    }
+    /** Resolves once nothing is executing and nothing is waiting. */
+    onIdle() {
+        if (this.busy.size === 0 && this.pending.length === 0)
+            return Promise.resolve();
+        return new Promise((resolve) => {
+            this.waiters.push(resolve);
+        });
+    }
+    pump() {
+        while (this.busy.size < this.limit) {
+            // First in arrival order whose lane is free. An earlier task on a busy
+            // lane is skipped, not waited for; a later task on that same lane is
+            // skipped with it, which is what keeps a lane in order.
+            const index = this.pending.findIndex((entry) => !this.busy.has(entry.lane));
+            if (index === -1)
+                break;
+            const [next] = this.pending.splice(index, 1);
+            if (!next)
+                break;
+            this.busy.add(next.lane);
+            void this.execute(next);
+        }
+        if (this.busy.size === 0 && this.pending.length === 0) {
+            const waiters = this.waiters;
+            this.waiters = [];
+            for (const resolve of waiters)
+                resolve();
+        }
+    }
+    async execute(entry) {
+        try {
+            await entry.task();
+        }
+        catch {
+            /* see `submit` — a task reports its own failure */
+        }
+        finally {
+            this.busy.delete(entry.lane);
+            this.pump();
+        }
+    }
+}

package/dist/agent/worktrees.d.ts

@@ -0,0 +1,136 @@
+/**
+ * A private checkout of each `--repos` repository, per conversation.
+ *
+ * A coding agent that opens pull requests does it in a git checkout, and until
+ * `--concurrency` there was only ever one run touching it. Two runs sharing one
+ * checkout switch each other's branch mid-edit and commit each other's
+ * half-finished files — the reason runs were serialized in the first place. So
+ * when the operator names the repositories the agent works on, each
+ * conversation gets its own `git worktree` of every one of them:
+ *
+ *     $PROHOST_HOME/worktrees/<conversation>/<repo-name>
+ *
+ * Per *conversation*, not per run, for the same reason sessions are: a
+ * follow-up in the thread ("address the review comments") resumes the session
+ * that made the branch, and must find that branch where it left it. Runs of one
+ * conversation never overlap (see `scheduler.ts`), so one checkout per
+ * conversation is exactly as isolated as one per run and far cheaper.
+ *
+ * The harness makes the worktree; the agent never needs `git worktree add`.
+ * The agent command's working directory does not change — it stays the
+ * workspace, because that is where its `.claude/` configuration lives and what
+ * its sessions are keyed on. The checkouts are named in the prompt and in
+ * `$PROHOST_RUN_REPOS`.
+ *
+ * Worktrees share the source repository's object store, so a branch committed
+ * in one survives the worktree being removed. That is what makes pruning safe:
+ * `git worktree remove` (never `--force`) refuses a checkout with uncommitted
+ * or untracked files, and those are the only things a worktree holds alone.
+ */
+/** Child-process env var naming the directory that holds this run's checkouts. */
+export declare const RUN_REPOS_ENV_VAR = "PROHOST_RUN_REPOS";
+/** A conversation's checkouts idle this long are removed, if clean. */
+export declare const WORKTREE_RETENTION_MS: number;
+export interface RunCheckout {
+    /** Directory name of the repository, e.g. `backend-service`. */
+    name: string;
+    /** This conversation's private worktree of it. */
+    path: string;
+    /** The operator's checkout it was made from — the one the agent must leave alone. */
+    source: string;
+}
+export interface CheckoutFailure {
+    source: string;
+    reason: string;
+}
+export interface PreparedCheckouts {
+    /** Directory holding the checkouts; exported as {@link RUN_REPOS_ENV_VAR}. */
+    dir: string;
+    checkouts: RunCheckout[];
+    failures: CheckoutFailure[];
+}
+export declare class ReposFlagError extends Error {
+}
+export declare function worktreesRoot(env?: NodeJS.ProcessEnv): string;
+/**
+ * Directory name for a lane (see `sessionKeyFor`).
+ *
+ * Readable enough to recognise in `ls`, and suffixed with a digest of the whole
+ * key because the readable part is lossy: two keys that differ only in
+ * punctuation, or in the agent they belong to, must not share a checkout.
+ */
+export declare function laneDirName(lane: string): string;
+/**
+ * Read `--repos a,b` into absolute paths, refusing what can never work.
+ *
+ * An empty list and two checkouts with one directory name are always errors.
+ * A path that is not a git checkout is an error only when `requireCheckouts`
+ * is set — which `install-daemon` does and `agent run` does not. At install
+ * time a mistyped path should stop the install; at run time the same check
+ * would make a daemon whose repository was moved, deleted or is on an
+ * unmounted volume exit before connecting, and the service manager restart it
+ * forever. A running harness instead reports the checkout it could not make on
+ * each run, and keeps answering.
+ *
+ * @throws ReposFlagError naming the entry that is wrong.
+ */
+export declare function parseReposFlag(raw: string, options?: {
+    requireCheckouts?: boolean;
+}): string[];
+/** Whether `repo` is the top of a git checkout (a clone, or a worktree of one). */
+export declare function isCheckout(repo: string): boolean;
+export interface WorktreeManagerOptions {
+    /** Absolute paths of the operator's checkouts. Empty disables the feature. */
+    repos: string[];
+    env?: NodeJS.ProcessEnv;
+    log?: (line: string) => void;
+    /** Clock seam for tests. */
+    now?: () => number;
+    retentionMs?: number;
+}
+/**
+ * Makes, reuses and prunes per-conversation worktrees for one harness.
+ *
+ * Every git mutation goes through one in-process queue. Lanes run side by
+ * side, and two `git worktree add` calls against the same repository race for
+ * the same administrative name; pruning, meanwhile, must never remove the
+ * directory a run arriving this instant is about to use.
+ */
+export declare class WorktreeManager {
+    private readonly repos;
+    private readonly env;
+    private readonly log;
+    private readonly now;
+    private readonly retentionMs;
+    private chain;
+    private lastPrunedAt;
+    constructor(options: WorktreeManagerOptions);
+    get enabled(): boolean;
+    private exclusive;
+    /**
+     * Ensure this lane has a worktree of every repository, and say where.
+     *
+     * Never rejects and never fails the run: most runs are questions that touch
+     * no repository at all, so a checkout that could not be made is reported in
+     * {@link PreparedCheckouts.failures} for the prompt to tell the agent about.
+     */
+    prepare(lane: string): Promise<PreparedCheckouts>;
+    /** The repository a checkout's history lives in, as one comparable path. */
+    private commonDir;
+    /**
+     * Keep an existing checkout — once it is proven to be a worktree of `source`.
+     *
+     * The directory is named after the repository's basename, so pointing
+     * `--repos` from `/old/app` to `/new/app` leaves every conversation with an
+     * `app` checkout of the *old* repository. Handing that to the agent as a
+     * checkout of the new one would have it commit to the wrong repository.
+     */
+    private reuse;
+    /** Returns why the worktree could not be made, or `undefined` when it was. */
+    private add;
+    private touch;
+    private pruneIfDue;
+    /** Remove idle conversations' checkouts, never the one about to be used. */
+    private prune;
+    private removeLane;
+}