@prohost/cli 0.2.3 → 0.4.0

package/dist/agent/exec.js

@@ -11,11 +11,57 @@
  * comes from the host's own flag — but event-derived text never reaches the
  * shell, so a crafted mention cannot inject a command.
  */
-import { spawn } from 'node:child_process';
+import { spawn, spawnSync } from 'node:child_process';
 /** Cap on captured output (in characters) so a runaway agent can't exhaust memory. */
 export const MAX_OUTPUT_CHARS = 512 * 1024;
-/** Grace period between SIGTERM and SIGKILL when a run overruns its timeout. */
+/** Grace period between SIGTERM and SIGKILL when a run is being stopped. */
 const KILL_GRACE_MS = 5_000;
+/** ``error`` on a result the caller stopped deliberately. */
+export const CANCELLED_ERROR = 'cancelled';
+/**
+ * What a timed-out run reports.
+ *
+ * Ends with the way out, because this string does not stay in `agent.log`: the
+ * server echoes a failed run's error into the thread, so the person who was
+ * waiting reads it. Told "timed out after 7200000ms" they learn only that
+ * something broke; told how to raise the ceiling they can fix it themselves.
+ * Kept to one line for the same reason.
+ */
+export function timeoutMessage(timeoutMs) {
+    return `timed out after ${timeoutMs}ms — raise the ceiling with --timeout <seconds> (0 = no limit)`;
+}
+/**
+ * Say a silence window the way a person would say it.
+ *
+ * The default is fifteen minutes and it wants to read as "15 minutes", not
+ * "900000ms": this string is shown to whoever was waiting on the agent, and a
+ * number they have to convert is a number they will misread.
+ */
+function renderWindow(ms) {
+    if (ms >= 60_000 && ms % 60_000 === 0) {
+        const minutes = ms / 60_000;
+        return `${minutes} minute${minutes === 1 ? '' : 's'}`;
+    }
+    const seconds = ms / 1_000;
+    const rendered = Number.isInteger(seconds)
+        ? String(seconds)
+        : seconds.toFixed(seconds < 1 ? 2 : 1);
+    return `${rendered} second${rendered === '1' ? '' : 's'}`;
+}
+/**
+ * What a run killed by the inactivity watchdog reports.
+ *
+ * Same job as {@link timeoutMessage}, and the same constraint: the server
+ * echoes a failed run's error into the thread, so this is read by the person
+ * who was waiting rather than by whoever tails `agent.log`. It has to say what
+ * happened ("nothing came out of it"), that this is an inference rather than a
+ * crash the agent reported, and how to change the judgement — in one line,
+ * because the notice it lands in has room for one.
+ */
+export function idleTimeoutMessage(idleTimeoutMs) {
+    return (`no output for ${renderWindow(idleTimeoutMs)} — treated as stalled; ` +
+        'tune with --idle-timeout <seconds> (0 = disabled)');
+}
 function clamp(text) {
     if (text.length <= MAX_OUTPUT_CHARS)
         return text;
@@ -24,9 +70,9 @@ function clamp(text) {
 /**
  * Execute the agent command with ``prompt`` on stdin and return its stdout.
  *
- * Never throws: a spawn failure, a non-zero exit, and a timeout all come back
- * as ``ok: false`` with the detail filled in, because the caller must report
- * *some* outcome to the server for every run.
+ * Never throws: a spawn failure, a non-zero exit, a timeout, and a cancellation
+ * all come back as ``ok: false`` with the detail filled in, because the caller
+ * must report *some* outcome to the server for every run.
  */
 export async function execAgent(options) {
     const spawnImpl = options.spawnImpl ?? spawn;
@@ -35,7 +81,25 @@ export async function execAgent(options) {
         let stdout = '';
         let stderr = '';
         let timedOut = false;
+        let idleTimedOut = false;
+        let cancelled = false;
         let settled = false;
+        // Nothing to stop if the stop already happened: a run cancelled while it
+        // sat in the queue must not spawn the agent at all.
+        if (options.cancelSignal?.aborted) {
+            resolve({
+                ok: false,
+                stdout: '',
+                stderr: '',
+                exitCode: null,
+                timedOut: false,
+                idleTimedOut: false,
+                cancelled: true,
+                error: CANCELLED_ERROR,
+                durationMs: Date.now() - started,
+            });
+            return;
+        }
         const spawnOptions = {
             shell: true,
             cwd: options.cwd,
@@ -58,27 +122,70 @@ export async function execAgent(options) {
                 stderr: '',
                 exitCode: null,
                 timedOut: false,
+                idleTimedOut: false,
+                cancelled: false,
                 error: err instanceof Error ? err.message : String(err),
                 durationMs: Date.now() - started,
             });
             return;
         }
+        let timer;
+        let idleTimer;
         let killTimer;
+        const idleMs = options.idleTimeoutMs ?? 0;
+        /**
+         * Why the run ended, when it ended because we ended it.
+         *
+         * Ordered by which decision was made first in practice: a deliberate stop
+         * outranks either clock, and the wall clock outranks the watchdog because
+         * `terminate` disarms the watchdog on its way out — so if both flags are
+         * somehow set, the ceiling is what actually fired.
+         */
+        const stopReason = () => cancelled
+            ? CANCELLED_ERROR
+            : timedOut
+                ? timeoutMessage(options.timeoutMs)
+                : idleTimedOut
+                    ? idleTimeoutMessage(idleMs)
+                    : undefined;
         const finish = (result) => {
             if (settled)
                 return;
             settled = true;
-            clearTimeout(timer);
+            if (timer !== undefined)
+                clearTimeout(timer);
+            if (idleTimer !== undefined)
+                clearTimeout(idleTimer);
             if (killTimer !== undefined)
                 clearTimeout(killTimer);
+            options.cancelSignal?.removeEventListener('abort', onAbort);
             resolve({ ...result, durationMs: Date.now() - started });
         };
         /**
-         * Signal the child's whole process group, falling back to the child alone
-         * when there is no group (Windows, or a spawn that never got a pid).
+         * Signal the child's whole process tree.
+         *
+         * POSIX: the child has its own process group (see `detached` above), so one
+         * signal reaches the shell and everything it started.
+         *
+         * Windows has no process groups to signal, and `shell: true` means the
+         * child is `cmd.exe` — killing it alone leaves the agent it launched
+         * running, which is a run reported as stopped that carries on acting with
+         * the operator's credentials. `taskkill /T` walks the tree instead, with
+         * `/F` as the SIGKILL half of the escalation. It is best-effort: if
+         * `taskkill` is missing or refuses, the fallback below still kills the
+         * shell, which is exactly what this did before.
          */
         const signalTree = (signal) => {
             const pid = child.pid;
+            if (pid !== undefined && process.platform === 'win32') {
+                try {
+                    const args = ['/pid', String(pid), '/T', ...(signal === 'SIGKILL' ? ['/F'] : [])];
+                    spawnSync('taskkill', args, { windowsHide: true });
+                }
+                catch {
+                    // Fall through — the child itself is still signalled below.
+                }
+            }
             try {
                 if (pid !== undefined && spawnOptions.detached)
                     process.kill(-pid, signal);
@@ -89,11 +196,33 @@ export async function execAgent(options) {
                 // ESRCH — the group is already gone, which is the outcome we wanted.
             }
         };
-        const timer = setTimeout(() => {
-            timedOut = true;
+        /**
+         * End the run: ask politely, then insist.
+         *
+         * The single path for both reasons a run is stopped from outside. They need
+         * identical handling — an agent that ignores SIGTERM ignores it whoever
+         * sent it — and having one caller escalate and the other not is how a
+         * cancelled run wedges the queue behind a child that never died.
+         */
+        const terminate = () => {
+            if (settled || killTimer !== undefined)
+                return;
+            // Both clocks are disarmed here rather than in `finish`, because a
+            // termination takes up to KILL_GRACE_MS and neither clock may re-decide
+            // the reason during it. A ceiling that lapses while a stalled run is
+            // being killed would otherwise overwrite "no output for 15 minutes" with
+            // "timed out", sending whoever reads it to the wrong flag — and the
+            // child's dying output would likewise re-arm a watchdog for a run that
+            // is already on its way out.
+            if (timer !== undefined) {
+                clearTimeout(timer);
+                timer = undefined;
+            }
+            if (idleTimer !== undefined) {
+                clearTimeout(idleTimer);
+                idleTimer = undefined;
+            }
             signalTree('SIGTERM');
-            // Escalate if the agent ignores SIGTERM, so a wedged child can't hold
-            // the harness open past its timeout.
             killTimer = setTimeout(() => {
                 signalTree('SIGKILL');
                 // A descendant can still hold the pipes open after the group is
@@ -104,16 +233,62 @@ export async function execAgent(options) {
                     stdout: clamp(stdout),
                     stderr: clamp(stderr),
                     exitCode: null,
-                    timedOut: true,
-                    error: `timed out after ${options.timeoutMs}ms`,
+                    timedOut,
+                    idleTimedOut,
+                    cancelled,
+                    error: stopReason(),
                 });
             }, KILL_GRACE_MS);
-        }, options.timeoutMs);
+        };
+        /**
+         * (Re)start the silence window.
+         *
+         * Armed once the child exists and again on every chunk it writes, so the
+         * window measures time since the last sign of life rather than time since
+         * the run began. Deliberately a no-op once a kill is under way: the run's
+         * fate is already decided, and the flag that explains it is already set.
+         */
+        const armIdleWatchdog = () => {
+            if (idleMs <= 0 || settled || killTimer !== undefined)
+                return;
+            if (idleTimer !== undefined)
+                clearTimeout(idleTimer);
+            idleTimer = setTimeout(() => {
+                idleTimedOut = true;
+                terminate();
+            }, idleMs);
+        };
+        function onAbort() {
+            cancelled = true;
+            terminate();
+        }
+        if (options.timeoutMs > 0) {
+            timer = setTimeout(() => {
+                timedOut = true;
+                terminate();
+            }, options.timeoutMs);
+        }
+        options.cancelSignal?.addEventListener('abort', onAbort, { once: true });
         child.stdout?.on('data', (chunk) => {
+            const text = chunk.toString();
+            armIdleWatchdog();
+            // Capture is bounded; the callback is not. A parser that stops being fed
+            // at 512 KB would report stale progress and salvage the wrong text for
+            // the rest of a long run — and the callback keeps nothing per chunk.
             if (stdout.length < MAX_OUTPUT_CHARS)
-                stdout += chunk.toString();
+                stdout += text;
+            try {
+                options.onStdout?.(text);
+            }
+            catch {
+                // A broken observer must not cost the run its output.
+            }
         });
         child.stderr?.on('data', (chunk) => {
+            // stderr counts as life exactly as much as stdout does. An agent that
+            // logs its progress to stderr and answers on stdout — several do — is
+            // plainly working, and a watchdog watching one stream would kill it.
+            armIdleWatchdog();
             if (stderr.length < MAX_OUTPUT_CHARS)
                 stderr += chunk.toString();
         });
@@ -124,19 +299,27 @@ export async function execAgent(options) {
                 stderr: clamp(stderr),
                 exitCode: null,
                 timedOut,
+                idleTimedOut,
+                cancelled,
                 error: err.message,
             });
         });
         child.on('close', (code) => {
             finish({
-                ok: !timedOut && code === 0,
+                ok: !timedOut && !idleTimedOut && !cancelled && code === 0,
                 stdout: clamp(stdout),
                 stderr: clamp(stderr),
                 exitCode: code,
                 timedOut,
-                error: timedOut ? `timed out after ${options.timeoutMs}ms` : undefined,
+                idleTimedOut,
+                cancelled,
+                error: stopReason(),
             });
         });
+        // Started here, not at the top of the function: the window measures the
+        // child's silence, and a child that has not been spawned yet — or that
+        // failed to spawn at all — has nothing to be silent about.
+        armIdleWatchdog();
         // stdin may already be closed if the command failed to start.
         try {
             child.stdin?.on('error', () => {

package/dist/agent/heartbeat.d.ts

@@ -0,0 +1,114 @@
+/**
+ * Keeps long runs alive, and is how a laptop hears "stop".
+ *
+ * The server holds a lease on every `running` agent-run row and reaps the ones
+ * that go quiet, because the alternative — trusting a laptop to always report —
+ * leaves rows spinning forever whenever someone closes a lid. A run that
+ * legitimately takes four hours therefore has to say so on a cadence.
+ *
+ * The response is the other half: this CLI dials *out* over a websocket and
+ * accepts no inbound connections, so the heartbeat's `cancel_requested` flag is
+ * the only channel by which "somebody pressed stop in the UI" can reach a
+ * machine behind NAT.
+ *
+ * Two properties this module is written around:
+ *
+ * * **Every accepted run beats, not just the executing one.** Runs are
+ *   serialized, so a run queued behind a four-hour job sits `running`
+ *   server-side for four hours without a single line of work being done on it.
+ *   Beating only for the executing run would have the server reap the queue.
+ * * **It never takes the daemon down.** A heartbeat is bookkeeping. Every
+ *   failure mode — an old server that 404s, a dropped network, a handler that
+ *   throws — costs at most the beat it happened on.
+ */
+import type { ApiOptions } from './api.js';
+/**
+ * How often each open run is reported.
+ *
+ * Well inside any sane server-side lease so a run survives losing a beat or
+ * two to a flaky link, and long enough that a queue of runs is a trickle of
+ * requests rather than a load source.
+ */
+export declare const HEARTBEAT_INTERVAL_MS = 60000;
+/** A cancelled timer. Returned by {@link Scheduler} implementations. */
+export interface ScheduledTick {
+    cancel(): void;
+}
+/**
+ * Deferred execution, injectable so tests drive the loop rather than wait it
+ * out. The default unrefs its timer: a heartbeat loop must never be the reason
+ * a process refuses to exit.
+ */
+export type Scheduler = (run: () => void, ms: number) => ScheduledTick;
+/** One run the loop is beating for. */
+export interface HeartbeatTarget {
+    /** Where the heartbeat POSTs — see `heartbeatPathFor`. */
+    path: string;
+    /**
+     * Latest one-line description of what the agent is doing, read fresh at each
+     * beat. Absent for a queued run, and for brains whose output we don't stream.
+     */
+    progress?: () => string | undefined;
+    /**
+     * The server wants this run stopped. Called at most once, and never again
+     * beaten for.
+     */
+    onCancelRequested: () => void | Promise<void>;
+    /**
+     * The server has already closed this run out (409) — reaped, or resolved by
+     * someone else. Called at most once. The work in flight is now pointless:
+     * whatever it produces has nowhere to go.
+     */
+    onResolved: () => void | Promise<void>;
+}
+export interface HeartbeatLoopOptions {
+    api: ApiOptions;
+    log: (line: string) => void;
+    /** Defaults to {@link HEARTBEAT_INTERVAL_MS}. */
+    intervalMs?: number;
+    /** Defaults to an unref'd `setTimeout`. */
+    scheduler?: Scheduler;
+}
+/**
+ * Beats for every open run on a fixed cadence.
+ *
+ * Idle when nothing is tracked — no timer is armed at all — so a harness that
+ * never receives a run makes no requests, and the first {@link track} starts
+ * the cadence.
+ */
+export declare class HeartbeatLoop {
+    private readonly options;
+    private readonly targets;
+    private readonly intervalMs;
+    private readonly scheduler;
+    private pending;
+    private stopped;
+    /** A tick in flight. Ticks never overlap; a slow one delays the next. */
+    private ticking;
+    constructor(options: HeartbeatLoopOptions);
+    /** Begin beating for a run. Idempotent on ``runId``. */
+    track(runId: string, target: HeartbeatTarget): void;
+    /** Stop beating for a run — it reached a terminal state, one way or another. */
+    untrack(runId: string): void;
+    /** Runs currently being beaten for. Exposed for assertions and logging. */
+    tracked(): string[];
+    /** Stop the loop for good. Called once, when the harness is shutting down. */
+    stop(): void;
+    /**
+     * Beat once for every tracked run.
+     *
+     * Public so tests can drive the cadence directly instead of waiting on a
+     * timer. Beats are issued concurrently: they are independent requests, and
+     * doing them in series would make one slow response delay every run behind it.
+     */
+    tick(): Promise<void>;
+    private arm;
+    private runTick;
+    /**
+     * One run's heartbeat, and what its answer means.
+     *
+     * Wrapped so that a throwing handler — or a fetch implementation that rejects
+     * rather than resolving — costs this run's beat and nothing else.
+     */
+    private beat;
+}

package/dist/agent/heartbeat.js

@@ -0,0 +1,165 @@
+/**
+ * Keeps long runs alive, and is how a laptop hears "stop".
+ *
+ * The server holds a lease on every `running` agent-run row and reaps the ones
+ * that go quiet, because the alternative — trusting a laptop to always report —
+ * leaves rows spinning forever whenever someone closes a lid. A run that
+ * legitimately takes four hours therefore has to say so on a cadence.
+ *
+ * The response is the other half: this CLI dials *out* over a websocket and
+ * accepts no inbound connections, so the heartbeat's `cancel_requested` flag is
+ * the only channel by which "somebody pressed stop in the UI" can reach a
+ * machine behind NAT.
+ *
+ * Two properties this module is written around:
+ *
+ * * **Every accepted run beats, not just the executing one.** Runs are
+ *   serialized, so a run queued behind a four-hour job sits `running`
+ *   server-side for four hours without a single line of work being done on it.
+ *   Beating only for the executing run would have the server reap the queue.
+ * * **It never takes the daemon down.** A heartbeat is bookkeeping. Every
+ *   failure mode — an old server that 404s, a dropped network, a handler that
+ *   throws — costs at most the beat it happened on.
+ */
+import { heartbeatRun } from './api.js';
+/**
+ * How often each open run is reported.
+ *
+ * Well inside any sane server-side lease so a run survives losing a beat or
+ * two to a flaky link, and long enough that a queue of runs is a trickle of
+ * requests rather than a load source.
+ */
+export const HEARTBEAT_INTERVAL_MS = 60_000;
+const defaultScheduler = (run, ms) => {
+    const timer = setTimeout(run, ms);
+    // `unref` exists on Node/Bun timers and not in the DOM lib's typing.
+    timer.unref?.();
+    return { cancel: () => clearTimeout(timer) };
+};
+/**
+ * Beats for every open run on a fixed cadence.
+ *
+ * Idle when nothing is tracked — no timer is armed at all — so a harness that
+ * never receives a run makes no requests, and the first {@link track} starts
+ * the cadence.
+ */
+export class HeartbeatLoop {
+    options;
+    targets = new Map();
+    intervalMs;
+    scheduler;
+    pending;
+    stopped = false;
+    /** A tick in flight. Ticks never overlap; a slow one delays the next. */
+    ticking = false;
+    constructor(options) {
+        this.options = options;
+        this.intervalMs = options.intervalMs ?? HEARTBEAT_INTERVAL_MS;
+        this.scheduler = options.scheduler ?? defaultScheduler;
+    }
+    /** Begin beating for a run. Idempotent on ``runId``. */
+    track(runId, target) {
+        if (this.stopped)
+            return;
+        this.targets.set(runId, { ...target, runId });
+        this.arm();
+    }
+    /** Stop beating for a run — it reached a terminal state, one way or another. */
+    untrack(runId) {
+        this.targets.delete(runId);
+    }
+    /** Runs currently being beaten for. Exposed for assertions and logging. */
+    tracked() {
+        return [...this.targets.keys()];
+    }
+    /** Stop the loop for good. Called once, when the harness is shutting down. */
+    stop() {
+        this.stopped = true;
+        this.targets.clear();
+        this.pending?.cancel();
+        this.pending = undefined;
+    }
+    /**
+     * Beat once for every tracked run.
+     *
+     * Public so tests can drive the cadence directly instead of waiting on a
+     * timer. Beats are issued concurrently: they are independent requests, and
+     * doing them in series would make one slow response delay every run behind it.
+     */
+    async tick() {
+        const targets = [...this.targets.values()];
+        await Promise.all(targets.map((target) => this.beat(target)));
+    }
+    arm() {
+        if (this.stopped || this.pending || this.ticking || this.targets.size === 0)
+            return;
+        this.pending = this.scheduler(() => {
+            this.pending = undefined;
+            void this.runTick();
+        }, this.intervalMs);
+    }
+    async runTick() {
+        this.ticking = true;
+        try {
+            await this.tick();
+        }
+        catch (err) {
+            // `tick` already contains every beat's failure; reaching here means the
+            // bookkeeping around them broke. Say so and keep the cadence — a harness
+            // that stops beating because of a bug here silently loses every long run.
+            this.options.log(`    ! heartbeat loop error: ${err instanceof Error ? err.message : String(err)}`);
+        }
+        finally {
+            this.ticking = false;
+            this.arm();
+        }
+    }
+    /**
+     * One run's heartbeat, and what its answer means.
+     *
+     * Wrapped so that a throwing handler — or a fetch implementation that rejects
+     * rather than resolving — costs this run's beat and nothing else.
+     */
+    async beat(target) {
+        try {
+            const outcome = await heartbeatRun(this.options.api, target.path, target.progress?.());
+            // The run may have finished while its own heartbeat was in flight; acting
+            // on a stale answer would cancel a run that is already reporting itself.
+            if (!this.targets.has(target.runId))
+                return;
+            if (outcome.kind === 'alive') {
+                if (!outcome.cancelRequested)
+                    return;
+                this.untrack(target.runId);
+                this.options.log(`    ⏹ run ${target.runId.slice(0, 12)} was cancelled from ProhostAI — stopping it`);
+                await target.onCancelRequested();
+                return;
+            }
+            if (outcome.kind === 'unknown') {
+                // Said once per run, then silence: on a server that predates
+                // heartbeats this is the answer for every run forever, and a line a
+                // minute per run would bury the log that operators actually read.
+                this.untrack(target.runId);
+                this.options.log(`    ◦ run ${target.runId.slice(0, 12)} cannot be kept alive (HTTP 404) — this ProhostAI ` +
+                    'server has no heartbeat endpoint, or no longer knows this run. Long runs may be ' +
+                    'reaped while they are still working.');
+                return;
+            }
+            if (outcome.kind === 'resolved') {
+                this.untrack(target.runId);
+                this.options.log(`    ⏹ run ${target.runId.slice(0, 12)} was already closed server-side — abandoning it`);
+                await target.onResolved();
+                return;
+            }
+            // Transient by assumption: keep the cadence rather than give up on the
+            // run. A run whose beats all fail is reaped server-side eventually, which
+            // is the correct end state for a machine that cannot reach the API.
+            this.options.log(`    ! heartbeat for run ${target.runId.slice(0, 12)} failed (HTTP ${outcome.status})` +
+                `${outcome.detail ? ` ${outcome.detail}` : ''}`);
+        }
+        catch (err) {
+            this.options.log(`    ! heartbeat for run ${target.runId.slice(0, 12)} threw: ` +
+                `${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+}

package/dist/agent/prompt.d.ts

@@ -39,6 +39,28 @@ export interface PromptContext {
      * decline work it is perfectly able to do.
      */
     toolsGated?: boolean;
+    /**
+     * How long this run may take before it is killed, in milliseconds. ``0`` is
+     * `--timeout 0`: no local cap at all.
+     *
+     * Threaded in from the harness rather than assumed, because the prompt used
+     * to say "this run has a hard time limit" without ever naming it — which an
+     * agent reads as "hurry", whatever the number happens to be. An agent given
+     * five minutes should triage; an agent given two hours should do the work.
+     * They cannot tell those apart from a sentence that says neither.
+     */
+    timeoutMs?: number;
+    /**
+     * How long the run may produce nothing at all before it is killed, in
+     * milliseconds. ``0`` is `--idle-timeout 0`: no watchdog.
+     *
+     * Threaded in for the same reason as {@link timeoutMs}, and it now carries
+     * most of the weight: with no wall-clock ceiling by default, this is usually
+     * the only local limit there is. It is also a limit an agent can act on,
+     * which a duration is not — "do not go completely silent for a quarter of an
+     * hour" is advice about how to work, not a request to hurry.
+     */
+    idleTimeoutMs?: number;
 }
 /** Build the stdin prompt for one run. */
 export declare function buildPrompt(run: AgentRunRequest, context: PromptContext): string;