@prohost/cli 0.15.0 → 0.16.1

package/dist/host/service.js

@@ -0,0 +1,1027 @@
+/**
+ * `prohost host run` — one service for every AI employee placed on this computer.
+ *
+ * It holds one event stream, authenticated as the computer. For each
+ * `agent.run_requested` frame it:
+ *
+ * 1. verifies the signature and queues the run (see `scheduler.ts` for the order);
+ * 2. when a slot is free and the run's AI login has room, **claims** it
+ *    (`POST /v1/computers/{id}/runs/{run_id}/claim`) — exactly one computer
+ *    gets it; a 409 means another did, and it is skipped;
+ * 3. writes the employee's skills into its workspace and runs it the way
+ *    `agent run` would (`executeHostedRun`), as the employee, with the run's
+ *    own token for the heartbeat, tool events, reply and completion.
+ *
+ * Every 30 s (and on any change) it reports the machine — memory, CPU,
+ * pressure, battery, slots, AI logins, what's running — and the answer is the
+ * computer's config: capacity, holds, sharing between logins, placements.
+ *
+ * Terminal sessions (Code threads) are hosted by the same service with the
+ * computer credential, when hosting is on for the code-host home.
+ *
+ * Stopping drains: no new run starts, runs going finish, then it returns. A
+ * newer CLI on disk (an update) drains the same way and exits for the service
+ * manager to restart it.
+ */
+import { mkdirSync, rmSync, statSync } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { spawn as nodeSpawn } from 'node:child_process';
+import WebSocket from 'ws';
+import { completeRun, describeFailure } from '../agent/api.js';
+import { defaultCommandRunner } from '../agent/accounts.js';
+import { claudeQuotaProvider, codexQuotaProvider } from '../agent/runtime_status.js';
+import { EVENT_AGENT_RUN_REQUESTED, eventData, heartbeatPathFor, parseRunRequest, } from '../agent/contract.js';
+import { ensureWorkspace, loadCredentials } from '../agent/credentials.js';
+import { HeartbeatLoop } from '../agent/heartbeat.js';
+import { executeHostedRun } from '../agent/run.js';
+import { SessionStore, sessionKeyFor } from '../agent/sessionstore.js';
+import { verifySignature } from '../agent/signature.js';
+import { CODE_THREAD_FOLDERS_CAPABILITY, CODE_THREAD_SKIP_PERMISSIONS_CAPABILITY, CODE_THREADS_CAPABILITY, CodeHost, EVENT_CODE_THREAD_APPROVAL_DECIDED, EVENT_CODE_THREAD_INPUT_REQUESTED, EVENT_CODE_THREAD_START_REQUESTED, REMOTE_TERMINAL_CAPABILITY, RefusedDeliveryError, } from '../codehost/host.js';
+import { HostStatsSampler } from '../codehost/host_stats.js';
+import { loadSettings as loadCodeHostSettings } from '../codehost/settings.js';
+import { isWebhookEvent } from '../forward.js';
+import { ServerLiveness, enableTcpKeepAlive, terminateSocket } from '../liveness.js';
+import { CLI_VERSION, USER_AGENT } from '../version.js';
+import { callJson, claimRun, eventsPath, statusPath } from './api.js';
+import { effectiveCapacity, holdReason, recommendCapacity } from './capacity.js';
+import { acquireHostLock, claudeTokenEnv, controlPath, employeeHome, hostActivityPath, readObject, statePath, writePrivateFile, writeSharedFile, } from './computer.js';
+import { EventJournal } from './events.js';
+import { SHARED_DIR_ENV } from './install.js';
+import { HostLedger } from './ledger.js';
+import { LoginPool, detectLogins } from './logins.js';
+import { PeakMemory } from './peak_memory.js';
+import { HostScheduler, laneOf, someoneWaiting } from './scheduler.js';
+import { INTERRUPTED_NOTE, copySession } from './session_move.js';
+import { writeSkills } from './skills.js';
+import { DEFAULT_CONFIG, buildStatusBody, localRunning, parseConfig, slotsFor, } from './status.js';
+/** Set by a dedicated install (`host install --dedicated`): reported as `dedicated` in the status. */
+export const DEDICATED_ENV = 'PROHOST_HOST_DEDICATED';
+/** What a terminal start is answered with while the computer is paused. */
+export const PAUSED_START_REASON = 'This computer is paused: it opens no new terminal sessions until it is resumed.';
+/** Declared in every status body: this is a `prohost host` service. */
+export const COMPUTER_HOST_CAPABILITY = 'computer_host_v1';
+export const STATUS_INTERVAL_MS = 30_000;
+/** Status on change goes out at most this often. */
+export const STATUS_MIN_SPACING_MS = 3_000;
+export const LOGIN_REFRESH_MS = 15 * 60_000;
+export const CONTROL_POLL_MS = 2_000;
+export const UPDATE_CHECK_MS = 60_000;
+/** A claim with no answer is tried again this many times before the run is left to the server. */
+export const MAX_CLAIM_ATTEMPTS = 5;
+/** How many times one run may move to another login after a usage limit. */
+export const MAX_LOGIN_MOVES = 3;
+/** Exit code that asks a service manager to restart (a newer CLI is installed). */
+export const RESTART_EXIT_CODE = 75;
+const ACK_ACCEPTED = 200;
+const ACK_IGNORED = 204;
+const ACK_REFUSED = 409;
+const ACK_REJECTED = 401;
+const ACK_RETRY = 503;
+const RECONNECT_MAX_DELAY_MS = 30_000;
+/** An employee id names a folder: nothing else gets in. */
+const SAFE_ID = /^[A-Za-z0-9-]{1,64}$/;
+function timestamp() {
+    return new Date().toISOString().slice(11, 19);
+}
+function defaultSleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/** The CLI's own entry file: when it changes on disk, an update was installed. */
+export function entrypointStamp(entry = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', 'index.js')) {
+    const read = () => {
+        try {
+            return statSync(entry).mtimeMs;
+        }
+        catch {
+            return undefined;
+        }
+    };
+    const initial = read();
+    return () => {
+        const now = read();
+        return initial !== undefined && now !== undefined && now !== initial;
+    };
+}
+export async function runHostService(options) {
+    // One service per home: a second would read the same ledger and report the
+    // first one's live runs as unfinished.
+    const lock = acquireHostLock(options.env ?? process.env);
+    if (!lock.ok) {
+        const fatal = `another prohost host service (pid ${lock.heldBy}) is already running for this computer`;
+        (options.log ?? ((line) => process.stdout.write(`${line}\n`)))(`${timestamp()} ✗ ${fatal}`);
+        return { restart: false, runsStarted: 0, runsSkipped: 0, fatal };
+    }
+    try {
+        return await serveHost(options);
+    }
+    finally {
+        lock.release();
+    }
+}
+async function serveHost(options) {
+    const env = options.env ?? process.env;
+    const log = options.log ?? ((line) => process.stdout.write(`${line}\n`));
+    const sleep = options.sleepImpl ?? defaultSleep;
+    const now = options.now ?? Date.now;
+    const computer = options.computer;
+    const machine = options.machine ?? os.hostname();
+    const execute = options.executeRun ?? executeHostedRun;
+    const Ctor = options.webSocketCtor ?? WebSocket;
+    const computerApi = { baseUrl: computer.base_url, apiKey: computer.api_key, fetchImpl: options.fetchImpl };
+    const result = { restart: false, runsStarted: 0, runsSkipped: 0 };
+    const ledger = HostLedger.load(env, log);
+    /** One session ledger per employee, shared by its runs. */
+    const sessionStores = new Map();
+    const sessionsFor = (agentId) => {
+        let store = sessionStores.get(agentId);
+        if (!store) {
+            store = SessionStore.load({ ...env, PROHOST_HOME: employeeHome(agentId, env) }, log);
+            sessionStores.set(agentId, store);
+        }
+        return store;
+    };
+    const pool = new LoginPool({ get: (lane) => ledger.conversationLogin(lane), set: (lane, key) => ledger.setConversationLogin(lane, key) }, now);
+    const scheduler = new HostScheduler();
+    const active = new Map();
+    const peak = options.peak ?? new PeakMemory();
+    const stats = options.stats ?? new HostStatsSampler({ slots: () => ({ busy: active.size, total: capacity() }) });
+    const hardware = options.hardware ?? { memoryGb: os.totalmem() / 1024 ** 3, cores: os.cpus().length };
+    const heartbeats = new HeartbeatLoop({ api: computerApi, log, intervalMs: options.heartbeatIntervalMs });
+    const background = [];
+    const announcedRests = new Map();
+    let config = { ...DEFAULT_CONFIG };
+    /** A dedicated service publishes its state here for the logged-in user, and reads `control.json` here too. */
+    const sharedDir = env[SHARED_DIR_ENV] || undefined;
+    const readPaused = () => readObject(controlPath(env))?.paused === true ||
+        (sharedDir !== undefined && readObject(path.join(sharedDir, 'control.json'))?.paused === true);
+    const journal = new EventJournal(env, now);
+    let localPaused = readPaused();
+    let held = null;
+    let online = false;
+    let draining = false;
+    let fatal;
+    let socket;
+    const recommended = () => recommendCapacity({ memoryGb: hardware.memoryGb, cores: hardware.cores, workload: options.workload });
+    function capacity() {
+        return effectiveCapacity(config.capacity, config.recommended_capacity ?? recommended());
+    }
+    log(`${timestamp()} ⚙ ${computer.name} — serving the AI employees placed here; ` +
+        'they run WITHOUT permission prompts (Claude --dangerously-skip-permissions, Codex --yolo). ' +
+        'Message approval rules still apply.');
+    /** The terminal-hosting home's own agent key (`host migrate`), if it has one. */
+    const agentKeyFallback = (() => {
+        if (!computer.code_host_home)
+            return undefined;
+        try {
+            return loadCredentials({ ...env, PROHOST_HOME: computer.code_host_home }).api_key;
+        }
+        catch {
+            return undefined;
+        }
+    })();
+    // ── Terminal sessions ────────────────────────────────────────────────────
+    const codeHostEnv = computer.code_host_home
+        ? { ...process.env, ...env, PROHOST_HOME: computer.code_host_home }
+        : { ...process.env, ...env };
+    const codeHost = options.codeHost !== undefined
+        ? (options.codeHost ?? undefined)
+        : loadCodeHostSettings(codeHostEnv).enabled
+            ? buildCodeHost()
+            : undefined;
+    function buildCodeHost() {
+        try {
+            return new CodeHost({
+                env: codeHostEnv,
+                baseUrl: computer.base_url,
+                apiKey: computer.api_key,
+                machineId: computer.machine_id,
+                machine,
+                clientVersion: CLI_VERSION,
+                accountId: computer.account.id,
+                repos: [],
+                cliEntrypoint: path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', 'index.js'),
+                log,
+                slots: () => ({ busy: active.size + terminals(), total: capacity() }),
+                // The adopted agent key, for a server that doesn't take the computer key on the relay yet.
+                ...(agentKeyFallback ? { fallbackApiKey: agentKeyFallback } : {}),
+                // Paused: no new terminal sessions (nobody types as a signed-out owner).
+                paused: () => localPaused || config.paused,
+                onRemoteChanged: () => statusChanged(),
+                onCodeHostChanged: () => statusChanged(),
+            });
+        }
+        catch (err) {
+            log(`${timestamp()} ! terminal sessions unavailable: ${err instanceof Error ? err.message : String(err)}`);
+            return undefined;
+        }
+    }
+    if (codeHost) {
+        await codeHost.start().catch((err) => {
+            log(`${timestamp()} ! terminal sessions did not start: ${err instanceof Error ? err.message : String(err)}`);
+        });
+    }
+    /** Open terminal sessions — counted even after terminal sessions are switched off: they still run. */
+    function terminals() {
+        return codeHost?.liveTerminals?.().length ?? 0;
+    }
+    function capabilities() {
+        const caps = [COMPUTER_HOST_CAPABILITY];
+        if (codeHost?.authorized() && config.terminal_sessions_enabled) {
+            caps.push(CODE_THREADS_CAPABILITY, CODE_THREAD_FOLDERS_CAPABILITY, codeHost.instanceCapability());
+            caps.push(...(codeHost.agentCapabilities?.() ?? []));
+            if (codeHost.remoteTerminalAllowed())
+                caps.push(REMOTE_TERMINAL_CAPABILITY, CODE_THREAD_SKIP_PERMISSIONS_CAPABILITY);
+        }
+        return caps;
+    }
+    // ── What is running ──────────────────────────────────────────────────────
+    function runningEntries() {
+        const runs = [...active.values()].map((run) => ({ ...run.entry, peak_mem_mb: peak.peakMb(run.entry.run_id) }));
+        const terms = (codeHost?.liveTerminals?.() ?? []).map((t) => ({
+            run_id: t.terminalId,
+            agent_id: t.agent,
+            conversation_key: `code_thread:${t.codeThreadId}`,
+            kind: 'terminal',
+            started_at: '',
+        }));
+        return [...runs, ...terms];
+    }
+    /** Runs being claimed right now, and the login each will run on. */
+    const claiming = new Map();
+    function runsOnLogin(accountKey) {
+        let n = 0;
+        for (const run of active.values())
+            if (run.accountKey === accountKey)
+                n += 1;
+        for (const key of claiming.values())
+            if (key === accountKey)
+                n += 1;
+        return n;
+    }
+    // ── Holds ────────────────────────────────────────────────────────────────
+    function updateHold() {
+        const latest = stats.latest();
+        const next = holdReason({
+            paused: localPaused || config.paused,
+            holdOnPressure: config.hold_on_pressure,
+            holdOnBattery: config.hold_on_battery,
+            memoryPressure: latest?.memoryPressure ?? null,
+            onBattery: latest?.onBattery ?? null,
+        });
+        if (next === held)
+            return;
+        if (held)
+            sendEvent({ kind: 'hold_ended', reason: held });
+        if (next)
+            sendEvent({ kind: 'hold_started', reason: next });
+        log(next ? `${timestamp()} ⏸ holding new work (${next})` : `${timestamp()} ▶ taking new work again`);
+        held = next;
+        statusChanged();
+        pump();
+    }
+    // ── Events and status ────────────────────────────────────────────────────
+    function sendEvent(event) {
+        // Kept locally too (24 h), for the desktop's notifications (`host status --json`).
+        journal.add({ ...event, computer_id: computer.computer_id, computer_name: config.name ?? computer.name });
+        if (sharedDir) {
+            try {
+                writeSharedFile(path.join(sharedDir, 'events.json'), { v: 1, events: journal.list() });
+            }
+            catch {
+                /* the shared copy is a convenience */
+            }
+        }
+        background.push(callJson(computerApi, eventsPath(computer.computer_id), event).then((response) => {
+            if (response.status < 200 || response.status >= 300) {
+                log(`${timestamp()} ! event ${String(event.kind)} not accepted (${response.status || response.error})`);
+            }
+        }));
+    }
+    let lastStatusAt = 0;
+    let statusTimer;
+    let statusInFlight;
+    let statusFailures = 0;
+    function statusChanged() {
+        writeLocalState();
+        if (statusTimer)
+            return;
+        const wait = Math.max(0, lastStatusAt + STATUS_MIN_SPACING_MS - now());
+        statusTimer = setTimeout(() => {
+            statusTimer = undefined;
+            void sendStatus();
+        }, wait);
+        statusTimer.unref?.();
+    }
+    async function sendStatus() {
+        if (statusInFlight)
+            return statusInFlight;
+        statusInFlight = (async () => {
+            lastStatusAt = now();
+            const body = buildStatusBody({
+                stats: stats.latest(),
+                slots: slotsFor(capacity(), active.size + terminals(), held),
+                logins: pool.all(),
+                conversationsPerLogin: ledger.conversationsPerLogin(),
+                running: runningEntries(),
+                cliVersion: CLI_VERSION,
+                capabilities: capabilities(),
+                recommendedCapacity: recommended(),
+                dedicated: env[DEDICATED_ENV] === '1',
+            });
+            const response = await callJson(computerApi, statusPath(computer.computer_id), body);
+            if (response.status >= 200 && response.status < 300) {
+                if (statusFailures > 0)
+                    log(`${timestamp()} ● status reaching ProhostAI again`);
+                statusFailures = 0;
+                const before = JSON.stringify(config);
+                config = parseConfig(response.body, config);
+                options.onConfig?.(config);
+                if (JSON.stringify(config) !== before) {
+                    updateHold();
+                    pump();
+                }
+                writeLocalState();
+                return;
+            }
+            if (response.status === 401 || response.status === 403 || response.status === 410) {
+                fatal = `ProhostAI refused this computer's key (HTTP ${response.status}) — it was removed or its key revoked. Add it again: prohost host pair`;
+                socket?.close();
+                return;
+            }
+            statusFailures += 1;
+            if (statusFailures === 1)
+                log(`${timestamp()} ! status not accepted (${response.status || response.error})`);
+        })().finally(() => {
+            statusInFlight = undefined;
+        });
+        return statusInFlight;
+    }
+    function writeLocalState() {
+        const running = runningEntries();
+        const state = {
+            v: 1,
+            pid: process.pid,
+            updatedAt: new Date(now()).toISOString(),
+            online,
+            paused: localPaused || config.paused,
+            draining,
+            slots: slotsFor(capacity(), active.size + terminals(), held),
+            capacity: capacity(),
+            recommended_capacity: config.recommended_capacity ?? recommended(),
+            running,
+            logins: [...pool.all()],
+            ...(config.name ? { name: config.name } : {}),
+        };
+        const activity = { v: 1, busy: running.length > 0, running: running.map(localRunning), updatedAt: state.updatedAt };
+        try {
+            writePrivateFile(statePath(env), state);
+            writePrivateFile(hostActivityPath(env), activity);
+        }
+        catch {
+            /* a read-only home costs the menu bar its numbers, not the runs */
+        }
+        if (sharedDir) {
+            // A dedicated service, for the person logged in: no login emails in this copy.
+            try {
+                writeSharedFile(path.join(sharedDir, 'state.json'), {
+                    ...state,
+                    home: env.PROHOST_HOME ?? null,
+                    computer: { id: computer.computer_id, name: config.name ?? computer.name, account: computer.account },
+                    logins: state.logins.map(({ email_local: _email, ...login }) => login),
+                });
+                writeSharedFile(path.join(sharedDir, 'activity.json'), activity);
+            }
+            catch {
+                /* the shared copy is a convenience */
+            }
+        }
+    }
+    // ── Logins ───────────────────────────────────────────────────────────────
+    const refreshLogins = options.detectLogins ??
+        (() => detectLogins({
+            env,
+            // A dedicated install's Claude token signs Claude in: its checks see it, nothing else does.
+            run: (command, args, runEnv) => (options.accountCommandRunner ?? defaultCommandRunner)(command, args, command === 'claude' ? { ...runEnv, ...claudeTokenEnv(env) } : runEnv),
+            includeEmail: true,
+            quota: (record) => {
+                const accountEnv = () => ({
+                    ...(record.dir ? (record.runtime === 'claude_code' ? { CLAUDE_CONFIG_DIR: record.dir } : { CODEX_HOME: record.dir }) : {}),
+                    ...(record.runtime === 'claude_code' ? claudeTokenEnv(env) : {}),
+                });
+                return record.runtime === 'claude_code'
+                    ? claudeQuotaProvider({ binary: 'claude', env: accountEnv })
+                    : codexQuotaProvider({ binary: 'codex', clientVersion: CLI_VERSION, env: accountEnv });
+            },
+        }));
+    async function loadLogins() {
+        try {
+            pool.update(await refreshLogins());
+        }
+        catch (err) {
+            log(`${timestamp()} ! could not read the AI logins: ${err instanceof Error ? err.message : String(err)}`);
+        }
+        announceRests();
+        statusChanged();
+        pump();
+    }
+    function waitingFor(runtime) {
+        return scheduler.queued().filter((q) => runtimeOf(q.agentId, q.data) === runtime).length;
+    }
+    function announceRests() {
+        for (const login of pool.all()) {
+            const until = pool.restingUntil(login.account_key);
+            if (until === undefined) {
+                announcedRests.delete(login.account_key);
+                continue;
+            }
+            if (announcedRests.get(login.account_key) === until)
+                continue;
+            announcedRests.set(login.account_key, until);
+            sendEvent({
+                kind: 'login_resting',
+                runtime: login.runtime,
+                account_key: login.account_key,
+                label: login.label,
+                resets_at: new Date(until).toISOString(),
+                waiting: waitingFor(login.runtime),
+            });
+        }
+    }
+    // ── Scheduling ───────────────────────────────────────────────────────────
+    function runtimeOf(agentId, data) {
+        const placed = config.placements.find((p) => p.agent_id === agentId);
+        if (placed)
+            return placed.runtime;
+        return data.runtime === 'codex' ? 'codex' : 'claude_code';
+    }
+    function limits() {
+        return new Map(config.placements.map((p) => [p.agent_id, { maxConcurrent: p.max_concurrent, keepOneReady: p.keep_one_ready }]));
+    }
+    function conversationOf(lane) {
+        return lane.startsWith('run:') ? undefined : lane;
+    }
+    function pump() {
+        if (draining || fatal)
+            return;
+        if (!config.ai_employees_enabled) {
+            // Switched off for this computer: queued runs were never claimed, so
+            // ProhostAI hands them elsewhere; none of them starts here.
+            for (const queued of [...scheduler.queued()])
+                scheduler.remove(queued.runId);
+            return;
+        }
+        for (;;) {
+            const next = scheduler.take({ capacity: capacity(), held: held !== null, terminals: terminals(), employees: limits() }, (run) => {
+                if (run.notBefore > now())
+                    return false;
+                const pick = pool.pick(runtimeOf(run.agentId, run.data), conversationOf(run.lane), config.share_logins[runtimeOf(run.agentId, run.data)], runsOnLogin);
+                return pick.kind === 'use' || pick.kind === 'move';
+            });
+            if (!next)
+                break;
+            background.push(startRun(next));
+        }
+    }
+    // A queued run waiting on a backoff or a login reset is looked at again.
+    const pumpTimer = setInterval(() => pump(), 5_000);
+    pumpTimer.unref?.();
+    async function startRun(queued) {
+        const runtime = runtimeOf(queued.agentId, queued.data);
+        const conversation = conversationOf(queued.lane);
+        const pick = pool.pick(runtime, conversation, config.share_logins[runtime], runsOnLogin);
+        if (pick.kind !== 'use' && pick.kind !== 'move') {
+            scheduler.requeue(queued);
+            return;
+        }
+        const login = pick.kind === 'use' ? pick.login : pick.to;
+        if (!config.ai_employees_enabled || draining) {
+            scheduler.finish(queued.runId);
+            return;
+        }
+        // Counted on its login from now, so the next run picked spreads to another.
+        claiming.set(queued.runId, login.account_key);
+        const claim = await claimRun(computerApi, computer.computer_id, queued.runId).finally(() => {
+            claiming.delete(queued.runId);
+        });
+        if (claim.kind !== 'claimed') {
+            if (claim.kind === 'retry' && queued.claimAttempts + 1 < MAX_CLAIM_ATTEMPTS && !draining) {
+                scheduler.requeue({ ...queued, claimAttempts: queued.claimAttempts + 1, notBefore: now() + 2_000 * 2 ** queued.claimAttempts });
+            }
+            else {
+                scheduler.finish(queued.runId);
+                result.runsSkipped += 1;
+                const why = claim.kind === 'elsewhere'
+                    ? 'another computer has it'
+                    : claim.kind === 'gone'
+                        ? 'it is no longer waiting'
+                        : `the claim was refused (${claim.status}${'error' in claim && claim.error ? `: ${claim.error}` : ''})`;
+                log(`${timestamp()} ◦ run ${queued.runId.slice(0, 12)} skipped — ${why}`);
+            }
+            pump();
+            return;
+        }
+        const run = parseRunRequest({ agent_id: queued.agentId, ...claim.payload });
+        if (!run) {
+            scheduler.finish(queued.runId);
+            log(`${timestamp()} ✗ run ${queued.runId.slice(0, 12)} was claimed but its payload is unusable — not running it`);
+            pump();
+            return;
+        }
+        // The frame's employee (checked when it arrived) names the home, whatever the payload says.
+        const agentId = queued.agentId;
+        ledger.claimed({
+            run_id: run.run_id,
+            agent_id: agentId,
+            lane: queued.lane,
+            account_key: login.account_key,
+            completion_path: run.completion_path,
+            run_token: claim.runToken,
+        });
+        if (pick.kind === 'move')
+            moveSession(run, agentId, pick.from, pick.to);
+        pool.stick(conversation, login.account_key);
+        result.runsStarted += 1;
+        const controller = new AbortController();
+        const activeRun = {
+            controller,
+            accountKey: login.account_key,
+            entry: {
+                run_id: run.run_id,
+                agent_id: agentId,
+                ...(run.agent_name ? { agent_name: run.agent_name } : {}),
+                ...(typeof claim.payload.agent_avatar_url === 'string' ? { agent_avatar_url: claim.payload.agent_avatar_url } : {}),
+                ...(typeof claim.payload.title === 'string'
+                    ? { title: claim.payload.title }
+                    : run.trigger_text
+                        ? { title: run.trigger_text.slice(0, 80) }
+                        : {}),
+                ...(conversation ? { conversation_key: conversation } : {}),
+                kind: 'employee',
+                started_at: new Date(now()).toISOString(),
+                ...(run.trigger_user_id ? { started_by_user_id: run.trigger_user_id } : {}),
+            },
+        };
+        active.set(run.run_id, activeRun);
+        statusChanged();
+        let moved = pick.kind === 'move' ? { from: pick.from.account_key, to: pick.to.account_key } : undefined;
+        const runApi = {
+            baseUrl: computer.base_url,
+            apiKey: claim.runToken,
+            fetchImpl: options.fetchImpl,
+            completionExtra: () => ({
+                ...(peak.peakMb(run.run_id) !== undefined ? { peak_mem_mb: peak.peakMb(run.run_id) } : {}),
+                ...(moved ? { login_moved: moved } : {}),
+            }),
+        };
+        heartbeats.track(run.run_id, {
+            path: heartbeatPathFor(run),
+            api: runApi,
+            progress: () => activeRun.progress,
+            onCancelRequested: () => {
+                activeRun.disposition = 'cancelled';
+                controller.abort();
+            },
+            onResolved: () => {
+                activeRun.disposition = 'resolved';
+                controller.abort();
+            },
+        });
+        const stopHard = () => controller.abort();
+        options.hardStop?.addEventListener('abort', stopHard, { once: true });
+        const home = employeeHome(agentId, env);
+        const homeEnv = { ...env, PROHOST_HOME: home };
+        const runDir = path.join(home, 'runs', run.run_id);
+        let outcome = 'failed';
+        try {
+            const workspace = workspaceFor(agentId, homeEnv);
+            try {
+                const written = writeSkills(workspace, runtime, claim.skills);
+                for (const skipped of written.skipped)
+                    log(`    ! skill ${skipped.key} not written: ${skipped.reason}`);
+            }
+            catch (err) {
+                log(`    ! could not write skills for run ${run.run_id.slice(0, 12)}: ${String(err)}`);
+            }
+            const control = {
+                signal: controller.signal,
+                disposition: () => activeRun.disposition,
+                report: (activity) => {
+                    activeRun.progress = activity;
+                },
+            };
+            let current = login;
+            let note = pick.kind === 'move' ? INTERRUPTED_NOTE : undefined;
+            for (let moves = 0;; moves += 1) {
+                const executed = await execute(run, runOptions(run, claim.runToken, agentId, runtime, current, homeEnv, runDir, note), control, { lane: queued.lane, parallel: capacity() > 1 }, runApi);
+                outcome = executed.outcome;
+                if (executed.outcome !== 'usage_limited')
+                    break;
+                const until = pool.rest(current.account_key, executed.retryAfter ? Date.parse(executed.retryAfter) : undefined);
+                announceRests();
+                const next = config.share_logins[runtime] && moves < MAX_LOGIN_MOVES
+                    ? pool.pick(runtime, conversation, true, runsOnLogin)
+                    : undefined;
+                const target = next?.kind === 'move' ? next.to : next?.kind === 'use' && next.login.account_key !== current.account_key ? next.login : undefined;
+                if (!target) {
+                    outcome = await completeLimited(run, runApi, runtime, until);
+                    break;
+                }
+                moveSession(run, agentId, current, target);
+                moved = { from: moved?.from ?? current.account_key, to: target.account_key };
+                sendEvent({ kind: 'login_moved', run_id: run.run_id, from: current.account_key, to: target.account_key });
+                ledger.setAccount(run.run_id, target.account_key);
+                pool.stick(conversation, target.account_key);
+                activeRun.accountKey = target.account_key;
+                current = target;
+                note = INTERRUPTED_NOTE;
+                log(`${timestamp()} ↪ run ${run.run_id.slice(0, 12)} continues on ${target.label}`);
+            }
+        }
+        catch (err) {
+            // Never reported by the run itself: report it here, with its own token,
+            // and keep it in the ledger for the next start if that can't land.
+            const message = err instanceof Error ? err.message : String(err);
+            log(`${timestamp()} ✗ run ${run.run_id.slice(0, 12)} failed in the host: ${message}`);
+            outcome = await completeFailed(run, runApi, { ok: false, error: `the computer could not run this: ${message}` }, 'the failure');
+        }
+        finally {
+            options.hardStop?.removeEventListener('abort', stopHard);
+            heartbeats.untrack(run.run_id);
+            peak.finish(run.run_id);
+            rmSync(runDir, { recursive: true, force: true });
+            if (outcome !== 'unreported' && outcome !== 'usage_limited')
+                ledger.completed(run.run_id);
+            active.delete(run.run_id);
+            scheduler.finish(run.run_id);
+            statusChanged();
+            pump();
+        }
+    }
+    /** Where an employee's runs work: its adopted workspace, else its own under its host home. */
+    function workspaceFor(agentId, homeEnv) {
+        const adopted = options.workdirFor?.(agentId);
+        if (adopted) {
+            mkdirSync(adopted, { recursive: true, mode: 0o700 });
+            return adopted;
+        }
+        return ensureWorkspace(homeEnv);
+    }
+    function moveSession(run, agentId, from, to) {
+        const homeEnv = { ...env, PROHOST_HOME: employeeHome(agentId, env) };
+        const sessionId = sessionsFor(agentId).get(sessionKeyFor(run));
+        if (!sessionId)
+            return;
+        try {
+            const copied = copySession({
+                runtime: from.runtime,
+                targetRuntime: to.runtime,
+                fromDir: from.dir,
+                toDir: to.dir,
+                sessionId,
+                cwd: workspaceFor(agentId, homeEnv),
+            });
+            if (!copied.ok)
+                log(`    ! the conversation's session was not copied to ${to.label}: ${copied.reason}`);
+        }
+        catch (err) {
+            log(`    ! copying the session to ${to.label} failed: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    /** No other login has room: the run is reported failed with the limit, so ProhostAI asks again after the reset. */
+    function completeLimited(run, api, runtime, until) {
+        const name = runtime === 'codex' ? 'Codex' : 'Claude';
+        return completeFailed(run, api, { ok: false, error: `${name} usage limit reached on ${machine}`, usageLimit: { retryAfter: new Date(until).toISOString() } }, 'the usage limit');
+    }
+    /**
+     * Report a run failed with its own token, retrying what may be transient.
+     * `unreported` keeps it in the ledger, so the next start reports it.
+     */
+    async function completeFailed(run, api, outcome, what) {
+        for (let attempt = 0; attempt < 4; attempt += 1) {
+            const completed = await completeRun(api, run.completion_path, outcome);
+            if (completed.ok)
+                return 'failed';
+            if (completed.status !== 0 && completed.status !== 429 && completed.status < 500) {
+                // Refused, not lost: kept in the ledger like any completion that did
+                // not land (as `agent run` does), and tried once more on the next start.
+                log(`    ! could not report ${what} (${describeFailure(completed)})`);
+                return 'unreported';
+            }
+            await sleep(1_000 * 3 ** attempt);
+        }
+        return 'unreported';
+    }
+    function runOptions(run, runToken, agentId, runtime, login, homeEnv, runDir, resumeNote) {
+        const spawnImpl = options.spawnImpl;
+        return {
+            exec: runtime === 'codex' ? 'codex exec' : 'claude -p',
+            credentials: {
+                version: 2,
+                base_url: computer.base_url,
+                ws_url: computer.ws_url,
+                // The run's own token: it acts as the employee, for this run only.
+                api_key: runToken,
+                webhook_secret: computer.webhook_secret,
+                subscription_id: null,
+                stream_only: true,
+                mcp_url: computer.mcp_url,
+                events: [],
+                agent: { id: agentId, name: run.agent_name ?? 'AI employee', user_id: run.agent_user_id ?? '' },
+                paired_at: computer.paired_at,
+                account: login.label,
+                prohost_account: computer.account,
+            },
+            env: homeEnv,
+            cwd: workspaceFor(agentId, homeEnv),
+            machine,
+            timeoutMs: options.timeoutMs,
+            idleTimeoutMs: options.idleTimeoutMs,
+            onLine: log,
+            fetchImpl: options.fetchImpl,
+            sleepImpl: options.sleepImpl,
+            probeImpl: options.probeImpl,
+            accountCommandRunner: options.accountCommandRunner,
+            capabilityProbe: options.capabilityProbe,
+            mcpConfigFile: path.join(runDir, 'mcp.json'),
+            deferUsageLimit: true,
+            quietStartup: true,
+            replyConfidence: true,
+            // The Claude token (dedicated install) for Claude runs only.
+            ...(runtime === 'claude_code' ? { extraEnv: claudeTokenEnv(env) } : {}),
+            sessions: sessionsFor(agentId),
+            ...(resumeNote ? { resumeNote } : {}),
+            spawnImpl: ((command, args, spawnOptions) => {
+                const real = spawnImpl ?? nodeSpawn;
+                const child = real(command, args, spawnOptions);
+                peak.addProcess(run.run_id, child.pid);
+                return child;
+            }),
+        };
+    }
+    // ── Recovery: runs claimed before a restart and never reported ──────────
+    for (const stale of ledger.unfinished()) {
+        if (!stale.run_token || !stale.completion_path) {
+            ledger.completed(stale.run_id);
+            continue;
+        }
+        log(`${timestamp()} ↻ run ${stale.run_id.slice(0, 12)} was claimed before a restart; reporting it without re-running it`);
+        const api = { baseUrl: computer.base_url, apiKey: stale.run_token, fetchImpl: options.fetchImpl };
+        background.push(completeRun(api, stale.completion_path, { ok: false, error: 'the computer restarted before this run was reported' }).then((completed) => {
+            if (completed.ok || (completed.status >= 400 && completed.status < 500 && completed.status !== 429)) {
+                ledger.completed(stale.run_id);
+            }
+        }));
+    }
+    // ── Timers ──────────────────────────────────────────────────────────────
+    const statusLoop = setInterval(() => {
+        void stats.sample().then(() => {
+            updateHold();
+            void sendStatus();
+        });
+    }, options.statusIntervalMs ?? STATUS_INTERVAL_MS);
+    statusLoop.unref?.();
+    const loginLoop = setInterval(() => void loadLogins(), LOGIN_REFRESH_MS);
+    loginLoop.unref?.();
+    const controlLoop = setInterval(() => {
+        const paused = readPaused();
+        if (paused !== localPaused) {
+            localPaused = paused;
+            log(`${timestamp()} ${paused ? '⏸ paused on this computer' : '▶ resumed on this computer'}`);
+            updateHold();
+        }
+    }, CONTROL_POLL_MS);
+    controlLoop.unref?.();
+    const updateAvailable = options.updateAvailable ?? entrypointStamp();
+    const updateLoop = setInterval(() => {
+        if (!draining && updateAvailable()) {
+            log(`${timestamp()} ⟳ a newer prohost CLI is installed — finishing the runs going, then restarting`);
+            result.restart = true;
+            beginDrain();
+        }
+    }, options.updateCheckMs ?? UPDATE_CHECK_MS);
+    updateLoop.unref?.();
+    let drainResolve = () => { };
+    const drained = new Promise((resolve) => {
+        drainResolve = resolve;
+    });
+    function beginDrain() {
+        if (draining)
+            return;
+        draining = true;
+        // Queued runs were never claimed: ProhostAI hands them to another computer.
+        for (const queued of [...scheduler.queued()])
+            scheduler.remove(queued.runId);
+        statusChanged();
+        socket?.close();
+        drainResolve();
+    }
+    if (options.signal?.aborted)
+        beginDrain();
+    options.signal?.addEventListener('abort', () => beginDrain(), { once: true });
+    await stats.sample().catch(() => undefined);
+    updateHold();
+    void loadLogins();
+    await sendStatus();
+    // ── The event stream ─────────────────────────────────────────────────────
+    let connections = 0;
+    let attempt = 0;
+    while (!draining && !fatal && (options.maxConnections === undefined || connections < options.maxConnections)) {
+        connections += 1;
+        const authed = await connectOnce();
+        online = false;
+        writeLocalState();
+        if (draining || fatal)
+            break;
+        attempt = authed ? 1 : attempt + 1;
+        const delay = Math.floor(Math.random() * Math.min(RECONNECT_MAX_DELAY_MS, 1_000 * 2 ** Math.max(0, attempt - 1)));
+        log(`${timestamp()} … disconnected; reconnecting in ${(delay / 1000).toFixed(1)}s`);
+        await Promise.race([sleep(delay), drained]);
+    }
+    // Drain: nothing new starts; the runs going finish.
+    draining = true;
+    for (const queued of [...scheduler.queued()])
+        scheduler.remove(queued.runId);
+    await codeHost?.stop().catch(() => undefined);
+    // A real timer, never the injectable sleep: this waits on runs, not on a backoff.
+    while (active.size > 0)
+        await new Promise((resolve) => setTimeout(resolve, 50));
+    await Promise.all(background);
+    clearInterval(statusLoop);
+    clearInterval(loginLoop);
+    clearInterval(controlLoop);
+    clearInterval(updateLoop);
+    clearInterval(pumpTimer);
+    if (statusTimer)
+        clearTimeout(statusTimer);
+    heartbeats.stop();
+    peak.stop();
+    online = false;
+    writeLocalState();
+    if (fatal) {
+        log(`${timestamp()} ✗ ${fatal}`);
+        result.fatal = fatal;
+    }
+    return result;
+    function connectOnce() {
+        const ws = new Ctor(computer.ws_url, { headers: { 'User-Agent': USER_AGENT }, handshakeTimeout: 30_000 });
+        socket = ws;
+        let authed = false;
+        return new Promise((resolve) => {
+            const liveness = new ServerLiveness({
+                timeoutMs: options.livenessTimeoutMs,
+                onSilence: () => {
+                    log(`${timestamp()} … no word from ProhostAI — reconnecting`);
+                    terminateSocket(ws);
+                },
+            });
+            ws.on('open', () => {
+                enableTcpKeepAlive(ws);
+                liveness.start();
+                ws.send(JSON.stringify({
+                    type: 'auth',
+                    api_key: computer.api_key,
+                    machine,
+                    machine_id: computer.machine_id,
+                    client_version: CLI_VERSION,
+                    ...(computer.subscription_id ? { subscription_id: computer.subscription_id } : {}),
+                }));
+            });
+            ws.on('message', (raw) => {
+                let frame;
+                try {
+                    frame = JSON.parse(raw.toString());
+                }
+                catch {
+                    liveness.observe(undefined);
+                    return;
+                }
+                const f = (frame ?? {});
+                liveness.observe(f.type);
+                if (f.type === 'auth_ok') {
+                    authed = true;
+                    online = true;
+                    log(`${timestamp()} ● connected as ${computer.name} — waiting for work`);
+                    statusChanged();
+                    return;
+                }
+                if (f.type === 'auth_error') {
+                    fatal = `auth_error: ${String(f.reason ?? 'unknown')}. Add this computer again: prohost host pair`;
+                    ws.close();
+                    return;
+                }
+                if (f.type === 'ping') {
+                    ws.send(JSON.stringify({ type: 'pong' }));
+                    return;
+                }
+                if (!isWebhookEvent(frame))
+                    return;
+                const ack = (status, reason) => {
+                    try {
+                        ws.send(JSON.stringify({ type: 'ack', delivery_id: frame.delivery_id, status, ...(reason ? { reason } : {}) }));
+                    }
+                    catch {
+                        /* best-effort */
+                    }
+                };
+                const verified = options.allowUnverified === true || verifySignature(computer.webhook_secret, frame.payload, frame.signature);
+                if (frame.event === EVENT_AGENT_RUN_REQUESTED) {
+                    if (!verified) {
+                        log(`${timestamp()} ✗ rejected ${frame.event} — signature did not verify; not running it`);
+                        ack(ACK_REJECTED);
+                        return;
+                    }
+                    ack(...acceptRun(eventData(frame.payload)));
+                    return;
+                }
+                if (frame.event === EVENT_CODE_THREAD_START_REQUESTED ||
+                    frame.event === EVENT_CODE_THREAD_INPUT_REQUESTED ||
+                    frame.event === EVENT_CODE_THREAD_APPROVAL_DECIDED) {
+                    if (!codeHost || !config.terminal_sessions_enabled) {
+                        ack(ACK_IGNORED);
+                        return;
+                    }
+                    if (!verified) {
+                        log(`${timestamp()} ✗ rejected ${frame.event} — signature did not verify`);
+                        ack(ACK_REJECTED);
+                        return;
+                    }
+                    const data = eventData(frame.payload);
+                    if (frame.event === EVENT_CODE_THREAD_START_REQUESTED && (localPaused || config.paused) && codeHost.failStart) {
+                        // Paused: the start is reported failed, so the thread doesn't wait in `starting`.
+                        background.push(codeHost
+                            .failStart(data, PAUSED_START_REASON)
+                            .then((reported) => ack(reported ? ACK_ACCEPTED : ACK_RETRY))
+                            .catch(() => ack(ACK_RETRY)));
+                        return;
+                    }
+                    if (!codeHost.admits(frame.event, data)) {
+                        const refusal = codeHost.refusal(frame.event);
+                        if (refusal)
+                            ack(ACK_REFUSED, refusal);
+                        else
+                            ack(ACK_IGNORED);
+                        return;
+                    }
+                    const starting = frame.event === EVENT_CODE_THREAD_START_REQUESTED;
+                    const handled = starting
+                        ? codeHost.handleStart(data)
+                        : frame.event === EVENT_CODE_THREAD_INPUT_REQUESTED
+                            ? codeHost.handleInput(data, frame.delivery_id)
+                            : codeHost.handleApproval(data, frame.delivery_id);
+                    background.push(handled
+                        .then((ok) => {
+                        ack(starting || ok ? ACK_ACCEPTED : ACK_IGNORED);
+                        statusChanged();
+                    })
+                        .catch((err) => {
+                        if (err instanceof RefusedDeliveryError)
+                            ack(ACK_REFUSED, err.reason);
+                        else
+                            ack(ACK_RETRY);
+                    }));
+                    return;
+                }
+                ack(ACK_IGNORED);
+            });
+            ws.on('close', () => {
+                liveness.stop();
+                if (socket === ws)
+                    socket = undefined;
+                resolve(authed);
+            });
+            ws.on('error', (err) => {
+                if (!authed)
+                    log(`${timestamp()} ✗ connection error: ${err.message}`);
+            });
+        });
+    }
+    /** Queue a run frame; the answer is the ack. */
+    function acceptRun(data) {
+        const runId = typeof data.run_id === 'string' ? data.run_id : '';
+        const agentId = typeof data.agent_id === 'string' ? data.agent_id : '';
+        if (!runId || !SAFE_ID.test(agentId))
+            return [ACK_IGNORED];
+        if (typeof data.computer_id === 'string' && data.computer_id !== computer.computer_id)
+            return [ACK_IGNORED];
+        if (scheduler.has(runId) || active.has(runId) || ledger.get(runId))
+            return [ACK_ACCEPTED];
+        if (draining)
+            return [ACK_RETRY, 'draining'];
+        if (!config.ai_employees_enabled)
+            return [ACK_REFUSED, 'ai_employees_disabled'];
+        scheduler.add({
+            runId,
+            agentId,
+            lane: laneOf(data, runId),
+            waiting: someoneWaiting(data),
+            queuedAt: now(),
+            data,
+            claimAttempts: 0,
+            notBefore: 0,
+        });
+        pump();
+        return [ACK_ACCEPTED];
+    }
+}