@hyperdrive.bot/fleet-server 0.3.99

package/dist/scripts/supervisor.js

@@ -0,0 +1,345 @@
+import { fork, spawn } from "child_process";
+import { mkdirSync } from "node:fs";
+import path from "node:path";
+import { createStream as createRotatingFileStream } from "rotating-file-stream";
+const DEFAULT_RESTART_BACKOFF = { initialMs: 1000, maxMs: 30000, factor: 2 };
+const DEFAULT_RESTART_BUDGET = {
+    maxFailures: 5,
+    windowMs: 300000,
+    stabilityMs: 60000,
+};
+function describeExit(code, signal) {
+    return signal ?? (typeof code === "number" ? `code ${code}` : "unknown");
+}
+function parseLifecycleMessage(msg) {
+    if (typeof msg !== "object" || msg === null || !("type" in msg)) {
+        return null;
+    }
+    const type = msg.type;
+    if (type === "paseo:shutdown") {
+        const reason = msg.reason;
+        return {
+            type: "paseo:shutdown",
+            ...(typeof reason === "string" && reason.trim().length > 0 ? { reason } : {}),
+        };
+    }
+    if (type === "paseo:ready") {
+        const listen = msg.listen;
+        if (typeof listen !== "string" || listen.trim().length === 0) {
+            return null;
+        }
+        return { type: "paseo:ready", listen };
+    }
+    if (type === "paseo:restart") {
+        const reason = msg.reason;
+        return {
+            type: "paseo:restart",
+            ...(typeof reason === "string" && reason.trim().length > 0 ? { reason } : {}),
+        };
+    }
+    return null;
+}
+function toRotatingFileStreamSize(size) {
+    const trimmed = size.trim();
+    const match = trimmed.match(/^(\d+)\s*([bBkKmMgG])?$/);
+    if (!match) {
+        return trimmed;
+    }
+    const value = match[1];
+    const unit = (match[2] ?? "M").toUpperCase();
+    return `${value}${unit}`;
+}
+function createSupervisorLogStream(options) {
+    if (!options) {
+        return null;
+    }
+    mkdirSync(path.dirname(options.path), { recursive: true });
+    return createRotatingFileStream(path.basename(options.path), {
+        path: path.dirname(options.path),
+        size: toRotatingFileStreamSize(options.rotate.maxSize),
+        maxFiles: options.rotate.maxFiles,
+    });
+}
+export function runSupervisor(options) {
+    const restartOnCrash = options.restartOnCrash ?? false;
+    const workerArgs = options.workerArgs ?? process.argv.slice(2);
+    const workerEnv = options.workerEnv ?? process.env;
+    const workerExecArgv = options.workerExecArgv ?? ["--import", "tsx"];
+    const resolveWorkerSpawnSpec = options.resolveWorkerSpawnSpec;
+    const backoff = { ...DEFAULT_RESTART_BACKOFF, ...options.restartBackoff };
+    const budget = { ...DEFAULT_RESTART_BUDGET, ...options.restartBudget };
+    let child = null;
+    let restarting = false;
+    let shuttingDown = false;
+    let exiting = false;
+    let consecutiveFailures = 0;
+    let nextBackoffMs = 0;
+    let lastSpawnAt = 0;
+    let windowStartedAt = 0;
+    let pendingRestart = null;
+    const logStream = createSupervisorLogStream(options.logFile);
+    const clearPendingRestart = () => {
+        if (pendingRestart) {
+            clearTimeout(pendingRestart);
+            pendingRestart = null;
+        }
+    };
+    const writeDurableChunk = (chunk) => {
+        logStream?.write(chunk);
+    };
+    const writeLifecycleLog = (message, fields = {}) => {
+        writeDurableChunk(`${JSON.stringify({
+            level: "info",
+            time: new Date().toISOString(),
+            pid: process.pid,
+            name: options.name,
+            msg: message,
+            ...fields,
+        })}\n`);
+    };
+    const log = (message) => {
+        process.stderr.write(`[${options.name}] ${message}\n`);
+        writeLifecycleLog(message);
+    };
+    const closeLogStream = () => new Promise((resolve) => {
+        if (!logStream) {
+            resolve();
+            return;
+        }
+        logStream.end(resolve);
+    });
+    const exitSupervisor = (code) => {
+        if (exiting) {
+            return;
+        }
+        exiting = true;
+        clearPendingRestart();
+        Promise.resolve(options.onSupervisorExit?.())
+            .catch((error) => {
+            const message = error instanceof Error ? error.message : String(error);
+            log(`Supervisor exit cleanup failed: ${message}`);
+        })
+            .then(closeLogStream)
+            .finally(() => {
+            process.exit(code);
+        });
+    };
+    const spawnWorker = () => {
+        let workerEntry;
+        try {
+            // Resolve at spawn time so restarts pick up current filesystem state.
+            workerEntry = options.resolveWorkerEntry();
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            log(`Failed to resolve worker entry: ${message}`);
+            exitSupervisor(1);
+            return;
+        }
+        const spawnSpec = resolveWorkerSpawnSpec?.(workerEntry) ?? null;
+        lastSpawnAt = Date.now();
+        writeLifecycleLog("Spawning worker", { workerEntry });
+        if (spawnSpec) {
+            child = spawn(spawnSpec.command, spawnSpec.args, {
+                stdio: ["inherit", "pipe", "pipe", "ipc"],
+                env: spawnSpec.env ?? workerEnv,
+            });
+        }
+        else {
+            child = fork(workerEntry, workerArgs, {
+                stdio: ["inherit", "pipe", "pipe", "ipc"],
+                env: workerEnv,
+                execArgv: options.resolveWorkerExecArgv?.(workerEntry) ?? workerExecArgv,
+            });
+        }
+        const currentChild = child;
+        const heartbeat = setInterval(() => {
+            const message = { type: "paseo:supervisor-heartbeat" };
+            if (currentChild.connected) {
+                currentChild.send?.(message, (error) => {
+                    if (error) {
+                        writeLifecycleLog("Worker heartbeat IPC send failed", {
+                            error: error instanceof Error ? error.message : String(error),
+                        });
+                    }
+                });
+            }
+            else {
+                writeLifecycleLog("Worker heartbeat skipped because IPC channel is disconnected");
+            }
+        }, 1000);
+        heartbeat.unref();
+        child.on("disconnect", () => {
+            writeLifecycleLog("Worker IPC channel disconnected");
+        });
+        child.stdout?.on("data", (chunk) => {
+            process.stdout.write(chunk);
+            writeDurableChunk(chunk);
+        });
+        child.stderr?.on("data", (chunk) => {
+            process.stderr.write(chunk);
+            writeDurableChunk(chunk);
+        });
+        child.on("message", (msg) => {
+            const lifecycleMessage = parseLifecycleMessage(msg);
+            if (!lifecycleMessage) {
+                return;
+            }
+            if (lifecycleMessage.type === "paseo:ready") {
+                writeLifecycleLog("Worker ready", { listen: lifecycleMessage.listen });
+                Promise.resolve(options.onWorkerReady?.({ listen: lifecycleMessage.listen })).catch((error) => {
+                    const message = error instanceof Error ? error.message : String(error);
+                    log(`Worker ready callback failed: ${message}`);
+                });
+                return;
+            }
+            if (lifecycleMessage.type === "paseo:shutdown") {
+                const reason = lifecycleMessage.reason ?? "worker_requested_shutdown";
+                writeLifecycleLog("Worker requested shutdown", { reason });
+                requestShutdown(reason);
+                return;
+            }
+            const reason = lifecycleMessage.reason ?? "worker_requested_restart";
+            writeLifecycleLog("Worker requested restart", { reason });
+            requestRestart(reason);
+        });
+        child.on("close", (code, signal) => {
+            clearInterval(heartbeat);
+            const exitDescriptor = describeExit(code, signal);
+            writeLifecycleLog("Worker exited", { code, signal, exit: exitDescriptor });
+            if (shuttingDown) {
+                log(`Worker exited (${exitDescriptor}). Supervisor shutting down.`);
+                exitSupervisor(0);
+                return;
+            }
+            const crashed = restartOnCrash &&
+                ((code !== 0 && code !== null) || (signal !== null && signal !== "SIGTERM"));
+            // An intentional restart (paseo:restart, e.g. a self-update swap) is never
+            // delayed and never counts against the crash budget.
+            if (restarting) {
+                restarting = false;
+                log(`Worker exited (${exitDescriptor}). Restarting worker...`);
+                spawnWorker();
+                return;
+            }
+            if (crashed) {
+                const now = Date.now();
+                const uptimeMs = lastSpawnAt === 0 ? 0 : now - lastSpawnAt;
+                // A worker that stayed up long enough is treated as healthy: forget the
+                // previous crash streak so unrelated crashes months apart never add up.
+                if (uptimeMs >= budget.stabilityMs) {
+                    consecutiveFailures = 0;
+                    nextBackoffMs = 0;
+                    windowStartedAt = 0;
+                }
+                if (windowStartedAt === 0 || now - windowStartedAt > budget.windowMs) {
+                    windowStartedAt = now;
+                    consecutiveFailures = 0;
+                }
+                consecutiveFailures += 1;
+                if (consecutiveFailures >= budget.maxFailures) {
+                    writeLifecycleLog("Restart budget exceeded; not respawning worker", {
+                        level: "error",
+                        failures: consecutiveFailures,
+                        windowMs: budget.windowMs,
+                        exit: exitDescriptor,
+                        uptimeMs,
+                    });
+                    log(`Worker crashed ${consecutiveFailures}x within ${budget.windowMs}ms (${exitDescriptor}). ` +
+                        `Restart budget exceeded; supervisor exiting instead of looping.`);
+                    void Promise.resolve(options.onRestartBudgetExceeded?.({
+                        failures: consecutiveFailures,
+                        windowMs: budget.windowMs,
+                        lastExit: exitDescriptor,
+                    }))
+                        .catch((error) => {
+                        const message = error instanceof Error ? error.message : String(error);
+                        log(`Restart-budget handler failed: ${message}`);
+                    })
+                        .finally(() => {
+                        exitSupervisor(1);
+                    });
+                    return;
+                }
+                const delayMs = nextBackoffMs;
+                nextBackoffMs =
+                    nextBackoffMs === 0
+                        ? backoff.initialMs
+                        : Math.min(backoff.maxMs, Math.round(nextBackoffMs * backoff.factor));
+                writeLifecycleLog("Worker crashed; scheduling restart", {
+                    exit: exitDescriptor,
+                    failures: consecutiveFailures,
+                    delayMs,
+                    uptimeMs,
+                });
+                log(`Worker crashed (${exitDescriptor}). Restarting worker` +
+                    (delayMs > 0 ? ` in ${delayMs}ms` : "") +
+                    ` (failure ${consecutiveFailures}/${budget.maxFailures})...`);
+                if (delayMs === 0) {
+                    spawnWorker();
+                    return;
+                }
+                clearPendingRestart();
+                // Deliberately NOT unref'd: this timer is what keeps the supervisor alive
+                // while it waits out the backoff.
+                pendingRestart = setTimeout(() => {
+                    pendingRestart = null;
+                    if (shuttingDown || exiting) {
+                        return;
+                    }
+                    spawnWorker();
+                }, delayMs);
+                return;
+            }
+            log(`Worker exited (${exitDescriptor}). Supervisor exiting.`);
+            exitSupervisor(typeof code === "number" ? code : 1);
+        });
+    };
+    const signalWorker = (signal, reason) => {
+        if (!child) {
+            return;
+        }
+        writeLifecycleLog("Supervisor sending signal to worker", {
+            reason,
+            signal,
+            supervisorPid: process.pid,
+            workerPid: child.pid ?? null,
+        });
+        child.kill(signal);
+    };
+    const requestRestart = (reason) => {
+        if (!child || restarting || shuttingDown) {
+            return;
+        }
+        restarting = true;
+        writeLifecycleLog("Restart requested", { reason });
+        log(`${reason}. Stopping worker for restart...`);
+        signalWorker("SIGTERM", reason);
+    };
+    const requestShutdown = (reason) => {
+        if (shuttingDown) {
+            return;
+        }
+        shuttingDown = true;
+        restarting = false;
+        writeLifecycleLog("Supervisor shutdown requested", { reason });
+        log(`${reason}. Stopping worker...`);
+        // A shutdown arriving mid-backoff must exit now, not after the timer fires.
+        const wasWaitingToRestart = pendingRestart !== null;
+        clearPendingRestart();
+        if (!child || wasWaitingToRestart) {
+            exitSupervisor(0);
+            return;
+        }
+        signalWorker("SIGTERM", reason);
+    };
+    const forwardSignal = (signal) => {
+        requestShutdown(`supervisor_received_${signal}`);
+    };
+    process.on("SIGINT", () => forwardSignal("SIGINT"));
+    process.on("SIGTERM", () => forwardSignal("SIGTERM"));
+    process.stdout.write(`[${options.name}] ${options.startupMessage}\n`);
+    writeLifecycleLog(options.startupMessage);
+    spawnWorker();
+}
+//# sourceMappingURL=supervisor.js.map

package/dist/server/executable-resolution/executable-resolution.d.ts

@@ -0,0 +1,10 @@
+import { existsSync } from "node:fs";
+export { quoteWindowsArgument, quoteWindowsCommand } from "../utils/windows-command.js";
+export declare function probeExecutable(executablePath: string, timeoutMs?: number): Promise<boolean>;
+/**
+ * Check a literal executable path. PATH search is handled by findExecutable().
+ */
+export declare function executableExists(executablePath: string, exists?: typeof existsSync): string | null;
+export declare function findExecutable(name: string, probeTimeoutMs?: number): Promise<string | null>;
+export declare function isCommandAvailable(command: string): Promise<boolean>;
+//# sourceMappingURL=executable-resolution.d.ts.map

package/dist/server/executable-resolution/executable-resolution.js

@@ -0,0 +1,118 @@
+import { createRequire } from "node:module";
+import { existsSync } from "node:fs";
+import { execCommand } from "../utils/spawn.js";
+import { isWindowsCommandScript } from "../utils/windows-command.js";
+import { windowsExecutableResolution } from "./windows.js";
+export { quoteWindowsArgument, quoteWindowsCommand } from "../utils/windows-command.js";
+const require = createRequire(import.meta.url);
+const which = require("which");
+const PROBE_TIMEOUT_MS = 2000;
+function hasPathSeparator(value) {
+    return value.includes("/") || value.includes("\\");
+}
+async function enumerateCandidates(name) {
+    if (process.platform !== "win32" && existsSync("/usr/bin/which")) {
+        return enumerateCandidatesViaSystemWhich(name);
+    }
+    return enumerateCandidatesViaLibrary(name);
+}
+async function enumerateCandidatesViaSystemWhich(name) {
+    try {
+        const { stdout } = await execCommand("/usr/bin/which", ["-a", name], {
+            timeout: 3000,
+            killSignal: "SIGKILL",
+        });
+        return Array.from(new Set(stdout.trim().split("\n").filter(Boolean)));
+    }
+    catch {
+        return [];
+    }
+}
+async function enumerateCandidatesViaLibrary(name) {
+    let candidates;
+    try {
+        candidates = await which(name, { all: true });
+    }
+    catch (error) {
+        // `which` throws ENOENT when the command is absent from PATH.
+        if (error.code === "ENOENT") {
+            return [];
+        }
+        throw error;
+    }
+    const seen = new Set();
+    return candidates.filter((candidate) => {
+        if (seen.has(candidate)) {
+            return false;
+        }
+        seen.add(candidate);
+        return true;
+    });
+}
+export async function probeExecutable(executablePath, timeoutMs = PROBE_TIMEOUT_MS) {
+    try {
+        await execCommand(executablePath, ["--version"], {
+            timeout: timeoutMs,
+            killSignal: "SIGKILL",
+            maxBuffer: 64 * 1024,
+            shell: isWindowsCommandScript(executablePath),
+        });
+        return true;
+    }
+    catch (error) {
+        return classifyProbeError(error);
+    }
+}
+function classifyProbeError(error) {
+    const err = error;
+    if (err.killed) {
+        return true;
+    }
+    if (typeof err.code === "number") {
+        return true;
+    }
+    if (err.code === "ENOENT" ||
+        err.code === "EACCES" ||
+        err.code === "ENOEXEC" ||
+        err.code === "UNKNOWN") {
+        return false;
+    }
+    return false;
+}
+/**
+ * Check a literal executable path. PATH search is handled by findExecutable().
+ */
+export function executableExists(executablePath, exists = existsSync) {
+    if (process.platform === "win32") {
+        return windowsExecutableResolution.exists(executablePath, { exists });
+    }
+    return exists(executablePath) ? executablePath : null;
+}
+export async function findExecutable(name, probeTimeoutMs = PROBE_TIMEOUT_MS) {
+    const trimmed = name.trim();
+    if (!trimmed) {
+        return null;
+    }
+    if (process.platform === "win32") {
+        return windowsExecutableResolution.find(trimmed, {
+            enumeratePathCandidates: enumerateCandidates,
+            probeExecutable,
+            exists: existsSync,
+            probeTimeoutMs,
+        });
+    }
+    if (hasPathSeparator(trimmed)) {
+        return (await probeExecutable(trimmed, probeTimeoutMs)) ? trimmed : null;
+    }
+    const candidates = await enumerateCandidates(trimmed);
+    for (const candidate of candidates) {
+        if (await probeExecutable(candidate, probeTimeoutMs)) {
+            return candidate;
+        }
+    }
+    return null;
+}
+export async function isCommandAvailable(command) {
+    return (await findExecutable(command)) !== null;
+}
+//# sourceMappingURL=executable-resolution.js.map

package/dist/server/executable-resolution/windows.d.ts

@@ -0,0 +1,18 @@
+interface WindowsFindExecutableOptions {
+    enumeratePathCandidates: (name: string) => Promise<string[]>;
+    probeExecutable: (executablePath: string, timeoutMs: number) => Promise<boolean>;
+    exists: (path: string) => boolean;
+    localAppData?: string;
+    probeTimeoutMs: number;
+}
+interface WindowsExecutableExistsOptions {
+    exists: (path: string) => boolean;
+}
+declare function find(input: string, options: WindowsFindExecutableOptions): Promise<string | null>;
+declare function exists(executablePath: string, options: WindowsExecutableExistsOptions): string | null;
+export declare const windowsExecutableResolution: {
+    exists: typeof exists;
+    find: typeof find;
+};
+export {};
+//# sourceMappingURL=windows.d.ts.map

package/dist/server/executable-resolution/windows.js

@@ -0,0 +1,65 @@
+import { readdirSync } from "node:fs";
+import { extname, join } from "node:path";
+function hasPathSeparator(value) {
+    return value.includes("/") || value.includes("\\");
+}
+function enumerateLiteralPathCandidates(executablePath) {
+    if (extname(executablePath)) {
+        return [executablePath];
+    }
+    return [executablePath, `${executablePath}.exe`, `${executablePath}.cmd`];
+}
+function enumerateWingetPackageCandidates(name, localAppData) {
+    if (!localAppData) {
+        return [];
+    }
+    const wingetPackages = join(localAppData, "Microsoft", "WinGet", "Packages");
+    let packageDirs;
+    try {
+        packageDirs = readdirSync(wingetPackages, { withFileTypes: true })
+            .filter((entry) => entry.isDirectory())
+            .map((entry) => entry.name);
+    }
+    catch {
+        return [];
+    }
+    const exeName = `${name}.exe`;
+    return packageDirs.map((packageDir) => join(wingetPackages, packageDir, exeName));
+}
+async function find(input, options) {
+    if (hasPathSeparator(input)) {
+        return findFirstProbeable(enumerateLiteralPathCandidates(input), options);
+    }
+    const pathCandidates = await options.enumeratePathCandidates(input);
+    const wingetCandidates = enumerateWingetPackageCandidates(input, options.localAppData ?? process.env.LOCALAPPDATA).filter(options.exists);
+    return findFirstProbeable([...pathCandidates, ...wingetCandidates], options);
+}
+async function findFirstProbeable(candidates, options) {
+    const seen = new Set();
+    for (const candidate of candidates) {
+        if (seen.has(candidate)) {
+            continue;
+        }
+        seen.add(candidate);
+        if (!options.exists(candidate)) {
+            continue;
+        }
+        if (await options.probeExecutable(candidate, options.probeTimeoutMs)) {
+            return candidate;
+        }
+    }
+    return null;
+}
+function exists(executablePath, options) {
+    for (const candidate of enumerateLiteralPathCandidates(executablePath)) {
+        if (options.exists(candidate)) {
+            return candidate;
+        }
+    }
+    return null;
+}
+export const windowsExecutableResolution = {
+    exists,
+    find,
+};
+//# sourceMappingURL=windows.js.map

package/dist/server/extensions/daemon-backend.d.ts

@@ -0,0 +1,11 @@
+import type { Logger } from "pino";
+import type { AgentManager } from "../server/agent/agent-manager.js";
+import type { ExtensionHostBackend } from "./extension-host-manager.js";
+export interface DaemonExtensionBackendDeps {
+    agentManager: Pick<AgentManager, "listAgents" | "getAgent" | "archiveAgent">;
+    serverId: string;
+    paseoHome: string;
+    logger: Logger;
+}
+export declare function createDaemonExtensionBackend(deps: DaemonExtensionBackendDeps): ExtensionHostBackend;
+//# sourceMappingURL=daemon-backend.d.ts.map