@x-tech-laboratories/x-libs 0.13.0 → 0.14.1

package/dist/agent/long_runs.js

@@ -1,316 +0,0 @@
-// Long runs: tasks that keep going on their own, in their own process group, so they outlive the connection that
-// started them and the agent itself. Each writes its output and exit code into its own folder under the state
-// directory, which is all a later agent needs to report on it, follow it, or stop it.
-//
-// A run's folder holds `run.json` (its pid and when the process began), `out` (stdout and stderr together), `exit`
-// once it has ended, and `stopped` once it has been asked to stop. Its stdin, which may carry a secret, is handed over
-// through a pipe and never written down.
-// ── Imports ──────────────────────────────────────────────────────────────────
-import { spawn } from "node:child_process";
-import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
-import { open } from "node:fs/promises";
-import { constants } from "node:os";
-import { join } from "node:path";
-import { Readable } from "node:stream";
-import { kill_grace_in_milliseconds, signal_group } from "./commands.js";
-import { is_run_id } from "./protocol.js";
-// ── Settings ─────────────────────────────────────────────────────────────────
-// A poll returns at most this much output, and the next poll carries on from there.
-export const poll_limit_in_bytes = 1024 * 1024;
-// A followed run is read in pieces of this size.
-const tail_chunk_in_bytes = 64 * 1024;
-export class LongRunError extends Error {
-    constructor(message) {
-        super(message);
-        this.name = "LongRunError";
-    }
-}
-// ── Long runs ────────────────────────────────────────────────────────────────
-export class LongRuns {
-    directory;
-    grace;
-    tail_interval;
-    runner;
-    // The runs this agent started, whose exit it hears of directly.
-    children = new Map();
-    constructor(state_directory, runner, options = {}) {
-        this.directory = join(state_directory, "runs");
-        this.grace = options.kill_grace_in_milliseconds ?? kill_grace_in_milliseconds;
-        this.tail_interval = options.tail_interval_in_milliseconds ?? 250;
-        this.runner = runner;
-        mkdirSync(this.directory, { recursive: true, mode: 0o700 });
-    }
-    // ── Starting ─────────────────────────────────────────────────────────────
-    start(run_id, task, stdin) {
-        const folder = this.folder(run_id);
-        if (existsSync(folder))
-            throw new LongRunError(`A run named '${run_id}' already exists.`);
-        mkdirSync(folder, { mode: 0o700 });
-        writeFileSync(join(folder, "out"), "", { mode: 0o600 });
-        const launch = this.runner.launch(task, folder, stdin !== null);
-        const child = spawn(launch.program, launch.args, {
-            detached: true,
-            stdio: [stdin === null ? "ignore" : "pipe", "ignore", "ignore"],
-        });
-        if (child.pid === undefined) {
-            rmSync(folder, { recursive: true, force: true });
-            throw new LongRunError(`Could not start ${launch.program}.`);
-        }
-        if (stdin !== null) {
-            child.stdin.on("error", () => { });
-            child.stdin.end(stdin);
-        }
-        // A process killed before it could write its exit code is given the shell's code for that signal.
-        child.on("exit", (code, signal) => {
-            this.children.delete(run_id);
-            const exit_file = join(folder, "exit");
-            if (existsSync(folder) && !existsSync(exit_file)) {
-                const exit_code = code ?? 128 + (signal === null ? 0 : constants.signals[signal]);
-                writeFileSync(exit_file, `${exit_code}\n`);
-            }
-        });
-        child.on("error", () => { });
-        child.unref();
-        this.children.set(run_id, child);
-        const record = { run_id, pid: child.pid, start_ticks: start_ticks_of(child.pid), started_epoch: now_seconds() };
-        write_atomically(join(folder, "run.json"), JSON.stringify(record));
-        return this.describe(record);
-    }
-    // ── Reading ──────────────────────────────────────────────────────────────
-    // Every run in the state directory, as found now, so a new agent knows the ones an earlier one started.
-    list() {
-        const runs = [];
-        for (const name of readdirSync(this.directory)) {
-            const record = this.record(name);
-            if (record !== null)
-                runs.push(this.describe(record));
-        }
-        return runs.sort((first, second) => first.started_epoch - second.started_epoch || first.run_id.localeCompare(second.run_id));
-    }
-    get(run_id) {
-        return this.describe(this.require(run_id));
-    }
-    async poll(run_id, offset, limit = poll_limit_in_bytes) {
-        const record = this.require(run_id);
-        // The state is read before the output, so output read after an ending is known to be complete.
-        const run = this.describe(record);
-        const out = join(this.folder(run_id), "out");
-        const size = statSync(out).size;
-        const start = Math.max(0, Math.min(Math.trunc(offset), size));
-        let bytes = await read_range(out, start, Math.min(limit, size - start));
-        if (start + bytes.length < size)
-            bytes = whole_text(bytes);
-        return {
-            state: run.state,
-            exit_code: run.exit_code,
-            alive: run.state === "running",
-            size,
-            new_output: bytes.toString("utf8"),
-            next_offset: start + bytes.length,
-        };
-    }
-    // Follows a run's output from an offset as it is written, ending once the run has ended and all of it is read.
-    tail(run_id, offset) {
-        this.require(run_id);
-        const out = join(this.folder(run_id), "out");
-        let position = Math.max(0, Math.trunc(offset));
-        let following = true;
-        let reading = false;
-        let wait_timer;
-        let wake;
-        let settle;
-        const done = new Promise((resolve) => (settle = resolve));
-        const pause = () => new Promise((resolve) => {
-            wake = resolve;
-            wait_timer = setTimeout(resolve, this.tail_interval);
-        });
-        const pump = async () => {
-            if (reading || !following)
-                return;
-            reading = true;
-            try {
-                while (following) {
-                    const run = this.describe(this.require(run_id));
-                    const chunk = await read_range(out, position, tail_chunk_in_bytes);
-                    if (!following)
-                        return;
-                    if (chunk.length > 0) {
-                        position += chunk.length;
-                        if (!stdout.push(chunk))
-                            return;
-                        continue;
-                    }
-                    if (run.state !== "running") {
-                        following = false;
-                        stdout.push(null);
-                        settle({ exit_code: run.exit_code ?? -1, stderr: run.state === "exited" ? "" : `The run was ${run.state}.` });
-                        return;
-                    }
-                    await pause();
-                }
-            }
-            catch (error) {
-                following = false;
-                stdout.destroy();
-                settle({ exit_code: -1, stderr: `Could not follow the run: ${error.message}` });
-            }
-            finally {
-                reading = false;
-            }
-        };
-        const stdout = new Readable({
-            read: () => void pump(),
-            destroy: (error, callback) => {
-                if (following)
-                    settle({ exit_code: -1, stderr: "Stopped following the run." });
-                following = false;
-                clearTimeout(wait_timer);
-                wake?.();
-                callback(error);
-            },
-        });
-        return { stdout, done, stop: () => stdout.destroy() };
-    }
-    // ── Stopping ─────────────────────────────────────────────────────────────
-    // Asks the run's process group to stop, kills it after the grace period, and settles once it has ended.
-    async stop(run_id) {
-        const record = this.require(run_id);
-        if (this.describe(record).state !== "running")
-            return this.describe(record);
-        writeFileSync(join(this.folder(run_id), "stopped"), "");
-        signal_group(record.pid, "SIGTERM");
-        const deadline = Date.now() + this.grace;
-        while (this.is_alive(record)) {
-            if (Date.now() >= deadline) {
-                signal_group(record.pid, "SIGKILL");
-                break;
-            }
-            await new Promise((resolve) => setTimeout(resolve, 50));
-        }
-        while (this.is_alive(record))
-            await new Promise((resolve) => setTimeout(resolve, 50));
-        return this.describe(record);
-    }
-    // Removes an ended run's folder.
-    forget(run_id) {
-        const record = this.require(run_id);
-        if (this.describe(record).state === "running")
-            throw new LongRunError(`The run '${run_id}' is still running.`);
-        rmSync(this.folder(run_id), { recursive: true, force: true });
-    }
-    // Forgets runs that ended longer ago than the age given.
-    prune(max_age_in_seconds) {
-        const cutoff = Date.now() - max_age_in_seconds * 1000;
-        for (const run of this.list()) {
-            if (run.state === "running")
-                continue;
-            const exit_file = join(this.folder(run.run_id), "exit");
-            const ended = existsSync(exit_file) ? statSync(exit_file).mtimeMs : run.started_epoch * 1000;
-            if (ended < cutoff)
-                this.forget(run.run_id);
-        }
-    }
-    // ── Helpers ──────────────────────────────────────────────────────────────
-    folder(run_id) {
-        if (!is_run_id(run_id))
-            throw new LongRunError(`'${run_id}' is not a valid run id.`);
-        return join(this.directory, run_id);
-    }
-    record(run_id) {
-        if (!is_run_id(run_id))
-            return null;
-        try {
-            return JSON.parse(readFileSync(join(this.folder(run_id), "run.json"), "utf8"));
-        }
-        catch {
-            return null;
-        }
-    }
-    require(run_id) {
-        const record = this.record(run_id);
-        if (record === null)
-            throw new LongRunError(`No run named '${run_id}'.`);
-        return record;
-    }
-    is_alive(record) {
-        const child = this.children.get(record.run_id);
-        if (child !== undefined)
-            return child.exitCode === null && child.signalCode === null;
-        try {
-            process.kill(record.pid, 0);
-        }
-        catch (error) {
-            if (error.code !== "EPERM")
-                return false;
-        }
-        return record.start_ticks === null || start_ticks_of(record.pid) === record.start_ticks;
-    }
-    describe(record) {
-        const folder = this.folder(record.run_id);
-        const stopped = existsSync(join(folder, "stopped"));
-        const exit_code = read_exit_code(join(folder, "exit"));
-        let state;
-        if (exit_code !== null)
-            state = stopped ? "stopped" : "exited";
-        else if (this.is_alive(record))
-            state = "running";
-        else
-            state = stopped ? "stopped" : "lost";
-        return { run_id: record.run_id, pid: record.pid, started_epoch: record.started_epoch, state, exit_code };
-    }
-}
-// ── Files ────────────────────────────────────────────────────────────────────
-function read_exit_code(path) {
-    try {
-        const text = readFileSync(path, "utf8").trim();
-        return /^-?\d+$/.test(text) ? Number(text) : null;
-    }
-    catch {
-        return null;
-    }
-}
-async function read_range(path, offset, length) {
-    if (length <= 0)
-        return Buffer.alloc(0);
-    const handle = await open(path, "r");
-    try {
-        const buffer = Buffer.alloc(length);
-        const { bytesRead } = await handle.read(buffer, 0, length, offset);
-        return buffer.subarray(0, bytesRead);
-    }
-    finally {
-        await handle.close();
-    }
-}
-// Cuts a piece of output back to its last whole line, or failing that its last whole character.
-function whole_text(bytes) {
-    const last_newline = bytes.lastIndexOf(0x0a);
-    if (last_newline !== -1)
-        return bytes.subarray(0, last_newline + 1);
-    let start = bytes.length - 1;
-    while (start > 0 && start > bytes.length - 4 && (bytes[start] & 0xc0) === 0x80)
-        start--;
-    const lead = bytes[start];
-    const needed = lead >= 0xf0 ? 4 : lead >= 0xe0 ? 3 : lead >= 0xc0 ? 2 : 1;
-    return bytes.length - start < needed ? bytes.subarray(0, start) : bytes;
-}
-function write_atomically(path, text) {
-    writeFileSync(`${path}.part`, text, { mode: 0o600 });
-    renameSync(`${path}.part`, path);
-}
-// When the process began, from /proc; null where /proc is missing.
-function start_ticks_of(pid) {
-    try {
-        const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
-        // The fields after the command name, which may itself hold spaces and parentheses; start time is field 22.
-        const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" ");
-        const ticks = Number(fields[19]);
-        return Number.isFinite(ticks) ? ticks : null;
-    }
-    catch {
-        return null;
-    }
-}
-function now_seconds() {
-    return Math.floor(Date.now() / 1000);
-}

package/dist/agent/protocol.d.ts

@@ -1,148 +0,0 @@
-export interface AgentApp<Task> {
-    name: string;
-    protocol_version: number;
-    task_field: string;
-    is_task(value: unknown): value is Task;
-}
-export declare function shell_app(name: string, protocol_version?: number): AgentApp<string>;
-export interface RunResult {
-    exit_code: number;
-    stdout: string;
-    stderr: string;
-}
-export type LongRunState = "running" | "exited" | "stopped" | "lost";
-export interface LongRun {
-    run_id: string;
-    pid: number;
-    started_epoch: number;
-    state: LongRunState;
-    exit_code: number | null;
-}
-export interface LongRunPoll {
-    state: LongRunState;
-    exit_code: number | null;
-    alive: boolean;
-    size: number;
-    new_output: string;
-    next_offset: number;
-}
-export interface AppMessage {
-    type: string;
-    [field: string]: unknown;
-}
-export interface Requests<Task> {
-    run: {
-        body: {
-            task: Task;
-            stdin: string | null;
-            timeout_in_seconds: number;
-        };
-        reply: RunResult;
-    };
-    start: {
-        body: {
-            run_id: string;
-            task: Task;
-            stdin: string | null;
-        };
-        reply: LongRun;
-    };
-    poll: {
-        body: {
-            run_id: string;
-            offset: number;
-        };
-        reply: LongRunPoll;
-    };
-    stop: {
-        body: {
-            run_id: string;
-        };
-        reply: LongRun;
-    };
-    forget: {
-        body: {
-            run_id: string;
-        };
-        reply: null;
-    };
-    runs: {
-        body: Record<never, never>;
-        reply: LongRun[];
-    };
-}
-export type RequestType = keyof Requests<unknown>;
-export type RequestMessage<Task> = {
-    [K in RequestType]: {
-        type: K;
-        id: string;
-    } & Requests<Task>[K]["body"];
-}[RequestType];
-export type ServerMessage<Task> = RequestMessage<Task> | {
-    type: "stream";
-    id: string;
-    task: Task;
-    stdin: string | null;
-    window: number;
-} | {
-    type: "attach";
-    id: string;
-    run_id: string;
-    offset: number;
-    window: number;
-} | {
-    type: "credit";
-    id: string;
-    bytes: number;
-} | {
-    type: "cancel";
-    id: string;
-} | {
-    type: "ping";
-    epoch_ms: number;
-} | {
-    type: "pong";
-    epoch_ms: number;
-} | {
-    type: "challenge";
-    nonce: string;
-};
-export type AgentMessage = {
-    type: "hello";
-    protocol: number;
-    host: string;
-    agent_version: string;
-    runs: LongRun[];
-    signature?: string;
-} | {
-    type: "reply";
-    id: string;
-    value: unknown;
-} | {
-    type: "failed";
-    id: string;
-    error: string;
-} | {
-    type: "stream_end";
-    id: string;
-    exit_code: number;
-    stderr: string;
-} | {
-    type: "ping";
-    epoch_ms: number;
-} | {
-    type: "pong";
-    epoch_ms: number;
-};
-export declare function parse_server_message<Task>(text: string, app: AgentApp<Task>): ServerMessage<Task> | AppMessage | null;
-export declare function parse_agent_message(text: string, app: AgentApp<unknown>): AgentMessage | AppMessage | null;
-export declare function is_server_message<Task>(message: ServerMessage<Task> | AppMessage): message is ServerMessage<Task>;
-export declare function is_agent_message(message: AgentMessage | AppMessage): message is AgentMessage;
-export declare function encode_message<Task>(message: ServerMessage<Task> | AgentMessage | AppMessage, app: AgentApp<Task>): string;
-export declare function encode_chunk(id: string, data: Uint8Array): Buffer<ArrayBuffer>;
-export declare function decode_chunk(frame: Uint8Array): {
-    id: string;
-    data: Buffer;
-} | null;
-export declare function key_host_header(app_name: string): string;
-export declare function is_run_id(value: string): boolean;

package/dist/agent/protocol.js

@@ -1,131 +0,0 @@
-// The messages the server and a host's agent exchange over the agent's WebSocket, shared by both sides.
-//
-// Every message but a stream's output is one JSON text frame with a `type`. A request from the server carries an `id`
-// the agent answers with one `reply` or `failed` of the same id. A stream or an attach sends its output as binary
-// frames tagged with its id, only as far as the credit the server has granted, then one `stream_end`. Either side
-// sends `ping` while idle and drops a connection that has been silent too long.
-//
-// A run, a long run and a stream each carry the app's task: a shell command for one app, a typed operation for another.
-// The app names the field its task travels in and checks it, so each app keeps its own wire format and version.
-//
-// An agent logging in with its host's key, rather than a token, names the host in the upgrade; the server's first
-// message is then a `challenge`, and the agent's hello carries its signature of that nonce (see keys.ts).
-//
-// An app adds its own messages beside these: any other `type` is handed to the app, and one carrying a string `id` is
-// a request the app's handler answers like any other.
-// An app whose tasks are shell command lines, sent in a `command` field.
-export function shell_app(name, protocol_version = 1) {
-    return { name, protocol_version, task_field: "command", is_task: (value) => typeof value === "string" };
-}
-const id_fields = { id: "string" };
-const run_fields = { ...id_fields, run_id: "string" };
-const server_fields = {
-    run: { ...id_fields, task: "task", stdin: "string_or_null", timeout_in_seconds: "number" },
-    start: { ...run_fields, task: "task", stdin: "string_or_null" },
-    poll: { ...run_fields, offset: "number" },
-    stop: run_fields,
-    forget: run_fields,
-    runs: id_fields,
-    stream: { ...id_fields, task: "task", stdin: "string_or_null", window: "number" },
-    attach: { ...run_fields, offset: "number", window: "number" },
-    credit: { ...id_fields, bytes: "number" },
-    cancel: id_fields,
-    ping: { epoch_ms: "number" },
-    pong: { epoch_ms: "number" },
-    challenge: { nonce: "string" },
-};
-const agent_fields = {
-    hello: { protocol: "number", host: "string", agent_version: "string", runs: "array" },
-    reply: { ...id_fields, value: "any" },
-    failed: { ...id_fields, error: "string" },
-    stream_end: { ...id_fields, exit_code: "number", stderr: "string" },
-    ping: { epoch_ms: "number" },
-    pong: { epoch_ms: "number" },
-};
-function has_kind(value, kind, app) {
-    if (kind === "any")
-        return value !== undefined;
-    if (kind === "task")
-        return app.is_task(value);
-    if (kind === "array")
-        return Array.isArray(value);
-    if (kind === "string_or_null")
-        return value === null || typeof value === "string";
-    if (kind === "number")
-        return typeof value === "number" && Number.isFinite(value);
-    return typeof value === kind;
-}
-// A known message with every field it needs, an app's message for any other type, or null for one that is neither.
-function parse_with(text, fields, app) {
-    let message;
-    try {
-        message = JSON.parse(text);
-    }
-    catch {
-        return null;
-    }
-    if (typeof message !== "object" || message === null || Array.isArray(message))
-        return null;
-    const record = message;
-    if (typeof record.type !== "string" || record.type === "")
-        return null;
-    if (!Object.hasOwn(fields, record.type))
-        return record;
-    // The task arrives in the app's own field and is read as `task`.
-    const known_fields = fields[record.type];
-    if (Object.hasOwn(known_fields, "task") && app.task_field !== "task") {
-        record.task = record[app.task_field];
-        delete record[app.task_field];
-    }
-    for (const [name, kind] of Object.entries(known_fields)) {
-        if (!has_kind(record[name], kind, app))
-            return null;
-    }
-    return message;
-}
-// A message from the server, or null for one that is not a message at all.
-export function parse_server_message(text, app) {
-    return parse_with(text, server_fields, app);
-}
-// A message from an agent, or null for one that is not a message at all.
-export function parse_agent_message(text, app) {
-    return parse_with(text, agent_fields, app);
-}
-// Whether a parsed message is one of this protocol's own rather than the app's.
-export function is_server_message(message) {
-    return Object.hasOwn(server_fields, message.type);
-}
-export function is_agent_message(message) {
-    return Object.hasOwn(agent_fields, message.type);
-}
-// The message as one text frame, with any task in the app's own field.
-export function encode_message(message, app) {
-    if (!Object.hasOwn(server_fields, message.type) || !("task" in message))
-        return JSON.stringify(message);
-    const { task, ...rest } = message;
-    return JSON.stringify({ ...rest, [app.task_field]: task });
-}
-// ── Output frames ────────────────────────────────────────────────────────────
-// A binary frame is one byte giving the id's length, the id, then the output bytes.
-export function encode_chunk(id, data) {
-    const id_bytes = Buffer.from(id, "utf8");
-    if (id_bytes.length > 255)
-        throw new Error("A stream id is at most 255 bytes.");
-    return Buffer.concat([Buffer.from([id_bytes.length]), id_bytes, data]);
-}
-export function decode_chunk(frame) {
-    const bytes = Buffer.from(frame.buffer, frame.byteOffset, frame.byteLength);
-    if (bytes.length < 1 || bytes.length < 1 + bytes[0])
-        return null;
-    return { id: bytes.subarray(1, 1 + bytes[0]).toString("utf8"), data: bytes.subarray(1 + bytes[0]) };
-}
-// ── Key login ────────────────────────────────────────────────────────────────
-// The upgrade header an agent logging in with its key names its host in, such as "x-nix-agent-host".
-export function key_host_header(app_name) {
-    return `${app_name}-agent-host`;
-}
-// ── Long run ids ─────────────────────────────────────────────────────────────
-// A long run's id names its folder on the agent, so it is kept to safe characters.
-export function is_run_id(value) {
-    return /^[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}$/.test(value);
-}

package/dist/agent/session.d.ts

@@ -1,50 +0,0 @@
-import type { KeyObject } from "node:crypto";
-import type { LongRuns } from "./long_runs.ts";
-import { type AgentApp, type AgentMessage, type AppMessage } from "./protocol.ts";
-import type { TaskRunner } from "./tasks.ts";
-export type AppRequests = Record<string, (message: AppMessage) => unknown>;
-export interface SessionHooks<Task> {
-    requests?: AppRequests;
-    on_message?: (message: AppMessage, session: AgentSession<Task>) => void;
-    on_hello?: (session: AgentSession<Task>) => void;
-}
-export interface SessionOptions<Task> extends SessionHooks<Task> {
-    app: AgentApp<Task>;
-    runner: TaskRunner<Task>;
-    host: string;
-    agent_version: string;
-    heartbeat_in_milliseconds: number;
-    key?: KeyObject | null;
-}
-export interface MessageSocket {
-    readonly readyState: number;
-    send(data: string | Uint8Array<ArrayBuffer>): void;
-    close(code?: number, reason?: string): void;
-}
-export declare class AgentSession<Task> {
-    private readonly socket;
-    private readonly runs;
-    private readonly options;
-    private readonly pumps;
-    private last_heard;
-    private readonly heartbeat;
-    private closed;
-    private greeted;
-    constructor(socket: MessageSocket, runs: LongRuns<Task>, options: SessionOptions<Task>);
-    private say_hello;
-    receive(text: string): void;
-    close(): void;
-    send(message: AgentMessage | AppMessage): void;
-    send_chunk(id: string, data: Buffer): void;
-    end_stream(id: string, result: {
-        exit_code: number;
-        stderr: string;
-    }): void;
-    private handle;
-    private handle_app;
-    private start_pump;
-    private answer;
-    private open_stream;
-    private value_for;
-    private beat;
-}