@rome-os/node-core 0.1.1 → 0.1.3

package/dist/executor.js

@@ -1,77 +1,142 @@
 import { spawn } from "node:child_process";
 import { platform } from "node:os";
 import { actionError, isRecord } from "./actions.js";
-export function createExecutor(name, send, secrets = []) {
+import { decodeMeta, encodeMeta, FRAME_TYPE, frameFits, MAX_FRAME_BYTES, } from "./frame.js";
+import { parseTransferMessage, TRANSFER_VERSION, TransferHost } from "./transfer.js";
+function execInput(input) {
+    if (!isRecord(input) ||
+        typeof input.command !== "string" ||
+        !input.command ||
+        input.command.includes("\0") ||
+        (input.args !== undefined &&
+            (!Array.isArray(input.args) ||
+                !input.args.every((arg) => typeof arg === "string" && !arg.includes("\0")))) ||
+        (input.cwd !== undefined && (typeof input.cwd !== "string" || input.cwd.includes("\0"))))
+        return null;
+    return {
+        command: input.command,
+        args: (input.args ?? []),
+        cwd: input.cwd,
+    };
+}
+export function redactBytes(data, secrets) {
+    for (const secret of secrets) {
+        const needle = Buffer.from(secret);
+        if (!needle.byteLength)
+            continue;
+        for (let at = data.indexOf(needle); at !== -1; at = data.indexOf(needle, at + needle.length))
+            data.fill(0x2a, at, at + needle.byteLength);
+    }
+    return data;
+}
+export function createExecutor(name, send, secrets = [], sendFrame = () => false) {
     const children = new Set();
+    const transfers = new TransferHost();
     let generation = 0;
-    const handlers = {
-        "system.info": async () => ({
-            type: "response",
-            ok: true,
-            result: {
-                name,
-                platform: platform() === "darwin" ? "macos" : platform() === "win32" ? "windows" : platform(),
-                actions: ["system.info", "exec"],
-            },
-        }),
-        exec: async (input) => {
-            if (!isRecord(input) ||
-                typeof input.command !== "string" ||
-                !input.command ||
-                input.command.includes("\0") ||
-                (input.args !== undefined &&
-                    (!Array.isArray(input.args) ||
-                        !input.args.every((arg) => typeof arg === "string" && !arg.includes("\0")))) ||
-                (input.cwd !== undefined && (typeof input.cwd !== "string" || input.cwd.includes("\0"))))
-                return actionError("invalid_args", "exec requires command, optional string args[], and optional cwd.");
-            if (children.size >= 8)
-                return actionError("busy", "The computer is already running eight programs.");
-            const command = input.command;
-            const args = (input.args ?? []);
-            const cwd = input.cwd;
-            return new Promise((resolve) => {
-                const environment = Object.fromEntries(Object.entries(process.env).filter(([key]) => !/^(?:ROME_.*(?:TOKEN|SECRET|CREDENTIAL)|NODE_OPTIONS)$/i.test(key)));
-                let child;
-                try {
-                    child = spawn(command, args, {
-                        cwd,
-                        env: environment,
-                        shell: false,
-                        windowsHide: true,
-                        stdio: ["ignore", "pipe", "pipe"],
-                    });
-                }
-                catch {
-                    resolve(actionError("exec_failed", "Could not start the program."));
-                    return;
-                }
-                children.add(child);
-                const output = { stdout: [], stderr: [] };
-                for (const stream of ["stdout", "stderr"]) {
-                    child[stream]?.on("data", (chunk) => {
-                        output[stream].push(chunk);
-                    });
-                }
-                child.once("error", () => {
-                    children.delete(child);
-                    resolve(actionError("exec_failed", "Could not start the program. Check the executable and working directory."));
+    const info = () => ({
+        type: "response",
+        ok: true,
+        result: {
+            name,
+            platform: platform() === "darwin" ? "macos" : platform() === "win32" ? "windows" : platform(),
+            actions: ["system.info", "exec"],
+            frameVersion: 1,
+            transferVersion: TRANSFER_VERSION,
+        },
+    });
+    const invalidExec = () => actionError("invalid_args", "exec requires command, optional string args[], and optional cwd.");
+    function runProgram({ command, args, cwd }, stdin) {
+        if (children.size >= 8)
+            return Promise.resolve(actionError("busy", "The computer is already running eight programs."));
+        return new Promise((resolve) => {
+            const environment = Object.fromEntries(Object.entries(process.env).filter(([key]) => !/^(?:ROME_.*(?:TOKEN|SECRET|CREDENTIAL)|NODE_OPTIONS)$/i.test(key)));
+            let child;
+            try {
+                child = spawn(command, args, {
+                    cwd,
+                    env: environment,
+                    shell: false,
+                    windowsHide: true,
+                    stdio: [stdin ? "pipe" : "ignore", "pipe", "pipe"],
                 });
-                child.once("close", (exitCode, signal) => {
-                    children.delete(child);
-                    const redact = (chunks) => secrets.reduce((text, secret) => text.replaceAll(secret, "[redacted]"), Buffer.concat(chunks).toString("utf8"));
-                    resolve({
-                        type: "response",
-                        ok: true,
-                        result: {
-                            exitCode,
-                            signal,
-                            stdout: redact(output.stdout),
-                            stderr: redact(output.stderr),
-                            truncated: { stdout: false, stderr: false },
-                        },
-                    });
+            }
+            catch {
+                resolve(actionError("exec_failed", "Could not start the program."));
+                return;
+            }
+            children.add(child);
+            if (stdin && child.stdin) {
+                child.stdin.on("error", () => { });
+                child.stdin.end(stdin);
+            }
+            const output = { stdout: [], stderr: [] };
+            for (const stream of ["stdout", "stderr"]) {
+                child[stream]?.on("data", (chunk) => {
+                    output[stream].push(chunk);
+                });
+            }
+            child.once("error", () => {
+                children.delete(child);
+                resolve(actionError("exec_failed", "Could not start the program. Check the executable and working directory."));
+            });
+            child.once("close", (exitCode, signal) => {
+                children.delete(child);
+                resolve({
+                    exitCode,
+                    signal,
+                    stdout: Buffer.concat(output.stdout),
+                    stderr: Buffer.concat(output.stderr),
                 });
             });
+        });
+    }
+    const redactText = (data) => secrets.reduce((text, secret) => text.replaceAll(secret, "[redacted]"), data.toString("utf8"));
+    const isOutput = (value) => !("type" in value);
+    const handlers = {
+        "system.info": async () => info(),
+        exec: async (input) => {
+            const parsed = execInput(input);
+            if (!parsed)
+                return invalidExec();
+            const output = await runProgram(parsed);
+            if (!isOutput(output))
+                return output;
+            return {
+                type: "response",
+                ok: true,
+                result: {
+                    exitCode: output.exitCode,
+                    signal: output.signal,
+                    stdout: redactText(output.stdout),
+                    stderr: redactText(output.stderr),
+                    truncated: { stdout: false, stderr: false },
+                },
+            };
+        },
+    };
+    const empty = new Uint8Array();
+    const frameHandlers = {
+        "system.info": async () => ({ response: info(), body: empty }),
+        exec: async (input, body) => {
+            const parsed = execInput(input);
+            if (!parsed)
+                return { response: invalidExec(), body: empty };
+            const output = await runProgram(parsed, body);
+            if (!isOutput(output))
+                return { response: output, body: empty };
+            return {
+                response: {
+                    type: "response",
+                    ok: true,
+                    result: {
+                        exitCode: output.exitCode,
+                        signal: output.signal,
+                        stderr: redactText(output.stderr),
+                        truncated: { stdout: false, stderr: false },
+                    },
+                },
+                body: redactBytes(output.stdout, secrets),
+            };
         },
     };
     return {
@@ -90,8 +155,57 @@ export function createExecutor(name, send, secrets = []) {
             if (current === generation)
                 send(reply);
         },
+        async receiveFrame(frame) {
+            if (frame.type !== FRAME_TYPE.request)
+                return;
+            const request = decodeMeta(frame.meta);
+            const transfer = parseTransferMessage(request);
+            if (transfer) {
+                transfers.receive(frame.id, frame.peer, transfer, frame.body);
+                return;
+            }
+            if (!isRecord(request) || request.type !== "request" || typeof request.action !== "string")
+                return;
+            const current = generation;
+            if (request.action === "transfer.open") {
+                void transfers.open(frame.id, frame.peer, request.args, (meta, body = empty) => {
+                    if (current !== generation)
+                        return false;
+                    return sendFrame({
+                        type: FRAME_TYPE.response,
+                        id: frame.id,
+                        peer: frame.peer,
+                        meta: encodeMeta(meta),
+                        body,
+                    });
+                });
+                return;
+            }
+            const handler = Object.hasOwn(frameHandlers, request.action)
+                ? frameHandlers[request.action]
+                : undefined;
+            const result = handler
+                ? await handler(request.args, frame.body).catch(() => ({
+                    response: actionError("exec_failed", "The action failed."),
+                    body: empty,
+                }))
+                : {
+                    response: actionError("unsupported_action", "The computer does not support this action."),
+                    body: empty,
+                };
+            if (current !== generation)
+                return;
+            let meta = encodeMeta(result.response);
+            let body = result.body;
+            if (!frameFits(meta.byteLength, body.byteLength)) {
+                meta = encodeMeta(actionError("output_too_large", `The output does not fit in one ${MAX_FRAME_BYTES / 1024 / 1024} MiB frame and was discarded. The program ran. Use rome-node cp to copy large files.`));
+                body = empty;
+            }
+            sendFrame({ type: FRAME_TYPE.response, id: frame.id, peer: frame.peer, meta, body });
+        },
         disconnect() {
             generation++;
+            transfers.abortAll();
             for (const child of children) {
                 child.kill("SIGTERM");
                 const timer = setTimeout(() => {

package/dist/frame.d.ts

@@ -0,0 +1,24 @@
+export declare const FRAME_VERSION = 1;
+export declare const FRAME_TYPE: {
+    readonly request: 1;
+    readonly response: 2;
+    readonly routeError: 3;
+};
+export type FrameType = (typeof FRAME_TYPE)[keyof typeof FRAME_TYPE];
+export declare const FRAME_HEADER_BYTES = 40;
+export declare const MAX_FRAME_BYTES: number;
+export declare function frameFits(metaBytes: number, bodyBytes: number): boolean;
+export interface Frame {
+    type: FrameType;
+    id: string;
+    peer: string;
+    meta: Uint8Array;
+    body: Uint8Array;
+}
+export declare function isUuid(value: unknown): value is string;
+export declare function uuidBytes(id: string): Uint8Array;
+export declare function uuidString(bytes: Uint8Array): string;
+export declare function parseFrame(data: Uint8Array): Frame | null;
+export declare function encodeFrame(frame: Frame): Uint8Array;
+export declare function encodeMeta(value: unknown): Uint8Array;
+export declare function decodeMeta(meta: Uint8Array): unknown;

package/dist/frame.js

@@ -0,0 +1,64 @@
+export const FRAME_VERSION = 1;
+export const FRAME_TYPE = { request: 1, response: 2, routeError: 3 };
+export const FRAME_HEADER_BYTES = 40;
+export const MAX_FRAME_BYTES = 32 * 1024 * 1024;
+export function frameFits(metaBytes, bodyBytes) {
+    return FRAME_HEADER_BYTES + metaBytes + bodyBytes <= MAX_FRAME_BYTES;
+}
+const PEER_OFFSET = 20;
+const META_LENGTH_OFFSET = 36;
+const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+export function isUuid(value) {
+    return typeof value === "string" && UUID.test(value);
+}
+export function uuidBytes(id) {
+    if (!UUID.test(id))
+        throw new TypeError("UUID required");
+    return Buffer.from(id.replaceAll("-", ""), "hex");
+}
+export function uuidString(bytes) {
+    const hex = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString("hex");
+    return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20, 32)}`;
+}
+export function parseFrame(data) {
+    if (data.byteLength < FRAME_HEADER_BYTES)
+        return null;
+    const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
+    const type = view.getUint8(1);
+    const metaEnd = FRAME_HEADER_BYTES + view.getUint32(META_LENGTH_OFFSET);
+    if (view.getUint8(0) !== FRAME_VERSION || view.getUint16(2) !== 0)
+        return null;
+    if (type < FRAME_TYPE.request || type > FRAME_TYPE.routeError || metaEnd > data.byteLength)
+        return null;
+    return {
+        type: type,
+        id: uuidString(data.subarray(4, PEER_OFFSET)),
+        peer: uuidString(data.subarray(PEER_OFFSET, META_LENGTH_OFFSET)),
+        meta: data.subarray(FRAME_HEADER_BYTES, metaEnd),
+        body: data.subarray(metaEnd),
+    };
+}
+export function encodeFrame(frame) {
+    const metaEnd = FRAME_HEADER_BYTES + frame.meta.byteLength;
+    const bytes = new Uint8Array(metaEnd + frame.body.byteLength);
+    const view = new DataView(bytes.buffer);
+    view.setUint8(0, FRAME_VERSION);
+    view.setUint8(1, frame.type);
+    view.setUint32(META_LENGTH_OFFSET, frame.meta.byteLength);
+    bytes.set(uuidBytes(frame.id), 4);
+    bytes.set(uuidBytes(frame.peer), PEER_OFFSET);
+    bytes.set(frame.meta, FRAME_HEADER_BYTES);
+    bytes.set(frame.body, metaEnd);
+    return bytes;
+}
+export function encodeMeta(value) {
+    return new TextEncoder().encode(JSON.stringify(value));
+}
+export function decodeMeta(meta) {
+    try {
+        return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(meta));
+    }
+    catch {
+        return undefined;
+    }
+}

package/dist/index.d.ts

@@ -1,4 +1,5 @@
 export { connectGateway, type GatewayClientOptions, type ConnectionStatus, type ClientSocket, } from "./client.js";
 export { MAX_MESSAGE_BYTES, byteLength, validId, type GatewayMessage, type InboundEnvelope, type OutboundEnvelope, } from "./protocol.js";
 export { CloudError, cloudOrigin, cloudRequest, gatewayConfig } from "./cloud.js";
-export { actionError, isRecord, parseResponse, type ActionResponse, type ActionRequest, } from "./actions.js";
+export { actionError, isRecord, parseResponse, type ActionResponse, type ActionRequest, type BinaryActionResult, } from "./actions.js";
+export { FRAME_TYPE, FRAME_VERSION, MAX_FRAME_BYTES, frameFits, decodeMeta, encodeFrame, encodeMeta, isUuid, parseFrame, uuidBytes, uuidString, type Frame, type FrameType, } from "./frame.js";

package/dist/index.js

@@ -2,3 +2,4 @@ export { connectGateway, } from "./client.js";
 export { MAX_MESSAGE_BYTES, byteLength, validId, } from "./protocol.js";
 export { CloudError, cloudOrigin, cloudRequest, gatewayConfig } from "./cloud.js";
 export { actionError, isRecord, parseResponse, } from "./actions.js";
+export { FRAME_TYPE, FRAME_VERSION, MAX_FRAME_BYTES, frameFits, decodeMeta, encodeFrame, encodeMeta, isUuid, parseFrame, uuidBytes, uuidString, } from "./frame.js";

package/dist/observe-protocol.d.ts

@@ -0,0 +1,100 @@
+import type { ConnectionStatus } from "./client.js";
+export type Outcome = "ok" | "error" | "aborted" | "unknown_outcome" | "interrupted";
+export declare const OUTCOMES: readonly Outcome[];
+export interface ActionRow {
+    id: number;
+    direction: "in" | "out";
+    peer: string | null;
+    action: string;
+    mode: "json" | "binary" | "copy";
+    command: string | null;
+    args: string[] | string | null;
+    cwd: string | null;
+    startedAt: number;
+    endedAt: number | null;
+    outcome: Outcome | null;
+    errorCode: string | null;
+    exitCode: number | null;
+    signal: string | null;
+    stdinBytes: number;
+    stdoutBytes: number;
+    stderrBytes: number;
+    transfer: {
+        direction: "push" | "pull";
+        localPath: string;
+        remotePath: string;
+        bytes: number | null;
+        sha256: string | null;
+    } | null;
+    truncated?: true;
+}
+export interface RoleStatus {
+    deviceId?: string;
+    name?: string;
+    auth: "required" | "authorized" | "revoked";
+    connection: ConnectionStatus;
+    since: number;
+    reconnects: number;
+    running: number;
+}
+export interface NodeStatus {
+    version: string;
+    startedAt: number;
+    roles: {
+        caller?: RoleStatus;
+        host?: RoleStatus;
+    };
+}
+export type NodeEvent = {
+    type: "snapshot";
+    status: NodeStatus;
+    running: ActionRow[];
+} | {
+    type: "connection";
+    role: "caller" | "host";
+    status: ConnectionStatus;
+    at: number;
+} | {
+    type: "action";
+    id: number;
+    phase: "start" | "end";
+    row: ActionRow;
+};
+export type ActionResult = "running" | "succeeded" | "failed" | "no_result";
+export declare const ACTION_RESULTS: readonly ActionResult[];
+export interface LogQuery {
+    kind: "actions";
+    before?: number;
+    limit?: number;
+    direction?: "in" | "out";
+    peer?: string;
+    outcome?: Outcome[];
+    result?: ActionResult[];
+    since?: number;
+    until?: number;
+    text?: string;
+}
+export interface LogStatsQuery {
+    since: number;
+    until?: number;
+    bucketMs: number;
+    direction?: "in" | "out";
+    peer?: string;
+}
+interface LogStatsGroup {
+    peer: string | null;
+    bucket: number;
+    result: ActionResult;
+    count: number;
+}
+export interface LogStats {
+    since: number;
+    until: number;
+    bucketMs: number;
+    groups: LogStatsGroup[];
+}
+export interface LogResult<T> {
+    rows: T[];
+    hasMore: boolean;
+}
+export {};

package/dist/observe-protocol.js

@@ -0,0 +1,13 @@
+export const OUTCOMES = [
+    "ok",
+    "error",
+    "aborted",
+    "unknown_outcome",
+    "interrupted",
+];
+export const ACTION_RESULTS = [
+    "running",
+    "succeeded",
+    "failed",
+    "no_result",
+];

package/dist/observe.d.ts

@@ -0,0 +1,67 @@
+import { type ActionRow, type LogQuery, type LogResult, type LogStats, type LogStatsQuery, type Outcome } from "./observe-protocol.js";
+export declare const MAX_ROW_CHARS: number;
+export interface ObserveStoreOptions {
+    path: string;
+    secrets?: string[];
+    retentionMs?: number;
+    maxBytes?: number;
+    now?: () => number;
+}
+export interface ActionStart {
+    direction: "in" | "out";
+    peer: string | null;
+    action: string;
+    mode: ActionRow["mode"];
+    command?: string | null;
+    args?: string[] | string | null;
+    cwd?: string | null;
+    stdinBytes?: number;
+    transfer?: {
+        direction: "push" | "pull";
+        localPath: string;
+        remotePath: string;
+    } | null;
+}
+export interface ActionEnd {
+    outcome: Exclude<Outcome, "interrupted">;
+    errorCode?: string | null;
+    exitCode?: number | null;
+    signal?: string | null;
+    stdoutBytes?: number;
+    stderrBytes?: number;
+    transferBytes?: number | null;
+    sha256?: string | null;
+}
+export declare class QueryError extends Error {
+}
+export declare class ObserveStore {
+    private readonly options;
+    private db?;
+    private statements;
+    private readonly secrets;
+    private readonly retentionMs;
+    private readonly maxBytes;
+    private readonly now;
+    constructor(options: ObserveStoreOptions);
+    open(): void;
+    private attach;
+    private connect;
+    private discard;
+    private guard;
+    private statement;
+    private run;
+    private all;
+    private mask;
+    startAction(start: ActionStart): ActionRow | null;
+    endAction(id: number, end: ActionEnd): ActionRow | null;
+    private action;
+    running(): ActionRow[];
+    query(query: LogQuery): LogResult<ActionRow>;
+    private limit;
+    private queryActions;
+    stats(query: LogStatsQuery): LogStats;
+    prune(): void;
+    private compact;
+    private usedBytes;
+    close(): void;
+}