@rome-os/node-core 0.1.1 → 0.1.3

package/dist/observe.js

@@ -0,0 +1,457 @@
+import { closeSync, existsSync, openSync, readdirSync, renameSync, rmSync } from "node:fs";
+import { basename, dirname, join } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { ACTION_RESULTS, OUTCOMES, } from "./observe-protocol.js";
+const SCHEMA_VERSION = 1;
+const DAY_MS = 24 * 60 * 60 * 1000;
+export const MAX_ROW_CHARS = 16 * 1024;
+const MAX_RESULT_BYTES = 768 * 1024;
+const MAX_LIMIT = 500;
+const MAX_BUCKETS = 1000;
+const RESULT_SQL = `CASE
+  WHEN a.ended_at IS NULL THEN 'running'
+  WHEN a.outcome = 'ok' AND coalesce(a.exit_code, 0) = 0 AND a.signal IS NULL THEN 'succeeded'
+  WHEN a.outcome IN ('ok', 'error') THEN 'failed'
+  ELSE 'no_result' END`;
+export class QueryError extends Error {
+}
+class CorruptDatabaseError extends Error {
+}
+class SchemaVersionError extends Error {
+}
+function isCorruption(error) {
+    if (error instanceof CorruptDatabaseError)
+        return true;
+    const code = error?.errcode;
+    return typeof code === "number" && ((code & 0xff) === 11 || (code & 0xff) === 26);
+}
+const SCHEMA = `
+CREATE TABLE actions (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  direction TEXT NOT NULL,
+  peer TEXT,
+  action TEXT NOT NULL,
+  mode TEXT NOT NULL,
+  command TEXT,
+  args TEXT,
+  cwd TEXT,
+  started_at INTEGER NOT NULL,
+  ended_at INTEGER,
+  outcome TEXT,
+  error_code TEXT,
+  exit_code INTEGER,
+  signal TEXT,
+  stdin_bytes INTEGER NOT NULL DEFAULT 0,
+  stdout_bytes INTEGER NOT NULL DEFAULT 0,
+  stderr_bytes INTEGER NOT NULL DEFAULT 0,
+  transfer_direction TEXT,
+  local_path TEXT,
+  remote_path TEXT,
+  transfer_bytes INTEGER,
+  sha256 TEXT
+);
+CREATE INDEX actions_started_at ON actions(started_at);
+CREATE INDEX actions_direction ON actions(direction, id);
+CREATE INDEX actions_peer ON actions(peer, id);
+CREATE INDEX actions_outcome ON actions(outcome, id);
+CREATE INDEX actions_open ON actions(id) WHERE ended_at IS NULL;
+CREATE VIRTUAL TABLE actions_fts USING fts5(
+  command, args, cwd, content='actions', content_rowid='id', tokenize='trigram'
+);
+CREATE TRIGGER actions_fts_insert AFTER INSERT ON actions BEGIN
+  INSERT INTO actions_fts(rowid, command, args, cwd)
+  VALUES (new.id, coalesce(new.command, ''), coalesce(new.args, ''), coalesce(new.cwd, ''));
+END;
+CREATE TRIGGER actions_fts_delete AFTER DELETE ON actions BEGIN
+  INSERT INTO actions_fts(actions_fts, rowid, command, args, cwd)
+  VALUES ('delete', old.id, coalesce(old.command, ''), coalesce(old.args, ''), coalesce(old.cwd, ''));
+END;
+`;
+function cut(value, budget) {
+    if (value === null)
+        return value;
+    if (value.length <= budget.remaining) {
+        budget.remaining -= value.length;
+        return value;
+    }
+    budget.truncated = true;
+    const kept = value.slice(0, budget.remaining);
+    budget.remaining = 0;
+    return kept;
+}
+function toActionRow(row) {
+    const state = { remaining: MAX_ROW_CHARS, truncated: false };
+    const command = cut(row.command, state);
+    const cwd = cut(row.cwd, state);
+    let args = null;
+    if (typeof row.args === "string") {
+        let parsed;
+        try {
+            parsed = row.command === null ? undefined : JSON.parse(row.args);
+        }
+        catch { }
+        if (Array.isArray(parsed)) {
+            args = [];
+            for (const arg of parsed) {
+                if (!state.remaining) {
+                    state.truncated = true;
+                    break;
+                }
+                args.push(cut(String(arg), state) ?? "");
+            }
+        }
+        else
+            args = cut(row.args, state);
+    }
+    const value = {
+        id: Number(row.id),
+        direction: row.direction,
+        peer: row.peer,
+        action: row.action,
+        mode: row.mode,
+        command,
+        args,
+        cwd,
+        startedAt: Number(row.started_at),
+        endedAt: row.ended_at === null ? null : Number(row.ended_at),
+        outcome: row.outcome,
+        errorCode: row.error_code,
+        exitCode: row.exit_code === null ? null : Number(row.exit_code),
+        signal: row.signal,
+        stdinBytes: Number(row.stdin_bytes),
+        stdoutBytes: Number(row.stdout_bytes),
+        stderrBytes: Number(row.stderr_bytes),
+        transfer: row.transfer_direction === null
+            ? null
+            : {
+                direction: row.transfer_direction,
+                localPath: cut(row.local_path, state) ?? "",
+                remotePath: cut(row.remote_path, state) ?? "",
+                bytes: row.transfer_bytes === null ? null : Number(row.transfer_bytes),
+                sha256: row.sha256,
+            },
+    };
+    if (state.truncated)
+        value.truncated = true;
+    return value;
+}
+const optionalId = (value, name) => {
+    if (value === undefined)
+        return undefined;
+    if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0)
+        throw new QueryError(`${name} must be a non-negative integer.`);
+    return value;
+};
+function commonFilters(query, where, params) {
+    if (query.direction !== undefined) {
+        if (query.direction !== "in" && query.direction !== "out")
+            throw new QueryError("direction must be in or out.");
+        where.push("a.direction = ?");
+        params.push(query.direction);
+    }
+    if (query.peer !== undefined) {
+        if (typeof query.peer !== "string")
+            throw new QueryError("peer must be a string.");
+        where.push("a.peer = ?");
+        params.push(query.peer);
+    }
+    const since = optionalId(query.since, "since");
+    const until = optionalId(query.until, "until");
+    if (since !== undefined)
+        where.push("a.started_at >= ?"), params.push(since);
+    if (until !== undefined)
+        where.push("a.started_at < ?"), params.push(until);
+}
+export class ObserveStore {
+    options;
+    db;
+    statements = new Map();
+    secrets;
+    retentionMs;
+    maxBytes;
+    now;
+    constructor(options) {
+        this.options = options;
+        this.secrets = (options.secrets ?? []).filter(Boolean);
+        this.retentionMs = options.retentionMs ?? 30 * DAY_MS;
+        this.maxBytes = options.maxBytes ?? 64 * 1024 * 1024;
+        this.now = options.now ?? Date.now;
+    }
+    open() {
+        this.db = this.attach();
+        if (!this.db)
+            return;
+        this.guard(() => {
+            this.run("UPDATE actions SET ended_at = ?, outcome = 'interrupted' WHERE ended_at IS NULL", this.now());
+        });
+        this.prune();
+    }
+    attach() {
+        try {
+            return this.connect();
+        }
+        catch (error) {
+            if (error instanceof SchemaVersionError) {
+                if (!this.discard(false))
+                    return undefined;
+            }
+            else if (isCorruption(error)) {
+                if (!this.discard(true))
+                    return undefined;
+            }
+            else
+                return undefined;
+        }
+        try {
+            return this.connect();
+        }
+        catch {
+            return undefined;
+        }
+    }
+    connect() {
+        const { path } = this.options;
+        if (!existsSync(path))
+            closeSync(openSync(path, "a", 0o600));
+        const db = new DatabaseSync(path);
+        try {
+            db.exec("PRAGMA busy_timeout = 1000");
+            const check = db.prepare("PRAGMA quick_check").get();
+            if (check?.quick_check !== "ok")
+                throw new CorruptDatabaseError();
+            const version = Number(db.prepare("PRAGMA user_version").get()?.user_version ?? 0);
+            if (version === 0) {
+                db.exec("PRAGMA auto_vacuum = INCREMENTAL");
+                db.exec("PRAGMA journal_mode = WAL");
+                db.exec(`BEGIN; ${SCHEMA} PRAGMA user_version = ${SCHEMA_VERSION}; COMMIT;`);
+            }
+            else if (version !== SCHEMA_VERSION) {
+                throw new SchemaVersionError();
+            }
+            else
+                db.exec("PRAGMA journal_mode = WAL");
+            db.exec("PRAGMA synchronous = NORMAL");
+            return db;
+        }
+        catch (error) {
+            db.close();
+            throw error;
+        }
+    }
+    discard(keepCopy) {
+        const { path } = this.options;
+        this.statements.clear();
+        try {
+            this.db?.close();
+        }
+        catch { }
+        this.db = undefined;
+        try {
+            if (keepCopy) {
+                const prefix = `${basename(path)}.corrupt-`;
+                for (const name of readdirSync(dirname(path)))
+                    if (name.startsWith(prefix))
+                        rmSync(join(dirname(path), name), { force: true });
+                if (existsSync(path))
+                    renameSync(path, `${path}.corrupt-${this.now()}`);
+            }
+            else
+                rmSync(path, { force: true });
+            for (const suffix of ["-wal", "-shm"])
+                rmSync(`${path}${suffix}`, { force: true });
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    guard(work, fallback) {
+        if (!this.db)
+            return fallback;
+        try {
+            return work();
+        }
+        catch (error) {
+            if (isCorruption(error))
+                this.db = this.discard(true) ? this.attach() : undefined;
+            return fallback;
+        }
+    }
+    statement(sql) {
+        let statement = this.statements.get(sql);
+        if (!statement) {
+            statement = this.db.prepare(sql);
+            this.statements.set(sql, statement);
+        }
+        return statement;
+    }
+    run(sql, ...params) {
+        return this.statement(sql).run(...params);
+    }
+    all(sql, ...params) {
+        return this.statement(sql).all(...params);
+    }
+    mask(value) {
+        if (value === null || value === undefined)
+            return null;
+        let masked = value;
+        for (const secret of this.secrets)
+            masked = masked.split(secret).join("*".repeat(secret.length));
+        return masked;
+    }
+    startAction(start) {
+        return (this.guard(() => {
+            const args = start.args === undefined || start.args === null
+                ? null
+                : typeof start.args === "string"
+                    ? this.mask(start.args)
+                    : JSON.stringify(start.args.map((arg) => this.mask(arg)));
+            const result = this.run(`INSERT INTO actions (direction, peer, action, mode, command, args, cwd, started_at,
+             stdin_bytes, transfer_direction, local_path, remote_path)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, start.direction, this.mask(start.peer), start.action, start.mode, this.mask(start.command), args, this.mask(start.cwd), this.now(), start.stdinBytes ?? 0, start.transfer?.direction ?? null, this.mask(start.transfer?.localPath), this.mask(start.transfer?.remotePath));
+            return this.action(Number(result.lastInsertRowid));
+        }, null) ?? null);
+    }
+    endAction(id, end) {
+        return (this.guard(() => {
+            this.run(`UPDATE actions SET ended_at = ?, outcome = ?, error_code = ?, exit_code = ?, signal = ?,
+             stdout_bytes = ?, stderr_bytes = ?, transfer_bytes = ?, sha256 = ?
+           WHERE id = ? AND ended_at IS NULL`, this.now(), end.outcome, end.errorCode ?? null, end.exitCode ?? null, end.signal ?? null, end.stdoutBytes ?? 0, end.stderrBytes ?? 0, end.transferBytes ?? null, end.sha256 ?? null, id);
+            return this.action(id);
+        }, null) ?? null);
+    }
+    action(id) {
+        const row = this.all("SELECT * FROM actions WHERE id = ?", id)[0];
+        return row ? toActionRow(row) : null;
+    }
+    running() {
+        return (this.guard(() => this.all("SELECT * FROM actions WHERE ended_at IS NULL ORDER BY id LIMIT ?", MAX_LIMIT).map(toActionRow), []) ?? []);
+    }
+    query(query) {
+        if (query.kind !== "actions")
+            throw new QueryError("Unknown query kind.");
+        return this.queryActions(query);
+    }
+    limit(value, fallback) {
+        if (value === undefined)
+            return fallback;
+        if (typeof value !== "number" || !Number.isInteger(value) || value < 1)
+            throw new QueryError("limit must be a positive integer.");
+        return Math.min(value, MAX_LIMIT);
+    }
+    queryActions(query) {
+        const where = [];
+        const params = [];
+        const before = optionalId(query.before, "before");
+        const limit = this.limit(query.limit, 100);
+        if (before !== undefined)
+            where.push("a.id < ?"), params.push(before);
+        commonFilters(query, where, params);
+        if (query.outcome !== undefined) {
+            if (!Array.isArray(query.outcome) ||
+                !query.outcome.length ||
+                !query.outcome.every((outcome) => OUTCOMES.includes(outcome)))
+                throw new QueryError("outcome must list known outcomes.");
+            where.push(`a.outcome IN (${query.outcome.map(() => "?").join(", ")})`);
+            params.push(...query.outcome);
+        }
+        if (query.result !== undefined) {
+            if (!Array.isArray(query.result) ||
+                !query.result.length ||
+                !query.result.every((result) => ACTION_RESULTS.includes(result)))
+                throw new QueryError("result must list known results.");
+            where.push(`${RESULT_SQL} IN (${query.result.map(() => "?").join(", ")})`);
+            params.push(...query.result);
+        }
+        let join = "";
+        if (query.text !== undefined) {
+            if (typeof query.text !== "string" || [...query.text].length < 3)
+                throw new QueryError("text must have at least three characters.");
+            join = "JOIN actions_fts ON actions_fts.rowid = a.id";
+            where.push("actions_fts MATCH ?");
+            params.push(`"${query.text.replaceAll('"', '""')}"`);
+        }
+        const sql = `SELECT a.* FROM actions a ${join} ${where.length ? `WHERE ${where.join(" AND ")}` : ""} ORDER BY a.id DESC LIMIT ?`;
+        return (this.guard(() => {
+            const raw = this.all(sql, ...params, limit + 1);
+            const rows = [];
+            let bytes = 0;
+            let hasMore = raw.length > limit;
+            for (const item of raw.slice(0, limit)) {
+                const row = toActionRow(item);
+                bytes += Buffer.byteLength(JSON.stringify(row));
+                if (rows.length && bytes > MAX_RESULT_BYTES) {
+                    hasMore = true;
+                    break;
+                }
+                rows.push(row);
+            }
+            return { rows, hasMore };
+        }, { rows: [], hasMore: false }) ?? { rows: [], hasMore: false });
+    }
+    stats(query) {
+        const since = optionalId(query.since, "since");
+        if (since === undefined)
+            throw new QueryError("since is required.");
+        const until = optionalId(query.until, "until") ?? this.now();
+        const bucketMs = query.bucketMs;
+        if (typeof bucketMs !== "number" || !Number.isSafeInteger(bucketMs) || bucketMs < 1000)
+            throw new QueryError("bucketMs must be an integer of at least 1000.");
+        if (until <= since || Math.ceil((until - since) / bucketMs) > MAX_BUCKETS)
+            throw new QueryError(`The range must hold between 1 and ${MAX_BUCKETS} buckets.`);
+        const where = [];
+        const params = [];
+        commonFilters({ ...query, since, until }, where, params);
+        const sql = `SELECT a.peer AS peer, CAST((a.started_at - ?) / ? AS INTEGER) AS bucket,
+        ${RESULT_SQL} AS result, count(*) AS count
+      FROM actions a WHERE ${where.join(" AND ")}
+      GROUP BY peer, bucket, result ORDER BY bucket`;
+        const groups = this.guard(() => this.all(sql, since, bucketMs, ...params).map((row) => ({
+            peer: row.peer,
+            bucket: Number(row.bucket),
+            result: row.result,
+            count: Number(row.count),
+        })), []) ?? [];
+        return { since, until, bucketMs, groups };
+    }
+    prune() {
+        this.guard(() => {
+            const cutoff = this.now() - this.retentionMs;
+            let deleted = Number(this.run("DELETE FROM actions WHERE started_at < ? AND ended_at IS NOT NULL", cutoff)
+                .changes);
+            if (this.usedBytes() > this.maxBytes) {
+                this.compact();
+                for (let round = 0; round < 64 && this.usedBytes() > this.maxBytes; round++) {
+                    const count = Number(this.all("SELECT count(*) AS n FROM actions")[0]?.n ?? 0);
+                    if (!count)
+                        break;
+                    deleted += Number(this.run(`DELETE FROM actions WHERE id IN (SELECT id FROM actions WHERE ended_at IS NOT NULL
+                 ORDER BY id LIMIT ?)`, Math.max(1, Math.ceil(count / 10))).changes);
+                    this.compact();
+                }
+            }
+            if (!deleted)
+                return;
+            this.db.exec("PRAGMA incremental_vacuum");
+            this.db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
+        });
+    }
+    compact() {
+        this.db.exec("INSERT INTO actions_fts(actions_fts) VALUES ('optimize')");
+        this.db.exec("PRAGMA incremental_vacuum");
+    }
+    usedBytes() {
+        const pages = Number(this.all("PRAGMA page_count")[0]?.page_count ?? 0);
+        const free = Number(this.all("PRAGMA freelist_count")[0]?.freelist_count ?? 0);
+        const size = Number(this.all("PRAGMA page_size")[0]?.page_size ?? 4096);
+        return (pages - free) * size;
+    }
+    close() {
+        this.statements.clear();
+        try {
+            this.db?.close();
+        }
+        catch { }
+        this.db = undefined;
+    }
+}

package/dist/transfer.d.ts

@@ -0,0 +1,102 @@
+import { type FileHandle } from "node:fs/promises";
+import { type ActionResponse } from "./actions.js";
+import type { CopyProgress, CopyRequest } from "./daemon-protocol.js";
+export declare const TRANSFER_VERSION = 1;
+export declare const TRANSFER_CHUNK_BYTES: number;
+export declare const TRANSFER_WINDOW_BYTES: number;
+export declare const TRANSFER_IDLE_MS = 60000;
+export declare const MAX_HOST_TRANSFERS = 4;
+export declare const PART_SUFFIX = ".rome-part";
+export type TransferMessage = {
+    type: "transfer";
+    kind: "data";
+    offset: number;
+} | {
+    type: "transfer";
+    kind: "ack";
+    offset: number;
+} | {
+    type: "transfer";
+    kind: "end";
+    size: number;
+    sha256: string;
+} | {
+    type: "transfer";
+    kind: "done";
+    size: number;
+    sha256: string;
+} | {
+    type: "transfer";
+    kind: "abort";
+    code: string;
+    message: string;
+};
+export declare function parseTransferMessage(value: unknown): TransferMessage | null;
+export declare class TransferError extends Error {
+    readonly code: string;
+    constructor(code: string, message: string);
+}
+export type TransferSend = (meta: unknown, body?: Uint8Array) => boolean;
+export interface TransferOptions {
+    chunkBytes?: number;
+    windowBytes?: number;
+    idleMs?: number;
+    onProgress?(bytes: number): void;
+}
+export interface TransferSummary {
+    size: number;
+    sha256: string;
+}
+export interface TransferEndpoint {
+    readonly result: Promise<TransferSummary>;
+    receive(message: TransferMessage, body: Uint8Array): void;
+    abort(code: string, message: string, notify?: boolean): void;
+}
+export interface ReceiverEndpoint extends TransferEndpoint {
+    expect(size: number): void;
+}
+export interface Source {
+    size: number;
+    start(send: TransferSend, options?: TransferOptions): TransferEndpoint;
+    close(): Promise<void>;
+}
+export declare function openSource(path: string): Promise<Source>;
+export interface Destination {
+    part: string;
+    start(send: TransferSend, options?: TransferOptions & {
+        size?: number;
+    }): ReceiverEndpoint;
+    cancel(): Promise<void>;
+}
+export declare function createPartFile(path: string, id?: string): Promise<{
+    part: string;
+    handle: FileHandle;
+}>;
+export declare function openDestination(path: string): Promise<Destination>;
+export declare class TransferHost {
+    private options;
+    private transfers;
+    constructor(options?: TransferOptions & {
+        limit?: number;
+    });
+    open(id: string, peer: string, args: unknown, send: TransferSend): Promise<void>;
+    receive(id: string, peer: string, message: TransferMessage, body: Uint8Array): void;
+    abortAll(): void;
+}
+export interface TransferChannel {
+    send: TransferSend;
+    close(): void;
+}
+export interface ChannelEvents {
+    frame(meta: unknown, body: Uint8Array): void;
+    lost(): void;
+}
+export interface CopyOptions extends Omit<TransferOptions, "onProgress"> {
+    signal?: AbortSignal;
+    onProgress?(progress: CopyProgress): void;
+}
+export declare function copyWithDevice(openChannel: (events: ChannelEvents) => Promise<TransferChannel | ActionResponse>, request: CopyRequest, options?: CopyOptions): Promise<{
+    bytes: number;
+    sha256: string;
+    ms: number;
+}>;