@starlove/orbit 1.4.0 → 2.0.0

package/lib/adapters/sqlite/board.d.ts

@@ -0,0 +1,77 @@
+/**
+ * One board's database, as one store opens it: the BoardStore of the port.
+ *
+ * - Writes go one at a time through an in-process queue, and between
+ *   processes through SQLite's write lock (`BEGIN IMMEDIATE`), taken with
+ *   busy_timeout 0 and retried on a timer, so a lock held elsewhere never
+ *   stops the thread. A body's reads see its own writes; a body that throws
+ *   rolls everything back, events and index included.
+ * - Reads run on a small pool of connections, each read one transaction, so
+ *   it sees one committed state (WAL lets reads and a write run side by side).
+ *   Connections open when needed and close after a while idle.
+ * - Watchers hear this store's writes as each commits, and other processes'
+ *   writes by polling `PRAGMA data_version`, then reading the events appended
+ *   after the last one heard.
+ */
+import type { Board, BoardChange, Id } from "../../core/model.ts";
+import type { BoardStore, BoardWriter } from "../../store/types.ts";
+import type { Params, Row } from "./database.ts";
+import { reader } from "./records.ts";
+/** The registry's side of a board's write: its listing brought in line, committed with the write. */
+export interface Follow {
+    commit(): void;
+    rollback(): void;
+}
+/** What a board needs from the store that opened it. */
+export interface BoardHost {
+    /** How long a write waits for a lock held elsewhere, in ms. */
+    readonly busyMs: number;
+    /** Whether a board other than `exceptId` has the slug. */
+    slugTaken(slug: string, exceptId: Id): boolean;
+    /** Starts updating the board's listing from its record; refuses a slug another board has. */
+    follow(board: Board): Promise<Follow>;
+}
+type Listener = (change: BoardChange) => void;
+/** One board's database files, opened by one store. */
+export declare class BoardFiles {
+    readonly id: Id;
+    readonly dbPath: string;
+    private readonly host;
+    private readonly writerConn;
+    private readonly lock;
+    private readonly idle;
+    private idleTimer;
+    private watching;
+    private fileIdentity;
+    private removed;
+    private closed;
+    private constructor();
+    /** Opens a board's database: refused untouched unless it is v2's, its index rebuilt once if its shape changed. */
+    static open(host: BoardHost, id: Id, dbPath: string, slug: string): Promise<BoardFiles>;
+    /** The documents stay; the table they live in is made again in its current shape. */
+    private rebuildIndex;
+    /** False once the board is removed, here or by another process (its file gone or made again). */
+    isLive(): boolean;
+    private live;
+    /** Async, so a board that's gone answers as a rejected read, as every other failure does. */
+    read<T>(body: (r: ReturnType<typeof reader>) => Promise<T>): Promise<T>;
+    /** One statement on a read connection, for the store's own lookups across boards. */
+    query(sql: string, params: Params): Row[];
+    private release;
+    write<T>(body: (w: BoardWriter) => Promise<T>): Promise<T>;
+    /** The board's first write: its own record, then what `seed` writes. Its listing is the store's to make. */
+    seed(board: Board, body: (w: BoardWriter) => Promise<void>): Promise<Board>;
+    /** One transaction: the body, the listing when the board's record changed, the commit, then the watchers. */
+    private transact;
+    watch(listener: Listener): () => void;
+    /** Runs `body` once this store's writes to the board are done, holding off new ones. */
+    exclusive<T>(body: () => Promise<T>): Promise<T>;
+    /** Lets go of the files after the board is gone; every later call answers board_not_found. */
+    markRemoved(): void;
+    /** Lets go of the files once this store's writes are done. */
+    close(): Promise<void>;
+    private shut;
+    /** The port's view of this board. */
+    store(): BoardStore;
+}
+export {};

package/lib/adapters/sqlite/board.js

@@ -0,0 +1,328 @@
+/**
+ * One board's database, as one store opens it: the BoardStore of the port.
+ *
+ * - Writes go one at a time through an in-process queue, and between
+ *   processes through SQLite's write lock (`BEGIN IMMEDIATE`), taken with
+ *   busy_timeout 0 and retried on a timer, so a lock held elsewhere never
+ *   stops the thread. A body's reads see its own writes; a body that throws
+ *   rolls everything back, events and index included.
+ * - Reads run on a small pool of connections, each read one transaction, so
+ *   it sees one committed state (WAL lets reads and a write run side by side).
+ *   Connections open when needed and close after a while idle.
+ * - Watchers hear this store's writes as each commits, and other processes'
+ *   writes by polling `PRAGMA data_version`, then reading the events appended
+ *   after the last one heard.
+ */
+import { statSync } from "node:fs";
+import { OrbitError } from "../../core/errors.js";
+import { changeOf } from "../../core/model.js";
+import { checkFile, Connection, untilFree } from "./database.js";
+import { Lock } from "./lock.js";
+import { BOARD, EVENT, noWait, reader, toRecord, writer } from "./records.js";
+import { SEARCH_INDEX_VERSION, SEARCH_TABLE } from "./schema.js";
+/** How long other processes' writes take to reach a watcher, at most. */
+const POLL_MS = 100;
+/** Read connections kept open between reads, and how long they stay open unused. */
+const IDLE_READERS = 4;
+const IDLE_MS = 30_000;
+/**
+ * Which file a path names now (its device and inode), so a board another
+ * process deleted, or deleted and made again, is noticed before the next read
+ * or write. Times aren't used: where a file system has no birth time, the one
+ * reported changes with every checkpoint.
+ */
+function identity(path) {
+    const stat = statSync(path, { throwIfNoEntry: false });
+    return stat ? `${String(stat.dev)}:${String(stat.ino)}` : null;
+}
+/** The watchers of one board, and the connection that polls for other processes' writes. */
+class Watching {
+    listeners = new Set();
+    files;
+    conn;
+    timer;
+    lastSeq;
+    dataVersion;
+    constructor(files) {
+        this.files = files;
+        this.conn = new Connection(files.dbPath, { queryOnly: true });
+        this.lastSeq = Number(this.conn.get("SELECT coalesce(max(seq), 0) AS seq FROM events")?.seq ?? 0);
+        this.dataVersion = this.version();
+        this.timer = setInterval(() => {
+            this.poll();
+        }, POLL_MS);
+        this.timer.unref();
+    }
+    version() {
+        return Number(this.conn.get("PRAGMA data_version")?.data_version ?? 0);
+    }
+    poll() {
+        if (!this.files.isLive())
+            return;
+        try {
+            const version = this.version();
+            if (version === this.dataVersion)
+                return;
+            this.dataVersion = version;
+            this.catchUp(this.conn);
+        }
+        catch {
+            // Busy or briefly unreadable: the next poll tries again.
+        }
+    }
+    /** This store's write just committed these events: hand them over, or read what came first if another process wrote in between. */
+    committed(events, c) {
+        const first = events[0];
+        if (!first)
+            return;
+        if (first.seq === this.lastSeq + 1) {
+            for (const { seq, event } of events)
+                this.deliver(seq, event);
+        }
+        else {
+            this.catchUp(c);
+        }
+    }
+    catchUp(c) {
+        for (const row of c.all(`SELECT seq, ${EVENT.columns.join(", ")} FROM events WHERE seq > ? ORDER BY seq`, [this.lastSeq])) {
+            this.deliver(Number(row.seq), toRecord(EVENT, row, this.files.id));
+        }
+    }
+    deliver(seq, event) {
+        if (seq <= this.lastSeq)
+            return;
+        this.lastSeq = seq;
+        for (const listener of [...this.listeners]) {
+            queueMicrotask(() => {
+                listener(changeOf(this.files.id, event));
+            });
+        }
+    }
+    stop() {
+        clearInterval(this.timer);
+        this.conn.close();
+    }
+}
+/** One board's database files, opened by one store. */
+export class BoardFiles {
+    id;
+    dbPath;
+    host;
+    writerConn;
+    lock = new Lock();
+    idle = [];
+    idleTimer = null;
+    watching = null;
+    fileIdentity;
+    removed = false;
+    closed = false;
+    constructor(host, id, dbPath) {
+        this.host = host;
+        this.id = id;
+        this.dbPath = dbPath;
+        this.fileIdentity = identity(dbPath);
+        this.writerConn = new Connection(dbPath);
+    }
+    /** Opens a board's database: refused untouched unless it is v2's, its index rebuilt once if its shape changed. */
+    static async open(host, id, dbPath, slug) {
+        // The message names the board and the way out, never the path: agents read it too.
+        if (!checkFile(dbPath)) {
+            throw new OrbitError("board_file_missing", {
+                message: `Board "${slug}" has lost its database file. Restore the newest copy in the data folder's backups folder (the README's "Storage, backup, and moving machines" says how: its database, and the contents of its .files folder beside it), or delete the board in Settings.`
+            });
+        }
+        const files = new BoardFiles(host, id, dbPath);
+        try {
+            const version = files.writerConn.get("SELECT value FROM meta WHERE key = 'search_index_version'")?.value;
+            if (String(version) !== String(SEARCH_INDEX_VERSION))
+                await files.rebuildIndex();
+        }
+        catch (error) {
+            files.writerConn.close();
+            throw error;
+        }
+        return files;
+    }
+    /** The documents stay; the table they live in is made again in its current shape. */
+    rebuildIndex() {
+        return this.lock.run(async () => {
+            const c = this.writerConn;
+            await untilFree(() => {
+                c.exec("BEGIN IMMEDIATE");
+            }, Date.now() + this.host.busyMs);
+            try {
+                const documents = c.all("SELECT rowid, title, description, type, details, comments FROM search");
+                c.exec(`DROP TABLE search; ${SEARCH_TABLE}`);
+                for (const d of documents) {
+                    c.run("INSERT INTO search (rowid, title, description, type, details, comments) VALUES (?, ?, ?, ?, ?, ?)", [d.rowid ?? null, d.title ?? "", d.description ?? "", d.type ?? "", d.details ?? "", d.comments ?? ""]);
+                }
+                c.run("INSERT OR REPLACE INTO meta (key, value) VALUES ('search_index_version', ?)", [String(SEARCH_INDEX_VERSION)]);
+                c.exec("COMMIT");
+            }
+            catch (error) {
+                c.rollback();
+                throw error;
+            }
+        });
+    }
+    /** False once the board is removed, here or by another process (its file gone or made again). */
+    isLive() {
+        if (this.closed || this.removed)
+            return false;
+        if (identity(this.dbPath) !== this.fileIdentity)
+            this.removed = true;
+        return !this.removed;
+    }
+    live() {
+        if (this.closed)
+            throw new OrbitError("storage_failed", { message: "This store is closed." });
+        if (!this.isLive())
+            throw new OrbitError("board_not_found");
+    }
+    /** Async, so a board that's gone answers as a rejected read, as every other failure does. */
+    async read(body) {
+        this.live();
+        const c = this.idle.pop() ?? new Connection(this.dbPath, { queryOnly: true });
+        c.exec("BEGIN");
+        const done = () => {
+            c.finishRead();
+            this.release(c);
+        };
+        let result;
+        try {
+            result = body(reader({ c, boardId: this.id, before: noWait }));
+        }
+        catch (error) {
+            done();
+            throw error;
+        }
+        return result.finally(done);
+    }
+    /** One statement on a read connection, for the store's own lookups across boards. */
+    query(sql, params) {
+        this.live();
+        const c = this.idle.pop() ?? new Connection(this.dbPath, { queryOnly: true });
+        try {
+            return c.all(sql, params);
+        }
+        finally {
+            this.release(c);
+        }
+    }
+    release(c) {
+        if (this.closed || this.removed || this.idle.length >= IDLE_READERS) {
+            c.close();
+            return;
+        }
+        this.idle.push(c);
+        if (this.idleTimer)
+            clearTimeout(this.idleTimer);
+        this.idleTimer = setTimeout(() => {
+            for (const idle of this.idle.splice(0))
+                idle.close();
+        }, IDLE_MS);
+        this.idleTimer.unref();
+    }
+    write(body) {
+        return this.lock.run(() => this.transact(body, null));
+    }
+    /** The board's first write: its own record, then what `seed` writes. Its listing is the store's to make. */
+    seed(board, body) {
+        return this.lock.run(async () => {
+            let made = board;
+            await this.transact(async (w) => {
+                await body(w);
+                made = await w.board.get();
+            }, board);
+            return made;
+        });
+    }
+    /** One transaction: the body, the listing when the board's record changed, the commit, then the watchers. */
+    async transact(body, first) {
+        this.live();
+        const c = this.writerConn;
+        await untilFree(() => {
+            c.exec("BEGIN IMMEDIATE");
+        }, Date.now() + this.host.busyMs);
+        const log = { events: [], board: null };
+        let open = true;
+        const before = async () => {
+            await Promise.resolve();
+            if (!open)
+                throw new OrbitError("write_closed");
+        };
+        let follow = null;
+        try {
+            if (first)
+                c.run(BOARD.insert, BOARD.columns.map((column) => first[column]));
+            const checks = { slugTaken: (slug) => this.host.slugTaken(slug, this.id) };
+            let result;
+            try {
+                result = await body(writer({ c, boardId: this.id, before }, log, checks));
+            }
+            finally {
+                open = false;
+            }
+            if (log.board && !first)
+                follow = await this.host.follow(log.board);
+            c.exec("COMMIT");
+            follow?.commit();
+            if (log.events.length > 0)
+                this.watching?.committed(log.events, c);
+            return result;
+        }
+        catch (error) {
+            c.rollback();
+            follow?.rollback();
+            throw error;
+        }
+    }
+    watch(listener) {
+        this.live();
+        this.watching ??= new Watching(this);
+        const watching = this.watching;
+        watching.listeners.add(listener);
+        return () => {
+            watching.listeners.delete(listener);
+            if (watching.listeners.size === 0 && this.watching === watching) {
+                watching.stop();
+                this.watching = null;
+            }
+        };
+    }
+    /** Runs `body` once this store's writes to the board are done, holding off new ones. */
+    exclusive(body) {
+        return this.lock.run(body);
+    }
+    /** Lets go of the files after the board is gone; every later call answers board_not_found. */
+    markRemoved() {
+        this.removed = true;
+        this.shut();
+    }
+    /** Lets go of the files once this store's writes are done. */
+    close() {
+        return this.lock.run(() => {
+            this.closed = true;
+            this.shut();
+            return Promise.resolve();
+        });
+    }
+    shut() {
+        this.watching?.stop();
+        this.watching = null;
+        if (this.idleTimer)
+            clearTimeout(this.idleTimer);
+        for (const idle of this.idle.splice(0))
+            idle.close();
+        this.writerConn.close();
+    }
+    /** The port's view of this board. */
+    store() {
+        return {
+            id: this.id,
+            read: (body) => this.read(body),
+            write: (body) => this.write(body),
+            watch: (listener) => this.watch(listener)
+        };
+    }
+}

package/lib/adapters/sqlite/database.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Connections to Orbit's SQLite files, and the rules every statement runs
+ * under: SQLite never waits on the server's thread (busy_timeout 0, retried
+ * here on a timer), a database's own error never reaches an answer (each is
+ * mapped to the port's codes), and a file without v2's version is never
+ * opened for writing.
+ */
+import type { SQLInputValue, SQLOutputValue } from "node:sqlite";
+import { OrbitError } from "../../core/errors.ts";
+import type { ErrorCode } from "../../core/errors.ts";
+export type Row = Record<string, SQLOutputValue>;
+export type Params = readonly SQLInputValue[];
+/** What a statement's failure means, where the schema alone can't say (a lane still holding cards). */
+export interface Meaning {
+    /** The code for a foreign key failure; `invalid_reference` by default. */
+    foreignKey?: ErrorCode;
+}
+/** An OrbitError for anything a statement throws, the original kept as its cause. */
+export declare function storageError(error: unknown, meaning?: Meaning): OrbitError;
+/**
+ * Runs `attempt` until it isn't refused as busy, waiting on a timer between
+ * tries (1 ms, doubling to 50 ms), so the thread stays free for everything
+ * else. After `deadline` (a Date.now() value) the refusal stands: `storage_busy`.
+ */
+export declare function untilFree<T>(attempt: () => T, deadline: number): Promise<T>;
+/** One connection to one file, with each statement prepared once. */
+export declare class Connection {
+    readonly path: string;
+    private readonly db;
+    private readonly prepared;
+    private closed;
+    constructor(path: string, options?: {
+        queryOnly?: boolean;
+    });
+    exec(sql: string): void;
+    private statement;
+    run(sql: string, params?: Params, meaning?: Meaning): {
+        changes: number;
+        lastInsertRowid: number;
+    };
+    get(sql: string, params?: Params, meaning?: Meaning): Row | undefined;
+    all(sql: string, params?: Params): Row[];
+    /**
+     * Ends a transaction left open by a failure; quiet if there is none (SQLite
+     * may have ended it with the failure). It asks SQLite by trying, which
+     * reads the same on every Node.
+     */
+    rollback(): void;
+    /** Ends a read's transaction. */
+    finishRead(): void;
+    close(): void;
+}
+/**
+ * Whether the file at `path` is one this build may open: true when it is v2's
+ * at this schema version, false when there is no file. Anything else is
+ * refused without being opened, so its bytes stay as they are: a file of 1.x
+ * or the first 2.0 (no version) with `data_needs_migration`, a newer
+ * schema's with `data_from_newer_orbit`.
+ */
+export declare function checkFile(path: string): boolean;
+/**
+ * Makes a database at `path` with `schema`, all at once: it is built under a
+ * temporary name and linked into place, so no other process ever sees a file
+ * without its version, and when two make the same file at once one wins and
+ * the other's attempt is discarded. True when this call made it.
+ */
+export declare function createFile(path: string, schema: string): boolean;