@platformatic/sqlite-js-stream 0.0.0-stage → 0.1.1

package/dist/backends/file/replica-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"replica-client.d.ts","sourceRoot":"","sources":["../../../src/backends/file/replica-client.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAKH,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,OAAO,EAKL,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,KAAK,mBAAmB,EACzB,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAE,QAAQ,EAAU,MAAM,oBAAoB,CAAC;AAI3D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAKnD,eAAO,MAAM,iBAAiB,SAAS,CAAC;AAExC,+DAA+D;AAC/D,qBAAa,iBAAkB,YAAW,aAAa;IACrD,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,MAAM,CAAsB;gBAExB,IAAI,EAAE,MAAM;IAIxB,IAAI,IAAI,MAAM;IAIR,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAI3B,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAI/B,yCAAyC;IACzC,IAAI,IAAI,MAAM;IAId,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAIlC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM;IAI7D,QAAQ,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,OAAO,CAAA;KAAE,GAAG,aAAa,CAAC,QAAQ,CAAC;IAyC/F,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,QAAQ,CAAC;IAqBxD,YAAY,CAAC,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,aAAa,CAAC,UAAU,GAAG,MAAM,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC;IAiEpG,cAAc,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBhD,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;CAOjC;AAED,2EAA2E;AAC3E,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,eAAe,GAAG,IAAI,EAC9B,SAAS,EAAE,MAAM,GAAG,IAAI,GACvB,aAAa,CAKf"}

package/dist/backends/file/replica-client.js

@@ -0,0 +1,209 @@
+/**
+ * File replica client — replicates LTX files to a local directory.
+ *
+ * Layout: `<root>/ltx/<level>/<minTXID>-<maxTXID>.ltx`
+ * Writes are atomic: data is streamed to a `.tmp` file, fsync'd, then renamed,
+ * the directory is fsync'd, and the file mtime is set from the LTX header
+ * timestamp to preserve original creation time.
+ */
+import { createReadStream } from 'node:fs';
+import { open, readdir, rename, rm, stat, utimes } from 'node:fs/promises';
+import nodePath from 'node:path';
+import { Readable } from 'node:stream';
+import { ltxFilePath as sharedLTXFilePath, ltxLevelDir as sharedLTXLevelDir, LTXError, parseFilename, } from "../../replica-client.js";
+import { HeaderSize, unmarshalHeader } from "../../ltx/index.js";
+import { concatBytes } from "../../util/stream.js";
+import { fsyncDir, mkdirAll } from "../../util/fs.js";
+import { FileReplicaError, InvalidArgumentError } from "../../errors.js";
+import { noopLogger } from "../../util/logger.js";
+import { registerReplicaClientFactory } from "../../replica-url.js";
+export const ReplicaClientType = 'file';
+/** FileReplicaClient writes LTX files to a local directory. */
+export class FileReplicaClient {
+    pathVal;
+    logger = noopLogger;
+    constructor(path) {
+        this.pathVal = path;
+    }
+    type() {
+        return ReplicaClientType;
+    }
+    async init() {
+        /* no initialization required */
+    }
+    setLogger(logger) {
+        this.logger = logger;
+    }
+    /** Returns the destination root path. */
+    path() {
+        return this.pathVal;
+    }
+    ltxLevelDir(level) {
+        return sharedLTXLevelDir(this.pathVal, level);
+    }
+    ltxFilePath(level, minTXID, maxTXID) {
+        return sharedLTXFilePath(this.pathVal, level, minTXID, maxTXID);
+    }
+    async *ltxFiles(opts) {
+        const dir = this.ltxLevelDir(opts.level);
+        let entries;
+        try {
+            entries = await readdir(dir, { withFileTypes: true });
+        }
+        catch (err) {
+            if (err.code === 'ENOENT')
+                return;
+            throw err;
+        }
+        const infos = [];
+        for (const entry of entries) {
+            if (!entry.isFile())
+                continue;
+            let minTXID;
+            let maxTXID;
+            try {
+                ({ minTXID, maxTXID } = parseFilename(entry.name));
+            }
+            catch {
+                continue; // skip non-conforming files
+            }
+            if (minTXID < opts.seek)
+                continue;
+            const full = nodePath.join(dir, entry.name);
+            const st = await stat(full);
+            infos.push({
+                level: opts.level,
+                minTXID,
+                maxTXID,
+                size: st.size,
+                createdAt: st.mtimeMs,
+            });
+        }
+        infos.sort((a, b) => {
+            if (a.minTXID !== b.minTXID)
+                return a.minTXID < b.minTXID ? -1 : 1;
+            return a.maxTXID < b.maxTXID ? -1 : a.maxTXID > b.maxTXID ? 1 : 0;
+        });
+        for (const info of infos)
+            yield info;
+    }
+    async openLtxFile(opts) {
+        const p = this.ltxFilePath(opts.level, opts.minTXID, opts.maxTXID);
+        let st;
+        try {
+            st = await stat(p);
+        }
+        catch (err) {
+            throw new LTXError('open', p, opts.level, opts.minTXID, opts.maxTXID, err);
+        }
+        if (opts.offset >= st.size) {
+            return Readable.from([]);
+        }
+        const end = opts.size > 0
+            ? Math.min(opts.offset + opts.size - 1, st.size - 1)
+            : st.size - 1;
+        return createReadStream(p, { start: opts.offset, end });
+    }
+    async writeLtxFile(opts, body) {
+        const dir = this.ltxLevelDir(opts.level);
+        await mkdirAll(dir);
+        const filename = this.ltxFilePath(opts.level, opts.minTXID, opts.maxTXID);
+        const tmp = `${filename}.tmp`;
+        let fh = null;
+        try {
+            fh = await open(tmp, 'w');
+        }
+        catch (err) {
+            throw new LTXError('create', tmp, opts.level, opts.minTXID, opts.maxTXID, err);
+        }
+        let headerBytes = null;
+        let collected = [];
+        let collectedLen = 0;
+        let size = 0;
+        try {
+            for await (const chunk of body) {
+                const bytes = typeof chunk === 'string' ? new TextEncoder().encode(chunk) : Uint8Array.from(chunk);
+                if (headerBytes === null) {
+                    collected.push(bytes);
+                    collectedLen += bytes.length;
+                    if (collectedLen >= HeaderSize) {
+                        const concat = concatBytes(collected);
+                        headerBytes = concat.subarray(0, HeaderSize);
+                        collected = [];
+                        collectedLen = 0;
+                    }
+                }
+                await writeAll(fh, bytes);
+                size += bytes.length;
+            }
+            if (headerBytes === null) {
+                throw new FileReplicaError('write ltx file: input too short to contain a header');
+            }
+            const hdr = unmarshalHeader(headerBytes);
+            const timestamp = Number(hdr.timestamp);
+            await fh.sync();
+            await fh.close();
+            fh = null;
+            await rename(tmp, filename);
+            await fsyncDir(dir);
+            await utimes(filename, new Date(timestamp), new Date(timestamp));
+            return {
+                level: opts.level,
+                minTXID: opts.minTXID,
+                maxTXID: opts.maxTXID,
+                size,
+                createdAt: timestamp,
+            };
+        }
+        catch (err) {
+            if (fh !== null)
+                await fh.close().catch(() => { });
+            await rm(tmp, { force: true }).catch(() => { });
+            throw err;
+        }
+    }
+    async deleteLtxFiles(infos) {
+        for (const info of infos) {
+            const p = this.ltxFilePath(info.level, info.minTXID, info.maxTXID);
+            this.logger.debug('deleting ltx file', {
+                level: info.level,
+                minTXID: info.minTXID,
+                maxTXID: info.maxTXID,
+                path: p,
+            });
+            try {
+                await rm(p);
+            }
+            catch (err) {
+                if (err.code !== 'ENOENT')
+                    throw err;
+            }
+        }
+    }
+    async deleteAll() {
+        try {
+            await rm(this.pathVal, { recursive: true, force: true });
+        }
+        catch (err) {
+            if (err.code !== 'ENOENT')
+                throw err;
+        }
+    }
+}
+/** NewReplicaClientFromURL creates a FileReplicaClient from a URL path. */
+export function NewReplicaClientFromURL(_scheme, _host, urlPath, _query, _userinfo) {
+    if (!urlPath) {
+        throw new InvalidArgumentError('file replica path required');
+    }
+    return new FileReplicaClient(urlPath);
+}
+// Mirrors Go's init(): register the file factory when this module loads.
+registerReplicaClientFactory('file', NewReplicaClientFromURL);
+/** Writes all of `bytes` to a FileHandle, handling partial writes. */
+async function writeAll(handle, bytes) {
+    let offset = 0;
+    while (offset < bytes.length) {
+        const { bytesWritten } = await handle.write(bytes, offset, bytes.length - offset);
+        offset += bytesWritten;
+    }
+}

package/dist/db.d.ts

@@ -0,0 +1,80 @@
+/**
+ * LocalDatabase is the internal engine that reads a SQLite database and its
+ * `-wal` file and writes LTX files into a local cache (`<path>-litestream/ltx/`).
+ *
+ * It is deliberately NOT part of the public API: `Replica`/`Store` take a
+ * database `path` and read directly, so the library is agnostic to whatever
+ * SQLite driver the application uses. To keep snapshots consistent, it holds a
+ * short-lived **read-only** `node:sqlite` read transaction that pins the WAL
+ * (SQLite will not checkpoint/truncate the WAL past an active reader), while
+ * reading the database + WAL files directly.
+ */
+import { type FileInfo } from './ltx/index.js';
+/**
+ * LocalDatabase reads a database + WAL from disk, tracks the local replicated
+ * position, and builds LTX files in the local cache.
+ */
+export declare class LocalDatabase {
+    readonly path: string;
+    readonly metaPath: string;
+    readonly pageSize: number;
+    private conn;
+    private constructor();
+    /**
+     * Open opens a local database reader for `path`. It does not create the
+     * database — the application owns it. `pageSize` is read from the database
+     * header; a missing file falls back to the default page size.
+     */
+    static open(path: string): Promise<LocalDatabase>;
+    /** WalPath returns the path to the database's WAL file. */
+    walPath(): string;
+    /** LTXDir returns the local LTX root directory. */
+    ltxDir(): string;
+    /** LTXLevelDir returns the local LTX directory for a level. */
+    ltxLevelDir(level: number): string;
+    /** LTXPath returns the local path of a single LTX file. */
+    ltxPath(level: number, minTXID: bigint, maxTXID: bigint): string;
+    /** Lists local level-0 LTX files, sorted by TXID. */
+    l0Files(): Promise<FileInfo[]>;
+    /** Lists local LTX files for any level, sorted by TXID. */
+    filesForLevel(level: number): Promise<FileInfo[]>;
+    /** Pos returns the maximum TXID in the local level-0 cache (0 if none). */
+    pos(): Promise<bigint>;
+    /** Returns the local LTX file with the highest maxTXID across all levels. */
+    private maxFile;
+    /**
+     * Opens (and caches) a read-only connection used to hold a snapshot during
+     * sync/snapshot. Not part of the public API; the DB file must exist.
+     */
+    private ensureConn;
+    /**
+     * Pins a WAL snapshot by starting a read transaction and issuing a read that
+     * touches the database (a plain `SELECT 1` does not acquire the WAL read lock).
+     */
+    private beginRead;
+    /** Releases the pinned read snapshot. */
+    private commitRead;
+    /** Current committed database size, in pages (as seen by the pinned reader). */
+    private pageCount;
+    /** Sync builds a level-0 LTX file and writes it to the local cache. */
+    sync(): Promise<FileInfo | null>;
+    /** Writes a full snapshot: every committed page, preferring WAL pages. */
+    private writePagesFromDB;
+    /** Writes an incremental file: changed WAL pages + DB growth pages. */
+    private writePagesFromWAL;
+    /**
+     * Snapshot writes a full-database LTX file at `SnapshotLevel` (9) with
+     * MinTXID=1 and MaxTXID equal to the current local position. Returns null if
+     * a snapshot would cover no data (empty database).
+     */
+    snapshot(): Promise<FileInfo | null>;
+    /** Reads a single page, preferring the WAL over the database file. */
+    private readPage;
+    /** Reads the WAL file bytes, returning empty if it does not exist. */
+    private readWalOrEmpty;
+    /** Decodes the header of the max-TXID local L0 file (or null if none). */
+    private prevSyncInfo;
+    /** Close releases the internal read-only connection. */
+    close(): void;
+}
+//# sourceMappingURL=db.d.ts.map

package/dist/db.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../src/db.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAMH,OAAO,EAQL,KAAK,QAAQ,EAEd,MAAM,gBAAgB,CAAC;AAoBxB;;;GAGG;AACH,qBAAa,aAAa;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,IAAI,CAA6B;IAEzC,OAAO;IAMP;;;;OAIG;WACU,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;IAkBvD,2DAA2D;IAC3D,OAAO,IAAI,MAAM;IAIjB,mDAAmD;IACnD,MAAM,IAAI,MAAM;IAIhB,+DAA+D;IAC/D,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAIlC,2DAA2D;IAC3D,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM;IAIhE,qDAAqD;IAC/C,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;IAIpC,2DAA2D;IACrD,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IA0BvD,2EAA2E;IACrE,GAAG,IAAI,OAAO,CAAC,MAAM,CAAC;IAK5B,6EAA6E;YAC/D,OAAO;IAUrB;;;OAGG;IACH,OAAO,CAAC,UAAU;IAQlB;;;OAGG;IACH,OAAO,CAAC,SAAS;IAMjB,yCAAyC;IACzC,OAAO,CAAC,UAAU;IAMlB,gFAAgF;IAChF,OAAO,CAAC,SAAS;IAKjB,uEAAuE;IACjE,IAAI,IAAI,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAwHtC,0EAA0E;YAC5D,gBAAgB;IAe9B,uEAAuE;YACzD,iBAAiB;IAoC/B;;;;OAIG;IACG,QAAQ,IAAI,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAmF1C,sEAAsE;YACxD,QAAQ;IAiBtB,sEAAsE;YACxD,cAAc;IAQ5B,0EAA0E;YAC5D,YAAY;IAiC1B,wDAAwD;IACxD,KAAK,IAAI,IAAI;CAMd"}