@platformatic/sqlite-js-stream 0.0.0-stage → 0.1.1

package/package.json

@@ -1,6 +1,71 @@
 {
   "name": "@platformatic/sqlite-js-stream",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.1.1",
+  "description": "Node.js port of Litestream: SQLite streaming replication & disaster recovery via LTX files",
+  "type": "module",
+  "engines": {
+    "node": ">=24"
+  },
+  "license": "Apache-2.0",
+  "author": "Matteo Collina",
+  "homepage": "https://github.com/platformatic/sqlite-js-stream",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/platformatic/sqlite-js-stream.git"
+  },
+  "bugs": {
+    "url": "https://github.com/platformatic/sqlite-js-stream/issues"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist",
+    "src",
+    "README.md",
+    "LICENSE",
+    "NOTICE",
+    "CHANGELOG.md"
+  ],
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org"
+  },
+  "dependencies": {
+    "lz4js": "^0.2.0"
+  },
+  "devDependencies": {
+    "@types/node": "^24.0.0",
+    "eslint": "^9.39.5",
+    "neostandard": "^0.13.0",
+    "typescript": "^5.7.0"
+  },
+  "keywords": [
+    "sqlite",
+    "litestream",
+    "replication",
+    "backup",
+    "ltx",
+    "disaster-recovery"
+  ],
+  "scripts": {
+    "test": "node --test 'test/**/*.test.ts'",
+    "pretypecheck": "pnpm build",
+    "typecheck": "tsc -p . --noEmit",
+    "lint": "eslint .",
+    "lint:fix": "eslint . --fix",
+    "build": "tsc -p tsconfig.build.json && node scripts/rewrite-dts.mjs dist",
+    "golden:go": "cd tools/golden && go run .",
+    "pretest:all": "pnpm build",
+    "test:all": "pnpm test && pnpm -r test",
+    "typecheck:all": "pnpm typecheck",
+    "bench": "node bench/bench-sync-restore.ts"
+  }
 }

package/src/backends/file/index.ts

@@ -0,0 +1,3 @@
+/** File backend. */
+export * from './replica-client.ts';
+export * from './leaser.ts';

package/src/backends/file/leaser.ts

@@ -0,0 +1,145 @@
+/**
+ * File-backend Leaser.
+ *
+ * Stores the lease as `<path>/lock.json` and uses an exclusive create + a
+ * content hash as the conditional-write token. The file backend is primarily
+ * for local/single-host use; for real multi-host safety use the S3 leaser
+ * (which uses S3 `If-None-Match`/`If-Match` conditional writes).
+ */
+
+import { createHash } from 'node:crypto';
+import { link, readFile, rename, rm, writeFile } from 'node:fs/promises';
+import nodePath from 'node:path';
+
+import {
+  DefaultLeasePath,
+  DefaultLeaseTTL,
+  ErrLeaseNotHeld,
+  Lease,
+  LeaseExistsError,
+  type Leaser,
+} from '../../leaser.ts';
+import { FileReplicaError } from '../../errors.ts';
+
+// Disambiguates temp files written in the same millisecond by this process.
+let tmpSeq = 0;
+
+function sha256(data: Uint8Array): string {
+  return createHash('sha256').update(data).digest('hex');
+}
+
+function encodeLease(l: Lease): Uint8Array {
+  return new TextEncoder().encode(
+    JSON.stringify({
+      generation: l.generation,
+      expires_at: new Date(l.expiresAt).toISOString(),
+      owner: l.owner,
+    }),
+  );
+}
+
+function decodeLease(data: Uint8Array, etag: string): Lease {
+  const parsed = JSON.parse(new TextDecoder().decode(data)) as {
+    generation?: number;
+    expires_at?: string;
+    owner?: string;
+  };
+  const expiresAt = Date.parse(parsed.expires_at ?? '');
+  if (Number.isNaN(expiresAt)) {
+    throw new FileReplicaError('invalid lease file: bad expires_at');
+  }
+  return new Lease(parsed.generation ?? 1, expiresAt, parsed.owner ?? '', etag);
+}
+
+/** Reads and decodes the lease file, returning null if it does not exist. */
+async function readLease(lockPath: string): Promise<Lease | null> {
+  try {
+    const data = await readFile(lockPath);
+    return decodeLease(data, sha256(data));
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null;
+    throw err;
+  }
+}
+
+/** FileLeaser implements {@link Leaser} for a local directory. */
+export class FileLeaser implements Leaser {
+  private lockPath: string;
+  private ttlMs: number;
+
+  constructor(path: string, lockName = DefaultLeasePath, ttlMs = DefaultLeaseTTL) {
+    this.lockPath = nodePath.join(path, lockName);
+    this.ttlMs = ttlMs;
+  }
+
+  async acquireLease(): Promise<Lease> {
+    const existing = await readLease(this.lockPath);
+    if (existing !== null && !existing.isExpired()) {
+      throw new LeaseExistsError(existing.owner, existing.expiresAt);
+    }
+
+    let generation = 1;
+    if (existing !== null) generation = existing.generation + 1;
+
+    const lease = new Lease(generation, Date.now() + this.ttlMs, this.owner());
+    const bytes = encodeLease(lease);
+
+    if (existing === null) {
+      // Exclusive create so we do not clobber a concurrent acquirer. Write a
+      // temp file and hard-link it into place so the lock appears atomically
+      // with its full contents; a racing reader never sees an empty file.
+      const tmp = `${this.lockPath}.tmp-${process.pid}-${Date.now()}-${tmpSeq++}`;
+      await writeFile(tmp, bytes);
+      try {
+        await link(tmp, this.lockPath);
+      } catch (err) {
+        const code = (err as NodeJS.ErrnoException).code;
+        if (code === 'EEXIST') {
+          const current = await readLease(this.lockPath);
+          if (current !== null && !current.isExpired()) {
+            throw new LeaseExistsError(current.owner, current.expiresAt);
+          }
+          // The other writer lost the race; retry the expired-overwrite path.
+          return this.acquireLease();
+        }
+        throw err;
+      } finally {
+        await rm(tmp, { force: true });
+      }
+    } else {
+      // Existing lease expired: atomically replace it.
+      const tmp = `${this.lockPath}.tmp-${process.pid}-${Date.now()}-${tmpSeq++}`;
+      await writeFile(tmp, bytes);
+      await rename(tmp, this.lockPath);
+    }
+
+    const written = await readLease(this.lockPath);
+    return written ?? lease;
+  }
+
+  async renewLease(lease: Lease): Promise<Lease> {
+    const current = await readLease(this.lockPath);
+    if (current === null) throw new ErrLeaseNotHeld();
+    if (current.etag !== lease.etag) throw new ErrLeaseNotHeld();
+
+    const renewed = new Lease(lease.generation, Date.now() + this.ttlMs, lease.owner);
+    const bytes = encodeLease(renewed);
+    const tmp = `${this.lockPath}.tmp-${process.pid}-${Date.now()}-${tmpSeq++}`;
+    await writeFile(tmp, bytes);
+    await rename(tmp, this.lockPath);
+
+    const written = await readLease(this.lockPath);
+    return written ?? renewed;
+  }
+
+  async releaseLease(lease: Lease): Promise<void> {
+    const current = await readLease(this.lockPath);
+    if (current === null) return; // already released
+    if (current.etag !== lease.etag) throw new ErrLeaseNotHeld();
+    await rm(this.lockPath, { force: true });
+  }
+
+  private owner(): string {
+    return `${process.pid}`;
+  }
+}

package/src/backends/file/replica-client.ts

@@ -0,0 +1,246 @@
+/**
+ * File replica client — replicates LTX files to a local directory.
+ *
+ * Layout: `<root>/ltx/<level>/<minTXID>-<maxTXID>.ltx`
+ * Writes are atomic: data is streamed to a `.tmp` file, fsync'd, then renamed,
+ * the directory is fsync'd, and the file mtime is set from the LTX header
+ * timestamp to preserve original creation time.
+ */
+
+import { createReadStream } from 'node:fs';
+import { open, readdir, rename, rm, stat, utimes, type FileHandle } from 'node:fs/promises';
+import nodePath from 'node:path';
+import { Readable } from 'node:stream';
+
+import {
+  ltxFilePath as sharedLTXFilePath,
+  ltxLevelDir as sharedLTXLevelDir,
+  LTXError,
+  parseFilename,
+  type OpenLTXFileOptions,
+  type ReplicaClient,
+  type WriteLTXFileOptions,
+} from '../../replica-client.ts';
+import type { FileInfo, Header } from '../../ltx/index.ts';
+import { HeaderSize, unmarshalHeader } from '../../ltx/index.ts';
+import { concatBytes } from '../../util/stream.ts';
+import { fsyncDir, mkdirAll } from '../../util/fs.ts';
+import type { Logger } from '../../util/logger.ts';
+import { FileReplicaError, InvalidArgumentError } from '../../errors.ts';
+import { noopLogger } from '../../util/logger.ts';
+import { registerReplicaClientFactory } from '../../replica-url.ts';
+
+export const ReplicaClientType = 'file';
+
+/** FileReplicaClient writes LTX files to a local directory. */
+export class FileReplicaClient implements ReplicaClient {
+  private pathVal: string;
+  private logger: Logger = noopLogger;
+
+  constructor(path: string) {
+    this.pathVal = path;
+  }
+
+  type(): string {
+    return ReplicaClientType;
+  }
+
+  async init(): Promise<void> {
+    /* no initialization required */
+  }
+
+  setLogger(logger: Logger): void {
+    this.logger = logger;
+  }
+
+  /** Returns the destination root path. */
+  path(): string {
+    return this.pathVal;
+  }
+
+  ltxLevelDir(level: number): string {
+    return sharedLTXLevelDir(this.pathVal, level);
+  }
+
+  ltxFilePath(level: number, minTXID: bigint, maxTXID: bigint): string {
+    return sharedLTXFilePath(this.pathVal, level, minTXID, maxTXID);
+  }
+
+  async *ltxFiles(opts: { level: number; seek: bigint; useMetadata: boolean }): AsyncIterable<FileInfo> {
+    const dir = this.ltxLevelDir(opts.level);
+    let entries;
+    try {
+      entries = await readdir(dir, { withFileTypes: true });
+    } catch (err) {
+      if ((err as NodeJS.ErrnoException).code === 'ENOENT') return;
+      throw err;
+    }
+
+    const infos: FileInfo[] = [];
+    for (const entry of entries) {
+      if (!entry.isFile()) continue;
+      let minTXID: bigint;
+      let maxTXID: bigint;
+      try {
+        ({ minTXID, maxTXID } = parseFilename(entry.name));
+      } catch {
+        continue; // skip non-conforming files
+      }
+      if (minTXID < opts.seek) continue;
+
+      const full = nodePath.join(dir, entry.name);
+      const st = await stat(full);
+      infos.push({
+        level: opts.level,
+        minTXID,
+        maxTXID,
+        size: st.size,
+        createdAt: st.mtimeMs,
+      });
+    }
+
+    infos.sort((a, b) => {
+      if (a.minTXID !== b.minTXID) return a.minTXID < b.minTXID ? -1 : 1;
+      return a.maxTXID < b.maxTXID ? -1 : a.maxTXID > b.maxTXID ? 1 : 0;
+    });
+
+    for (const info of infos) yield info;
+  }
+
+  async openLtxFile(opts: OpenLTXFileOptions): Promise<Readable> {
+    const p = this.ltxFilePath(opts.level, opts.minTXID, opts.maxTXID);
+    let st;
+    try {
+      st = await stat(p);
+    } catch (err) {
+      throw new LTXError('open', p, opts.level, opts.minTXID, opts.maxTXID, err);
+    }
+
+    if (opts.offset >= st.size) {
+      return Readable.from([]);
+    }
+
+    const end =
+      opts.size > 0
+        ? Math.min(opts.offset + opts.size - 1, st.size - 1)
+        : st.size - 1;
+
+    return createReadStream(p, { start: opts.offset, end });
+  }
+
+  async writeLtxFile(opts: WriteLTXFileOptions, body: AsyncIterable<Uint8Array | string>): Promise<FileInfo> {
+    const dir = this.ltxLevelDir(opts.level);
+    await mkdirAll(dir);
+
+    const filename = this.ltxFilePath(opts.level, opts.minTXID, opts.maxTXID);
+    const tmp = `${filename}.tmp`;
+
+    let fh: FileHandle | null = null;
+    try {
+      fh = await open(tmp, 'w');
+    } catch (err) {
+      throw new LTXError('create', tmp, opts.level, opts.minTXID, opts.maxTXID, err);
+    }
+
+    let headerBytes: Uint8Array | null = null;
+    let collected: Uint8Array[] = [];
+    let collectedLen = 0;
+    let size = 0;
+
+    try {
+      for await (const chunk of body) {
+        const bytes =
+          typeof chunk === 'string' ? new TextEncoder().encode(chunk) : Uint8Array.from(chunk);
+        if (headerBytes === null) {
+          collected.push(bytes);
+          collectedLen += bytes.length;
+          if (collectedLen >= HeaderSize) {
+            const concat = concatBytes(collected);
+            headerBytes = concat.subarray(0, HeaderSize);
+            collected = [];
+            collectedLen = 0;
+          }
+        }
+        await writeAll(fh, bytes);
+        size += bytes.length;
+      }
+
+      if (headerBytes === null) {
+        throw new FileReplicaError('write ltx file: input too short to contain a header');
+      }
+      const hdr: Header = unmarshalHeader(headerBytes);
+      const timestamp = Number(hdr.timestamp);
+
+      await fh.sync();
+      await fh.close();
+      fh = null;
+
+      await rename(tmp, filename);
+      await fsyncDir(dir);
+      await utimes(filename, new Date(timestamp), new Date(timestamp));
+
+      return {
+        level: opts.level,
+        minTXID: opts.minTXID,
+        maxTXID: opts.maxTXID,
+        size,
+        createdAt: timestamp,
+      };
+    } catch (err) {
+      if (fh !== null) await fh.close().catch(() => {});
+      await rm(tmp, { force: true }).catch(() => {});
+      throw err;
+    }
+  }
+
+  async deleteLtxFiles(infos: FileInfo[]): Promise<void> {
+    for (const info of infos) {
+      const p = this.ltxFilePath(info.level, info.minTXID, info.maxTXID);
+      this.logger.debug('deleting ltx file', {
+        level: info.level,
+        minTXID: info.minTXID,
+        maxTXID: info.maxTXID,
+        path: p,
+      });
+      try {
+        await rm(p);
+      } catch (err) {
+        if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err;
+      }
+    }
+  }
+
+  async deleteAll(): Promise<void> {
+    try {
+      await rm(this.pathVal, { recursive: true, force: true });
+    } catch (err) {
+      if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err;
+    }
+  }
+}
+
+/** NewReplicaClientFromURL creates a FileReplicaClient from a URL path. */
+export function NewReplicaClientFromURL(
+  _scheme: string,
+  _host: string,
+  urlPath: string,
+  _query: URLSearchParams | null,
+  _userinfo: string | null,
+): ReplicaClient {
+  if (!urlPath) {
+    throw new InvalidArgumentError('file replica path required');
+  }
+  return new FileReplicaClient(urlPath);
+}
+
+// Mirrors Go's init(): register the file factory when this module loads.
+registerReplicaClientFactory('file', NewReplicaClientFromURL);
+
+/** Writes all of `bytes` to a FileHandle, handling partial writes. */
+async function writeAll(handle: FileHandle, bytes: Uint8Array): Promise<void> {
+  let offset = 0;
+  while (offset < bytes.length) {
+    const { bytesWritten } = await handle.write(bytes, offset, bytes.length - offset);
+    offset += bytesWritten;
+  }
+}