@harperfast/rocksdb-js 2.3.0 → 2.4.0

package/dist/index.mjs

@@ -3,7 +3,7 @@ import { execSync } from "node:child_process";
 import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, statSync } from "node:fs";
 import { dirname, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
-import { mkdir } from "node:fs/promises";
+import { access, cp, mkdir, readdir, rm } from "node:fs/promises";
 import * as orderedBinary from "ordered-binary";
 import { ExtendedIterable } from "@harperfast/extended-iterable";
 import { Encoder } from "msgpackr";
@@ -89,6 +89,21 @@ const coolTransactionLogs = binding.coolTransactionLogs;
 * the underlying mapping rather than leaving it retained.
 */
 const transactionLogMapCount = binding.transactionLogMapCount;
+/**
+* Creates a native file lock using the specified file path (`flock` on POSIX,
+* `LockFileEx` on Windows). Returns an opaque non-zero token to pass to
+* `fileLockRelease`, or `0` if another holder — in any process, container, or
+* worker thread — currently has it. Throws if `file` is missing or on a hard
+* error. The OS handle is owned entirely in native code (no fd crosses into
+* JS), and the kernel releases the lock when the handle closes, including on
+* process death.
+*/
+const tryFileLock = binding.tryFileLock;
+/**
+* Releases a file lock acquired via `tryFileLock`. A no-op for
+* token `0` or an unknown token.
+*/
+const fileLockRelease = binding.fileLockRelease;
 const nativeBackupRestore = binding.backupRestore;
 const nativeBackupList = binding.backupList;
 const nativeBackupDelete = binding.backupDelete;
@@ -99,6 +114,126 @@ const version = binding.version;
 
 //#endregion
 //#region src/backup.ts
+/** Subdirectory (under a backup directory) holding per-backup transaction log snapshots. */
+const TRANSACTION_LOGS_DIRNAME = "transaction_logs";
+/** Non-blocking existence check (`fs.existsSync` would block the event loop). */
+async function exists(path) {
+	try {
+		await access(path);
+		return true;
+	} catch {
+		return false;
+	}
+}
+/**
+* Runs `fn` while holding a native file lock for `backupDir`, releasing it when
+* `fn` settles. Used by the writable-engine operations `backups.delete` and
+* `backups.purge`; backup creation (`Store.backup`) takes the same lock on the
+* same file natively inside `Database::Backup` (see `runCreateBackup` in
+* `src/binding/database/backup.cpp`), where the backup directory is also
+* created. Throws immediately (without running `fn`) if another writer holds
+* the directory lock.
+*
+* Read-only operations (`list`, `verify`, and a restore's source read) use
+* `BackupEngineReadOnly` and are not locked: concurrent readers are safe.
+* Running a reader concurrently with a `delete`/`purge` on the same directory is
+* a caller-managed hazard, matching RocksDB's one-writable-engine-per-dir model.
+*
+* RocksDB only serializes work within a single `BackupEngine` instance and has
+* no lock on the backup directory itself. Two writable engines — in any
+* processes — creating/deleting/purging backups in the same directory
+* concurrently race on the per-backup staging directory and both fail,
+* potentially leaving the directory with no usable backup. An in-memory lock
+* cannot prevent this across processes, so the lock lives on disk: a kernel
+* advisory lock (`flock` on POSIX, `LockFileEx` on Windows) held on a
+* `.backup.lock` file at the directory root.
+*
+* The lock is taken entirely in native code (`tryFileLock`), which opens,
+* locks, and later closes the file's OS handle without ever exposing a
+* descriptor to JS — the addon statically links its own C runtime, so a
+* Node/libuv fd is not usable across that boundary. The kernel owning the lock
+* buys two properties a pidfile cannot:
+*
+* - No staleness heuristic. The lock is released when the holder's handle closes
+*   — normal release, crash, `kill -9`, container exit — so there is nothing to
+*   reclaim and no liveness check.
+* - True cross-context exclusion. The lock conflicts across processes, containers
+*   sharing a volume (same kernel), and `worker_threads`.
+*
+* The lock file is never deleted, only locked and unlocked; an unlocked, empty
+* `.backup.lock` at the directory root is the steady state.
+*/
+async function withBackupDirLock(backupDir, fn) {
+	const token = tryFileLock(join(backupDir, ".backup.lock"));
+	if (token === 0) throw new Error(`Backup directory is locked: ${backupDir}`);
+	try {
+		return await fn();
+	} finally {
+		fileLockRelease(token);
+	}
+}
+/**
+* Removes `<backupDir>/transaction_logs/<id>` subtrees whose backup id no longer
+* corresponds to a surviving backup. Called after a `purge` (which holds the
+* backup-dir lock); a no-op when no transaction logs were backed up. Self-healing:
+* it drops every log subtree without a live backup. Note the surviving set comes
+* from `nativeBackupList`, which omits corrupt backups — a still-present but
+* corrupt backup's logs may be pruned (acceptable: it cannot be restored anyway).
+* This also sweeps `.staging-*` leftovers from a crashed backup process (a
+* staging name never matches a live backup id); backup creation sweeps them
+* natively as well, under the backup-dir lock.
+*/
+async function pruneOrphanedTransactionLogs(backupDir) {
+	const logsRoot = join(backupDir, TRANSACTION_LOGS_DIRNAME);
+	let ids;
+	try {
+		ids = await readdir(logsRoot);
+	} catch (err) {
+		if (err.code === "ENOENT") return;
+		throw err;
+	}
+	const list = await new Promise((resolve, reject) => nativeBackupList(resolve, reject, backupDir));
+	const surviving = new Set(list.map((info) => String(info.backupId)));
+	await Promise.all(ids.filter((id) => !surviving.has(id)).map((id) => rm(join(logsRoot, id), {
+		recursive: true,
+		force: true
+	})));
+}
+/**
+* Restores the transaction log snapshot for the restored backup into
+* `<dbDir>/transaction_logs/`, wiping the destination first so that restoring an
+* older backup over a newer one cannot leave stale (newer) log files behind.
+*
+* Only acts when THIS backup actually captured logs: if the restored id has no
+* snapshot, the destination is left untouched. Wiping unconditionally would
+* destroy on-disk logs that a mixed-mode (`transactionLogs:false`) backup never
+* managed.
+*
+* Offline only: the files are picked up when the database is next opened. mtimes
+* are preserved because the store derives file age (rotation/retention) from
+* mtime — a fresh mtime would break retention.
+*/
+async function restoreTransactionLogs(backupDir, dbDir, backupId) {
+	const logsRoot = join(backupDir, TRANSACTION_LOGS_DIRNAME);
+	if (!await exists(logsRoot)) return;
+	let id = backupId;
+	if (id === void 0) {
+		const list = await new Promise((resolve, reject) => nativeBackupList(resolve, reject, backupDir));
+		if (list.length === 0) return;
+		id = Math.max(...list.map((info) => info.backupId));
+	}
+	const logsSrc = join(logsRoot, String(id));
+	if (!await exists(logsSrc)) return;
+	const logsDest = join(dbDir, TRANSACTION_LOGS_DIRNAME);
+	await rm(logsDest, {
+		recursive: true,
+		force: true
+	});
+	await cp(logsSrc, logsDest, {
+		recursive: true,
+		preserveTimestamps: true
+	});
+}
 /**
 * Backup management operations that act on a backup directory and do not
 * require an open database. To create a backup, use the `db.backup()` instance
@@ -124,11 +259,12 @@ const backups = {
 		const walDir = options?.walDir ?? dbDir;
 		await mkdir(dbDir, { recursive: true });
 		if (walDir !== dbDir) await mkdir(walDir, { recursive: true });
-		return new Promise((resolve, reject) => nativeBackupRestore(resolve, reject, backupDir, dbDir, walDir, {
+		await new Promise((resolve, reject) => nativeBackupRestore(resolve, reject, backupDir, dbDir, walDir, {
 			backupId: options?.backupId,
 			keepLogFiles: options?.keepLogFiles,
 			mode: options?.mode
 		}));
+		await restoreTransactionLogs(backupDir, dbDir, options?.backupId);
 	},
 	/**
 	* Lists the non-corrupt backups in a backup directory, ordered by id.
@@ -140,14 +276,27 @@ const backups = {
 	* Deletes a specific backup. Shared files are reference-counted and only
 	* removed once no remaining backup references them.
 	*/
-	delete(backupDir, backupId) {
-		return new Promise((resolve, reject) => nativeBackupDelete(resolve, reject, backupDir, backupId));
+	async delete(backupDir, backupId) {
+		return withBackupDirLock(backupDir, async () => {
+			await new Promise((resolve, reject) => nativeBackupDelete(resolve, reject, backupDir, backupId));
+			try {
+				await rm(join(backupDir, TRANSACTION_LOGS_DIRNAME, String(backupId)), {
+					recursive: true,
+					force: true
+				});
+			} catch {}
+		});
 	},
 	/**
 	* Deletes all but the newest `keepCount` backups.
 	*/
-	purge(backupDir, keepCount) {
-		return new Promise((resolve, reject) => nativeBackupPurge(resolve, reject, backupDir, keepCount));
+	async purge(backupDir, keepCount) {
+		return withBackupDirLock(backupDir, async () => {
+			await new Promise((resolve, reject) => nativeBackupPurge(resolve, reject, backupDir, keepCount));
+			try {
+				await pruneOrphanedTransactionLogs(backupDir);
+			} catch {}
+		});
 	},
 	/**
 	* Verifies a backup's file sizes, and optionally their checksums (which
@@ -518,6 +667,258 @@ var DBI = class DBI {
 	}
 };
 
+//#endregion
+//#region src/tar.ts
+/**
+* A minimal, dependency-free streaming USTAR (POSIX tar) encoder.
+*
+* Built for streaming whole-database backups: the set of live RocksDB files
+* (SSTs, blobs, MANIFEST, CURRENT, OPTIONS, WAL) is serialized into a single
+* tar byte stream that can be piped to any destination and unpacked with
+* standard tools (`tar -xf`). The encoder never buffers a whole file — each
+* entry's payload is streamed through in chunks — so peak memory stays flat
+* regardless of database size.
+*
+* Backpressure is the caller's lever: every method `await`s the {@link TarSink},
+* so wiring the sink to a stream writer's `write()` naturally paces production
+* to the consumer.
+*
+* Only what a RocksDB backup needs is implemented: regular files. Paths longer
+* than USTAR's 100-byte `name` field are split across the `name` and `prefix`
+* fields (~255 bytes total), which covers `transaction_logs/<store>/<file>`
+* entries; GNU/pax long-name extensions are not implemented, so a path that
+* still doesn't fit throws rather than being silently truncated. File sizes
+* larger than the octal `size` field can hold (~8 GiB) fall back to the GNU
+* base-256 encoding.
+*/
+/** Size of a tar block. Every header and the archive length are multiples of this. */
+const BLOCK_SIZE = 512;
+/** Maximum bytes for the USTAR `name` field. */
+const MAX_NAME_LENGTH = 100;
+/** Maximum bytes for the USTAR `prefix` field (holds the leading directories). */
+const MAX_PREFIX_LENGTH = 155;
+const textEncoder = new TextEncoder();
+const textDecoder = new TextDecoder();
+/**
+* Writes a UTF-8 string into a fixed-width header field, left-justified and
+* NUL-padded (the block is already zero-filled, so nothing else is needed).
+* Throws if the encoded value does not fit — tar fields are silently lossy
+* otherwise, which corrupts the archive.
+*/
+function writeField(block, value, offset, length) {
+	const bytes = textEncoder.encode(value);
+	if (bytes.length > length) throw new RangeError(`tar field value ${JSON.stringify(value)} exceeds ${length} bytes`);
+	block.set(bytes, offset);
+}
+/**
+* Writes a non-negative integer as a NUL-terminated, zero-padded octal string
+* into a `length`-byte field (so `length - 1` octal digits). Values too large
+* for octal fall back to the GNU base-256 extension, where the high bit of the
+* first byte is set and the magnitude is stored big-endian in the remainder.
+*/
+function writeOctal(block, value, offset, length) {
+	const digits = length - 1;
+	if (value > 8 ** digits - 1) {
+		let v = value;
+		for (let i = offset + length - 1; i > offset; i--) {
+			block[i] = v & 255;
+			v = Math.floor(v / 256);
+		}
+		block[offset] = 128;
+		return;
+	}
+	const str = value.toString(8).padStart(digits, "0");
+	for (let i = 0; i < digits; i++) block[offset + i] = str.charCodeAt(i);
+}
+/**
+* Computes and writes the USTAR header checksum: the unsigned sum of all 512
+* header bytes with the checksum field itself taken as ASCII spaces. Stored as
+* six octal digits, a NUL, then a space (the conventional encoding).
+*/
+function writeChecksum(block) {
+	for (let i = 148; i < 156; i++) block[i] = 32;
+	let sum = 0;
+	for (let i = 0; i < BLOCK_SIZE; i++) sum += block[i];
+	const str = sum.toString(8).padStart(6, "0");
+	for (let i = 0; i < 6; i++) block[148 + i] = str.charCodeAt(i);
+	block[154] = 0;
+	block[155] = 32;
+}
+/**
+* Splits a path into USTAR `name` (<=100 bytes) and `prefix` (<=155 bytes)
+* fields at a `/` boundary; a reader rejoins them as `prefix + "/" + name`.
+* Returns an empty prefix when the whole path fits in `name`. Throws when the
+* path cannot be represented — a trailing component over 100 bytes, or leading
+* directories that don't fit in 155. RocksDB and transaction-log filenames never
+* hit this; only a pathological log-store name would.
+*/
+function splitTarName(path) {
+	const encoded = textEncoder.encode(path);
+	if (encoded.length <= MAX_NAME_LENGTH) return {
+		name: path,
+		prefix: ""
+	};
+	let best = -1;
+	for (let i = 0; i < encoded.length; i++) {
+		if (encoded[i] !== 47) continue;
+		const prefixLength = i;
+		const nameLength = encoded.length - i - 1;
+		if (nameLength > 0 && nameLength <= MAX_NAME_LENGTH && prefixLength <= MAX_PREFIX_LENGTH) best = i;
+	}
+	if (best === -1) throw new RangeError(`tar entry name ${JSON.stringify(path)} does not fit the USTAR name (<=${MAX_NAME_LENGTH}) + prefix (<=${MAX_PREFIX_LENGTH}) fields`);
+	return {
+		name: textDecoder.decode(encoded.subarray(best + 1)),
+		prefix: textDecoder.decode(encoded.subarray(0, best))
+	};
+}
+/**
+* Builds a single 512-byte USTAR header block for a regular file. Long paths are
+* split across the `name` and `prefix` fields (see {@link splitTarName}).
+*/
+function buildHeader(name, size, mode, mtime) {
+	const block = new Uint8Array(BLOCK_SIZE);
+	const split = splitTarName(name);
+	writeField(block, split.name, 0, MAX_NAME_LENGTH);
+	writeOctal(block, mode & 4095, 100, 8);
+	writeOctal(block, 0, 108, 8);
+	writeOctal(block, 0, 116, 8);
+	writeOctal(block, size, 124, 12);
+	writeOctal(block, Math.floor(mtime), 136, 12);
+	block[156] = 48;
+	writeField(block, "ustar", 257, 6);
+	block[263] = 48;
+	block[264] = 48;
+	if (split.prefix.length > 0) writeField(block, split.prefix, 345, MAX_PREFIX_LENGTH);
+	writeChecksum(block);
+	return block;
+}
+/**
+* Streaming USTAR encoder. Usage is a strict sequence per file —
+* {@link TarEncoder.addFile} once, then {@link TarEncoder.writeData} until the
+* declared size is reached — followed by a single {@link TarEncoder.finalize}:
+*
+* ```ts
+* const tar = new TarEncoder((chunk) => writer.write(chunk));
+* await tar.addFile('CURRENT', current.length);
+* await tar.writeData(current);
+* await tar.addFile('000123.sst', sstSize);
+* for await (const chunk of sstChunks) await tar.writeData(chunk);
+* await tar.finalize();
+* ```
+*
+* The size passed to `addFile` is authoritative: tar records it in the header
+* before any payload, so it must be known up front (it is — RocksDB's
+* `GetLiveFilesStorageInfo` returns each file's exact size). Writing more or
+* fewer bytes than declared throws.
+*/
+var TarEncoder = class {
+	#sink;
+	/** Payload bytes still expected for the current entry. */
+	#remaining = 0;
+	/** Zero-padding bytes owed after the current entry's payload reaches a block boundary. */
+	#pad = 0;
+	#finalized = false;
+	constructor(sink) {
+		this.#sink = sink;
+	}
+	/**
+	* Emits the header for a new file entry. Must be preceded by completing any
+	* prior entry's payload. After this, exactly `size` bytes must be supplied
+	* via {@link TarEncoder.writeData} before the next `addFile` or `finalize`.
+	*/
+	async addFile(name, size, options) {
+		if (this.#finalized) throw new Error("TarEncoder already finalized");
+		if (this.#remaining !== 0) throw new Error(`previous tar entry incomplete: ${this.#remaining} byte(s) unwritten`);
+		if (!Number.isInteger(size) || size < 0) throw new RangeError("tar entry size must be a non-negative integer");
+		const header = buildHeader(name, size, options?.mode ?? 420, options?.mtime ?? 0);
+		this.#remaining = size;
+		this.#pad = (BLOCK_SIZE - size % BLOCK_SIZE) % BLOCK_SIZE;
+		await this.#sink(header);
+	}
+	/**
+	* Streams a chunk of the current entry's payload. Chunks may be any size; the
+	* total across all `writeData` calls for an entry must equal the size passed
+	* to {@link TarEncoder.addFile}. Block padding is emitted automatically once
+	* the final byte of an entry is written.
+	*/
+	async writeData(chunk) {
+		if (this.#finalized) throw new Error("TarEncoder already finalized");
+		if (chunk.length === 0) return;
+		if (chunk.length > this.#remaining) throw new RangeError(`tar data overruns entry size by ${chunk.length - this.#remaining} byte(s)`);
+		this.#remaining -= chunk.length;
+		await this.#sink(chunk);
+		if (this.#remaining === 0 && this.#pad > 0) {
+			const pad = this.#pad;
+			this.#pad = 0;
+			await this.#sink(new Uint8Array(pad));
+		}
+	}
+	/**
+	* Convenience for an entry whose contents are already fully in memory (e.g.
+	* the small `CURRENT`/`OPTIONS` files RocksDB returns inline).
+	*/
+	async addFileData(name, data, options) {
+		await this.addFile(name, data.length, options);
+		await this.writeData(data);
+	}
+	/**
+	* Writes the two zero-filled blocks that mark end-of-archive. The encoder is
+	* unusable afterward. Throws if the current entry is not fully written, so a
+	* truncated stream cannot be silently finalized into a valid-looking archive.
+	*/
+	async finalize() {
+		if (this.#finalized) throw new Error("TarEncoder already finalized");
+		if (this.#remaining !== 0) throw new Error(`cannot finalize: current tar entry has ${this.#remaining} byte(s) unwritten`);
+		this.#finalized = true;
+		await this.#sink(new Uint8Array(BLOCK_SIZE * 2));
+	}
+};
+
+//#endregion
+//#region src/backup-stream.ts
+/** Native event discriminator: a new file header (vs. a payload chunk). */
+const EVENT_FILE = 0;
+/**
+* Streams a consistent snapshot of the database to a `WritableStream` as a tar
+* archive, with no intermediate copy on disk. The set of live RocksDB files is
+* enumerated and read natively; each file header and payload chunk is framed
+* into the {@link TarEncoder} here in JS and written to `stream`.
+*
+* Backpressure flows end to end: the encoder awaits each `writer.write()`, and
+* the native producer awaits each `emit()` before reading the next chunk, so a
+* slow consumer paces the entire pipeline rather than buffering in memory.
+*
+* The resulting archive can be unpacked with any tar tool (`tar -xf`) into a
+* directory that opens directly as a RocksDB database.
+*/
+async function backupToStream(db, stream, options) {
+	let tarDestination;
+	let piped;
+	if (options?.gzip) {
+		const gzip = new CompressionStream("gzip");
+		piped = gzip.readable.pipeTo(stream);
+		tarDestination = gzip.writable;
+	} else tarDestination = stream;
+	const writer = tarDestination.getWriter();
+	const tar = new TarEncoder((chunk) => writer.write(chunk));
+	const emit = async (kind, data, size, mtime) => {
+		if (kind === EVENT_FILE) await tar.addFile(data, size, { mtime });
+		else await tar.writeData(data);
+	};
+	try {
+		await new Promise((resolve, reject) => {
+			db.backupStream(resolve, reject, emit, options);
+		});
+		await tar.finalize();
+		await writer.close();
+		if (piped) await piped;
+	} catch (err) {
+		await writer.abort(err).catch(() => {});
+		if (piped) await piped.catch(() => {});
+		throw err;
+	}
+}
+
 //#endregion
 //#region src/dbi-iterator.ts
 const { ITERATOR_RESULT_DONE, ITERATOR_RESULT_FAST } = constants;
@@ -900,19 +1301,9 @@ var Store = class {
 		}
 		return new Promise((resolve, reject) => this.db.compact(resolve, reject, startBuffer, endBuffer));
 	}
-	/**
-	* Creates a backup of the entire database (all column families) into the
-	* given directory and resolves with the new backup id. Parent directories
-	* are created as needed. See `backups` for restore and management.
-	*
-	* @example
-	* ```typescript
-	* const id = await db.backup('/path/to/backups');
-	* ```
-	*/
-	async backup(backupDir, options) {
-		await mkdir(backupDir, { recursive: true });
-		return new Promise((resolve, reject) => this.db.backup(resolve, reject, backupDir, options));
+	async backup(target, options) {
+		if (typeof target !== "string" && typeof target?.getWriter === "function") return backupToStream(this.db, target, options);
+		return new Promise((resolve, reject) => this.db.backup(resolve, reject, target, options));
 	}
 	/**
 	* Compacts the entire key range of the database synchronously.
@@ -1481,25 +1872,9 @@ var RocksDatabase = class RocksDatabase extends DBI {
 	compact(options) {
 		return this.store.compact(options);
 	}
-	/**
-	* Creates a backup of the entire database (all column families) into the
-	* given directory, creating parent directories as needed, and resolves with
-	* the new backup id. Use the `backups` namespace to restore, list, delete,
-	* purge, or verify backups.
-	*
-	* When the database was opened with `disableWAL`, the memtable is flushed
-	* before the backup by default so unflushed data is not lost.
-	*
-	* @example
-	* ```typescript
-	* import { backups } from '@harperfast/rocksdb-js';
-	*
-	* const db = RocksDatabase.open('/path/to/database');
-	* const id = await db.backup('/path/to/backups');
-	* ```
-	*/
-	backup(backupDir, options) {
-		return this.store.backup(backupDir, options);
+	backup(target, options) {
+		if (typeof target === "string") return this.store.backup(target, options);
+		return this.store.backup(target, options);
 	}
 	/**
 	* Creates a hardlinked, point-in-time, fully independent copy of the entire
@@ -2384,9 +2759,9 @@ function loadLastPosition(transactionLog, readUncommitted) {
 //#region src/index.ts
 const versions = {
 	rocksdb: version,
-	"rocksdb-js": "2.3.0"
+	"rocksdb-js": "2.4.0"
 };
 
 //#endregion
-export { DBI, DBIterator, RocksDatabase, Store, Transaction, TransactionLog, backups, constants, coolTransactionLogs, currentThreadId, parseTransactionLog, registryStatus, shutdown, stats, versions };
+export { DBI, DBIterator, RocksDatabase, Store, Transaction, TransactionLog, backups, constants, coolTransactionLogs, currentThreadId, fileLockRelease, parseTransactionLog, registryStatus, shutdown, stats, tryFileLock, versions };
 //# sourceMappingURL=index.mjs.map