@harperfast/rocksdb-js 2.4.0 → 2.5.0

package/dist/index.mjs

@@ -1,7 +1,7 @@
 import { createRequire } from "node:module";
 import { execSync } from "node:child_process";
 import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, statSync } from "node:fs";
-import { dirname, join, resolve } from "node:path";
+import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { fileURLToPath } from "node:url";
 import { access, cp, mkdir, readdir, rm } from "node:fs/promises";
 import * as orderedBinary from "ordered-binary";
@@ -9,7 +9,7 @@ import { ExtendedIterable } from "@harperfast/extended-iterable";
 import { Encoder } from "msgpackr";
 
 //#region \0rolldown/runtime.js
-var __require = /* @__PURE__ */ createRequire(import.meta.url);
+var __require = /* #__PURE__ */ (() => createRequire(import.meta.url))();
 
 //#endregion
 //#region src/load-binding.ts
@@ -90,13 +90,21 @@ const coolTransactionLogs = binding.coolTransactionLogs;
 */
 const transactionLogMapCount = binding.transactionLogMapCount;
 /**
+* Test-only: force the next `count` transaction commits to fail with TryAgain (rolled back, so
+* no data is committed), reproducing a stranded-snapshot conflict deterministically. Pass 0 to
+* disarm. Used by the ERR_TRY_AGAIN retry regression test.
+*/
+const forceTryAgainForTesting = binding.forceTryAgainForTesting;
+/**
 * Creates a native file lock using the specified file path (`flock` on POSIX,
-* `LockFileEx` on Windows). Returns an opaque non-zero token to pass to
-* `fileLockRelease`, or `0` if another holder — in any process, container, or
-* worker thread — currently has it. Throws if `file` is missing or on a hard
-* error. The OS handle is owned entirely in native code (no fd crosses into
-* JS), and the kernel releases the lock when the handle closes, including on
-* process death.
+* `LockFileEx` on Windows), creating the file and any missing parent
+* directories. Exclusive by default; pass `shared` for a shared (reader) lock
+* that coexists with other shared holders but conflicts with an exclusive
+* holder in either direction. Returns an opaque non-zero token to pass to
+* `fileLockRelease`, or `0` if a conflicting holder — in any process,
+* container, or worker thread — currently has it. Throws on a hard error. The
+* OS handle is owned entirely in native code (no fd crosses into JS), and the
+* kernel releases the lock when the handle closes, including on process death.
 */
 const tryFileLock = binding.tryFileLock;
 /**
@@ -109,9 +117,51 @@ const nativeBackupList = binding.backupList;
 const nativeBackupDelete = binding.backupDelete;
 const nativeBackupPurge = binding.backupPurge;
 const nativeBackupVerify = binding.backupVerify;
+const nativeValidateTransactionLog = binding.validateTransactionLog;
 const stats = binding.stats;
 const version = binding.version;
 
+//#endregion
+//#region src/validate-transaction-log.ts
+/**
+* Validates a transaction log store directory: every `<sequence>.txnlog` file
+* is checked for a valid header (token, version) and intact v1 entry framing
+* (using the same scan as open-time crash recovery), file-name and sequence
+* continuity are checked, and the `txn.state` side file (when present) is
+* checked for shape and a plausible flushed position. The result reports
+* per-file entry counts, errors, and warnings; it does not throw when the
+* store is invalid — check `result.valid`.
+*
+* Runs entirely in native code on a worker thread and does not require (or
+* open) a database, so it works on the store of a closed database and on the
+* per-backup snapshots under `<backupDir>/transaction_logs/<backupId>/` alike.
+*
+* Validating a store that is being actively appended to can spuriously report
+* a torn tail for the current log file — the tail of an in-flight append is
+* indistinguishable from a crash artifact.
+*
+* Rejects if `path` does not exist or is not a directory (a bad argument, as
+* opposed to an invalid store).
+*
+* @param path The store directory, e.g. `<dbDir>/transaction_logs/<name>`.
+* @param options Validation options.
+*
+* @example
+* ```typescript
+* import { validateTransactionLogStore } from '@harperfast/rocksdb-js';
+*
+* const result = await validateTransactionLogStore('/path/to/db/transaction_logs/mylog');
+* if (!result.valid) {
+*   for (const file of result.files) {
+*     console.error(file.file, file.errors);
+*   }
+* }
+* ```
+*/
+function validateTransactionLogStore(path, options) {
+	return new Promise((resolve, reject) => nativeValidateTransactionLog(resolve, reject, path, options?.strict ?? false));
+}
+
 //#endregion
 //#region src/backup.ts
 /** Subdirectory (under a backup directory) holding per-backup transaction log snapshots. */
@@ -127,17 +177,29 @@ async function exists(path) {
 }
 /**
 * Runs `fn` while holding a native file lock for `backupDir`, releasing it when
-* `fn` settles. Used by the writable-engine operations `backups.delete` and
-* `backups.purge`; backup creation (`Store.backup`) takes the same lock on the
-* same file natively inside `Database::Backup` (see `runCreateBackup` in
+* `fn` settles. The lock is exclusive by default — used by the writable-engine
+* operations `backups.delete` and `backups.purge`; backup creation
+* (`Store.backup`) takes the same exclusive lock on the same file natively
+* inside `Database::Backup` (see `runCreateBackup` in
 * `src/binding/database/backup.cpp`), where the backup directory is also
-* created. Throws immediately (without running `fn`) if another writer holds
-* the directory lock.
+* created. Throws immediately (without running `fn`) if the backup directory
+* is missing — `tryFileLock` creates missing parents, but `delete`/`purge`/
+* `restore` on a nonexistent directory is a caller error and must not conjure
+* an empty one — or if a conflicting holder has the directory lock.
 *
-* Read-only operations (`list`, `verify`, and a restore's source read) use
-* `BackupEngineReadOnly` and are not locked: concurrent readers are safe.
-* Running a reader concurrently with a `delete`/`purge` on the same directory is
-* a caller-managed hazard, matching RocksDB's one-writable-engine-per-dir model.
+* `backups.restore` holds the lock in `shared` mode for its source read:
+* shared holders coexist (concurrent restores are safe), but a writer's
+* exclusive acquisition rejects while a restore is in flight — and vice versa.
+* Without this, a `purge`/`delete` racing a restore can delete the very files
+* the restore is copying, after the restore's default `purgeAllFiles` mode has
+* already wiped the destination — failing the restore with no fallback. A
+* rejected writer, by contrast, just retries later.
+*
+* The remaining read-only operations (`list`, `verify`) use
+* `BackupEngineReadOnly` and are not locked: concurrent readers are safe, and
+* locking them would make cheap listings reject during a long backup. Running
+* them concurrently with a `delete`/`purge` on the same directory is a
+* caller-managed hazard, matching RocksDB's one-writable-engine-per-dir model.
 *
 * RocksDB only serializes work within a single `BackupEngine` instance and has
 * no lock on the backup directory itself. Two writable engines — in any
@@ -163,8 +225,9 @@ async function exists(path) {
 * The lock file is never deleted, only locked and unlocked; an unlocked, empty
 * `.backup.lock` at the directory root is the steady state.
 */
-async function withBackupDirLock(backupDir, fn) {
-	const token = tryFileLock(join(backupDir, ".backup.lock"));
+async function withBackupDirLock(backupDir, fn, options) {
+	if (!await exists(backupDir)) throw new Error(`Backup directory does not exist: ${backupDir}`);
+	const token = tryFileLock(join(backupDir, ".backup.lock"), options?.shared === true);
 	if (token === 0) throw new Error(`Backup directory is locked: ${backupDir}`);
 	try {
 		return await fn();
@@ -173,6 +236,40 @@ async function withBackupDirLock(backupDir, fn) {
 	}
 }
 /**
+* Throws if `backupDir` resolves to the database directory itself or any path
+* beneath it. Backing up into the live database directory would write backup
+* files (the `.backup.lock`, `meta/`, `shared/`, `private/` subtrees) on top of
+* RocksDB's own files, and every subsequent backup would recursively capture
+* the prior ones. Both paths are resolved first so trailing slashes and
+* relative/absolute variants of the same location cannot slip past the check.
+*
+* Only the "backup dir at or under the database dir" direction is rejected —
+* the case the user hits by pointing a backup at `<db>` or `<db>/backups`. The
+* reverse (a database nested under the backup dir) is left to the caller: a
+* backup engine writes into named subtrees, not on top of arbitrary sibling
+* files, so it does not clobber the database in place.
+*
+* On case-insensitive filesystems `<db>` and `<db>/../DB/backup` are the same
+* directory on disk, but `relative()` compares case-sensitively and would call
+* the second one "outside". We case-fold both paths on the platforms whose
+* default filesystem is case-insensitive (macOS, Windows) so a casing
+* difference cannot slip a backup into the database directory. This is a
+* platform heuristic, not a per-volume probe (a case-sensitive macOS/Windows
+* volume exists), so it can over-reject a legitimately differently-cased
+* sibling there — the safe direction for a guard against clobbering the live
+* database.
+*/
+function assertBackupDirOutsideDatabase(dbPath, backupDir) {
+	let resolvedDb = resolve(dbPath);
+	let resolvedBackup = resolve(backupDir);
+	if (process.platform === "win32" || process.platform === "darwin") {
+		resolvedDb = resolvedDb.toLowerCase();
+		resolvedBackup = resolvedBackup.toLowerCase();
+	}
+	const rel = relative(resolvedDb, resolvedBackup);
+	if (!(rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel))) throw new Error(`Backup directory must not be inside the database directory: ${backupDir} is at or within ${dbPath}`);
+}
+/**
 * Removes `<backupDir>/transaction_logs/<id>` subtrees whose backup id no longer
 * corresponds to a surviving backup. Called after a `purge` (which holds the
 * backup-dir lock); a no-op when no transaction logs were backed up. Self-healing:
@@ -253,18 +350,32 @@ const backups = {
 	* Restores a database from a backup directory into a (closed) database
 	* directory. The default mode purges the destination directory, so it must
 	* not point at a live database.
+	*
+	* Holds the backup directory lock in shared mode for the duration: multiple
+	* restores can read concurrently, but a writer (`db.backup()`, `delete`,
+	* `purge`) rejects while a restore is in flight rather than deleting files
+	* out from under it — and a restore rejects while a writer holds the lock.
+	*
+	* The shared lock needs no write access to `backupDir`, so restoring from a
+	* read-only backup directory (immutable/WORM store, read-only-mounted volume)
+	* works: the lock is taken on an existing `.backup.lock` read-only, degrading
+	* to a no-op only when the media is read-only for every process (`EROFS`). A
+	* mere permission denial hard-fails instead — it doesn't prove a privileged
+	* writer isn't running.
 	*/
 	async restore(backupDir, dbDir, options) {
 		if (resolve(backupDir) === resolve(dbDir)) throw new Error("Backup directory and database directory must be different");
-		const walDir = options?.walDir ?? dbDir;
-		await mkdir(dbDir, { recursive: true });
-		if (walDir !== dbDir) await mkdir(walDir, { recursive: true });
-		await new Promise((resolve, reject) => nativeBackupRestore(resolve, reject, backupDir, dbDir, walDir, {
-			backupId: options?.backupId,
-			keepLogFiles: options?.keepLogFiles,
-			mode: options?.mode
-		}));
-		await restoreTransactionLogs(backupDir, dbDir, options?.backupId);
+		return withBackupDirLock(backupDir, async () => {
+			const walDir = options?.walDir ?? dbDir;
+			await mkdir(dbDir, { recursive: true });
+			if (walDir !== dbDir) await mkdir(walDir, { recursive: true });
+			await new Promise((resolve, reject) => nativeBackupRestore(resolve, reject, backupDir, dbDir, walDir, {
+				backupId: options?.backupId,
+				keepLogFiles: options?.keepLogFiles,
+				mode: options?.mode
+			}));
+			await restoreTransactionLogs(backupDir, dbDir, options?.backupId);
+		}, { shared: true });
 	},
 	/**
 	* Lists the non-corrupt backups in a backup directory, ordered by id.
@@ -301,9 +412,28 @@ const backups = {
 	/**
 	* Verifies a backup's file sizes, and optionally their checksums (which
 	* requires reading all backed-up data).
-	*/
-	verify(backupDir, backupId, options) {
-		return new Promise((resolve, reject) => nativeBackupVerify(resolve, reject, backupDir, backupId, options?.verifyWithChecksum ?? false));
+	*
+	* When the backup was created with `transactionLogs: true`, the backup's
+	* transaction log snapshot (`<backupDir>/transaction_logs/<backupId>/`) is
+	* also validated — every log file's header and entry framing must be intact
+	* (snapshots are copied on committed entry boundaries, so even a torn tail
+	* is a verification failure). Set `verifyTransactionLogs: false` to skip.
+	*/
+	async verify(backupDir, backupId, options) {
+		await new Promise((resolve, reject) => nativeBackupVerify(resolve, reject, backupDir, backupId, options?.verifyWithChecksum ?? false));
+		if (options?.verifyTransactionLogs === false) return;
+		const logsDir = join(backupDir, TRANSACTION_LOGS_DIRNAME, String(backupId));
+		if (!await exists(logsDir)) return;
+		const failures = [];
+		for (const entry of await readdir(logsDir, { withFileTypes: true })) {
+			if (!entry.isDirectory()) continue;
+			const result = await validateTransactionLogStore(join(logsDir, entry.name), { strict: true });
+			if (!result.valid) {
+				const details = [...result.errors, ...result.files.flatMap((file) => file.errors.map((error) => `${file.file}: ${error}`))];
+				failures.push(`${entry.name}: ${details.join("; ")}`);
+			}
+		}
+		if (failures.length > 0) throw new Error(`Backup ${backupId} transaction log verification failed: ${failures.join("\n")}`);
 	}
 };
 
@@ -1132,6 +1262,16 @@ var Store = class {
 	*/
 	maxKeySize;
 	/**
+	* The maximum number of table files RocksDB keeps open. `0` (the default)
+	* derives a budget from the effective per-process open-file limit (an
+	* eighth of the limit — several databases can share one process — clamped
+	* to `[1024, 262144]`); `-1` holds every table file open (the RocksDB
+	* default, which can exhaust the process file-descriptor limit when
+	* compaction falls behind under sustained ingest); a positive `int32` is
+	* an explicit cap.
+	*/
+	maxOpenFiles;
+	/**
 	* The maximum number of memtables that can be queued per column family
 	* before writes stall. Higher values absorb write bursts while flushes catch
 	* up, at the cost of memory.
@@ -1252,6 +1392,7 @@ var Store = class {
 		this.keyBuffer = KEY_BUFFER;
 		this.keyEncoding = keyEncoding;
 		this.maxKeySize = options?.maxKeySize ?? MAX_KEY_SIZE;
+		this.maxOpenFiles = options?.maxOpenFiles;
 		this.maxWriteBufferNumber = options?.maxWriteBufferNumber;
 		this.maxWriteBufferSizeToMaintain = options?.maxWriteBufferSizeToMaintain;
 		this.name = options?.name ?? "default";
@@ -1303,6 +1444,7 @@ var Store = class {
 	}
 	async backup(target, options) {
 		if (typeof target !== "string" && typeof target?.getWriter === "function") return backupToStream(this.db, target, options);
+		assertBackupDirOutsideDatabase(this.path, target);
 		return new Promise((resolve, reject) => this.db.backup(resolve, reject, target, options));
 	}
 	/**
@@ -1543,6 +1685,7 @@ var Store = class {
 			dbWriteBufferSize: this.dbWriteBufferSize,
 			disableWAL: this.disableWAL,
 			enableStats: this.enableStats,
+			maxOpenFiles: this.maxOpenFiles,
 			maxWriteBufferNumber: this.maxWriteBufferNumber,
 			maxWriteBufferSizeToMaintain: this.maxWriteBufferSizeToMaintain,
 			mode: this.pessimistic ? "pessimistic" : "optimistic",
@@ -1661,8 +1804,18 @@ const RETRY_NOW = constants.RETRY_NOW_VALUE;
 var TransactionAlreadyAbortedError = class extends Error {
 	code = "ERR_ALREADY_ABORTED";
 };
-var TransactionIsBusyError = class extends Error {
-	code = "ERR_BUSY";
+/**
+* Base for commit conflicts the native layer has already recovered from by resetting the
+* transaction in place (fresh RocksDB snapshot; `committedPosition` preserved, so a staged
+* transaction-log batch stays write-once): the caller should re-run the transaction body
+* and retry the commit — the handling is identical for every subclass. `hasLog` reports
+* whether the transaction wrote to a transaction log (drives the default retry decision).
+*
+* {@link TransactionAbandonedError} shares this field shape but deliberately does NOT
+* extend this class — an abandoned transaction is not retryable, and retryability is the
+* boundary this hierarchy expresses.
+*/
+var TransactionRetryableError = class extends Error {
 	hasLog;
 	txn;
 	constructor(error, txn) {
@@ -1671,6 +1824,21 @@ var TransactionIsBusyError = class extends Error {
 		this.txn = txn;
 	}
 };
+/**
+* An optimistic commit detected a write conflict with a concurrently committed
+* transaction (RocksDB `Busy`).
+*/
+var TransactionIsBusyError = class extends TransactionRetryableError {
+	code = "ERR_BUSY";
+};
+/**
+* An optimistic commit could not be validated: the transaction's snapshot was stranded
+* outside the memtable window (its sequence history was flushed away — RocksDB
+* `TryAgain`), so the conflict check had nothing to validate against.
+*/
+var TransactionTryAgainError = class extends TransactionRetryableError {
+	code = "ERR_TRY_AGAIN";
+};
 var TransactionAbandonedError = class extends Error {
 	code = "ERR_TRANSACTION_ABANDONED";
 	hasLog;
@@ -1765,6 +1933,7 @@ var Transaction = class extends DBI {
 		if (err instanceof Error && "code" in err) {
 			if (err.code === "ERR_ALREADY_ABORTED") return new TransactionAlreadyAbortedError(err.message);
 			if (err.code === "ERR_BUSY") return new TransactionIsBusyError(err, this);
+			if (err.code === "ERR_TRY_AGAIN") return new TransactionTryAgainError(err, this);
 		}
 		return err;
 	}
@@ -2345,7 +2514,7 @@ var RocksDatabase = class RocksDatabase extends DBI {
 				commitResult = await txn.commit();
 			} catch (commitErr) {
 				if (commitErr instanceof TransactionAlreadyAbortedError) return;
-				if (commitErr instanceof TransactionIsBusyError && (options?.retryOnBusy ?? commitErr.hasLog) && attempt < maxRetries) continue;
+				if (commitErr instanceof TransactionRetryableError && (options?.retryOnBusy ?? commitErr.hasLog) && attempt < maxRetries) continue;
 				this.#abandonTransaction(txn, commitErr);
 				return;
 			}
@@ -2377,7 +2546,7 @@ var RocksDatabase = class RocksDatabase extends DBI {
 		if (typeof callback !== "function") throw new TypeError("Callback must be a function");
 		const maxRetries = options?.maxRetries ?? 3;
 		const isRetryable = (err, attempt) => {
-			return err instanceof TransactionIsBusyError && (options?.retryOnBusy ?? err.hasLog) && attempt <= maxRetries;
+			return err instanceof TransactionRetryableError && (options?.retryOnBusy ?? err.hasLog) && attempt <= maxRetries;
 		};
 		const runAttempt = (attempt) => {
 			const txn = new Transaction(this.store, options);
@@ -2573,7 +2742,7 @@ function parseTransactionLog(path, options = {}) {
 
 //#endregion
 //#region src/transaction-log-reader.ts
-const FLOAT_TO_UINT32 = new Float64Array(1);
+const FLOAT_TO_UINT32 = /* @__PURE__ */ new Float64Array(1);
 const UINT32_FROM_FLOAT = new Uint32Array(FLOAT_TO_UINT32.buffer);
 const { TRANSACTION_LOG_FILE_HEADER_SIZE, TRANSACTION_LOG_ENTRY_HEADER_SIZE } = constants;
 /**
@@ -2759,9 +2928,9 @@ function loadLastPosition(transactionLog, readUncommitted) {
 //#region src/index.ts
 const versions = {
 	rocksdb: version,
-	"rocksdb-js": "2.4.0"
+	"rocksdb-js": "2.5.0"
 };
 
 //#endregion
-export { DBI, DBIterator, RocksDatabase, Store, Transaction, TransactionLog, backups, constants, coolTransactionLogs, currentThreadId, fileLockRelease, parseTransactionLog, registryStatus, shutdown, stats, tryFileLock, versions };
+export { DBI, DBIterator, RocksDatabase, Store, Transaction, TransactionLog, backups, constants, coolTransactionLogs, currentThreadId, fileLockRelease, parseTransactionLog, registryStatus, shutdown, stats, tryFileLock, validateTransactionLogStore, versions };
 //# sourceMappingURL=index.mjs.map