@harperfast/rocksdb-js 2.1.0 → 2.3.0

package/dist/index.mjs

@@ -1,8 +1,9 @@
 import { createRequire } from "node:module";
 import { execSync } from "node:child_process";
-import { closeSync, openSync, readFileSync, readSync, readdirSync, statSync } from "node:fs";
+import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, statSync } from "node:fs";
 import { dirname, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
+import { mkdir } from "node:fs/promises";
 import * as orderedBinary from "ordered-binary";
 import { ExtendedIterable } from "@harperfast/extended-iterable";
 import { Encoder } from "msgpackr";
@@ -55,6 +56,7 @@ function locateBinding() {
 }
 const binding = req(locateBinding());
 const config = binding.config;
+const FRESH_VERSION_FLAG$1 = binding.constants.FRESH_VERSION_FLAG;
 const addGlobalListener = binding.addListener;
 const removeGlobalListener = binding.removeListener;
 const globalListenerCount = binding.listenerCount;
@@ -67,9 +69,95 @@ const TransactionLog = binding.TransactionLog;
 const registryStatus = binding.registryStatus;
 const shutdown = binding.shutdown;
 const currentThreadId = binding.currentThreadId;
+/**
+* Advises the kernel that the file-backed pages of every mapped transaction log
+* are cold (Linux MADV_COLD), so they are reclaimed first under memory pressure
+* without being freed — useful during replication catch-up, where a full read of
+* the logs would otherwise inflate the container's reclaimable cache toward its
+* cgroup limit. No-op on kernels < 5.4, macOS, and Windows.
+*
+* The transaction log registry is a process-global singleton shared across all
+* worker threads, so a single call cools every worker's maps. Call it on an
+* interval from one thread (e.g. an `unref()`ed timer on the main thread).
+*
+* @returns the number of maps cooled and total file-backed bytes advised.
+*/
+const coolTransactionLogs = binding.coolTransactionLogs;
+/**
+* Number of live transaction-log memory maps across the process. Internal —
+* used by tests to verify that releasing a frozen log's external buffer unmaps
+* the underlying mapping rather than leaving it retained.
+*/
+const transactionLogMapCount = binding.transactionLogMapCount;
+const nativeBackupRestore = binding.backupRestore;
+const nativeBackupList = binding.backupList;
+const nativeBackupDelete = binding.backupDelete;
+const nativeBackupPurge = binding.backupPurge;
+const nativeBackupVerify = binding.backupVerify;
 const stats = binding.stats;
 const version = binding.version;
 
+//#endregion
+//#region src/backup.ts
+/**
+* Backup management operations that act on a backup directory and do not
+* require an open database. To create a backup, use the `db.backup()` instance
+* method instead (it needs a live database for a consistent snapshot).
+*
+* @example
+* ```typescript
+* import { backups } from '@harperfast/rocksdb-js';
+*
+* const id = await db.backup('/path/to/backups');
+* db.close();
+* await backups.restore('/path/to/backups', '/path/to/restored-db');
+* ```
+*/
+const backups = {
+	/**
+	* Restores a database from a backup directory into a (closed) database
+	* directory. The default mode purges the destination directory, so it must
+	* not point at a live database.
+	*/
+	async restore(backupDir, dbDir, options) {
+		if (resolve(backupDir) === resolve(dbDir)) throw new Error("Backup directory and database directory must be different");
+		const walDir = options?.walDir ?? dbDir;
+		await mkdir(dbDir, { recursive: true });
+		if (walDir !== dbDir) await mkdir(walDir, { recursive: true });
+		return new Promise((resolve, reject) => nativeBackupRestore(resolve, reject, backupDir, dbDir, walDir, {
+			backupId: options?.backupId,
+			keepLogFiles: options?.keepLogFiles,
+			mode: options?.mode
+		}));
+	},
+	/**
+	* Lists the non-corrupt backups in a backup directory, ordered by id.
+	*/
+	list(backupDir) {
+		return new Promise((resolve, reject) => nativeBackupList(resolve, reject, backupDir));
+	},
+	/**
+	* Deletes a specific backup. Shared files are reference-counted and only
+	* removed once no remaining backup references them.
+	*/
+	delete(backupDir, backupId) {
+		return new Promise((resolve, reject) => nativeBackupDelete(resolve, reject, backupDir, backupId));
+	},
+	/**
+	* Deletes all but the newest `keepCount` backups.
+	*/
+	purge(backupDir, keepCount) {
+		return new Promise((resolve, reject) => nativeBackupPurge(resolve, reject, backupDir, keepCount));
+	},
+	/**
+	* Verifies a backup's file sizes, and optionally their checksums (which
+	* requires reading all backed-up data).
+	*/
+	verify(backupDir, backupId, options) {
+		return new Promise((resolve, reject) => nativeBackupVerify(resolve, reject, backupDir, backupId, options?.verifyWithChecksum ?? false));
+	}
+};
+
 //#endregion
 //#region src/util.ts
 /**
@@ -188,11 +276,11 @@ var DBI = class DBI {
 	*/
 	get(key, options) {
 		if (this.store.decoderCopies) return when(() => this.getBinaryFast(key, options), (result) => {
-			if (result === void 0) return;
+			if (result === void 0 || result === FRESH_VERSION_FLAG$1) return result;
 			if (options?.skipDecode) return result;
 			return this.store.decodeValue(result);
 		});
-		return when(() => this.getBinary(key, options), (result) => result === void 0 ? void 0 : this.store.encoding === "binary" || !this.store.decoder || options?.skipDecode ? result : this.store.decodeValue(result));
+		return when(() => this.getBinary(key, options), (result) => result === void 0 || result === FRESH_VERSION_FLAG$1 ? result : this.store.encoding === "binary" || !this.store.decoder || options?.skipDecode ? result : this.store.decodeValue(result));
 	}
 	/**
 	* Retrieves the binary data for the given key. This is just like `get()`,
@@ -202,7 +290,7 @@ var DBI = class DBI {
 	*/
 	getBinary(key, options) {
 		if (!this.store.isOpen()) return Promise.reject(/* @__PURE__ */ new Error("Database not open"));
-		return this.store.get(this._context, key, true, this.store.getTxnId(options));
+		return this.store.get(this._context, key, true, options);
 	}
 	/**
 	* Synchronously retrieves the binary data for the given key.
@@ -223,7 +311,7 @@ var DBI = class DBI {
 	*/
 	getBinaryFast(key, options) {
 		if (!this.store.isOpen()) return Promise.reject(/* @__PURE__ */ new Error("Database not open"));
-		return this.store.get(this._context, key, false, this.store.getTxnId(options));
+		return this.store.get(this._context, key, false, options);
 	}
 	/**
 	* Synchronously retrieves the binary data for the given key using a
@@ -282,12 +370,12 @@ var DBI = class DBI {
 	getSync(key, options) {
 		if (this.store.decoderCopies) {
 			const bytes = this.getBinaryFastSync(key, options);
-			return bytes === void 0 ? void 0 : this.store.decodeValue(bytes);
+			return bytes === void 0 || bytes === FRESH_VERSION_FLAG$1 ? bytes : this.store.decodeValue(bytes);
 		}
 		if (this.store.encoding === "binary") return this.getBinarySync(key, options);
 		if (this.store.decoder) {
 			const result = this.getBinarySync(key, options);
-			return result ? this.store.decodeValue(result) : void 0;
+			return !result || result === FRESH_VERSION_FLAG$1 ? result : this.store.decodeValue(result);
 		}
 		if (!this.store.isOpen()) throw new Error("Database not open");
 		return this.store.decodeValue(this.store.getSync(this._context, key, true, options));
@@ -450,17 +538,25 @@ var DBIterator = class {
 	iterator;
 	store;
 	#includeValues;
-	constructor(iterator, store, includeValues) {
+	#limit;
+	#yielded = 0;
+	constructor(iterator, store, includeValues, limit) {
 		this.iterator = iterator;
 		this.store = store;
 		this.#includeValues = includeValues;
+		this.#limit = limit ?? void 0;
 	}
 	[Symbol.iterator]() {
 		return this;
 	}
 	next() {
+		if (this.#limit !== void 0 && this.#yielded >= this.#limit) {
+			this.iterator.return?.();
+			return DONE_RESULT;
+		}
 		const result = this.iterator.next();
 		if (result === ITERATOR_RESULT_DONE) return DONE_RESULT;
+		this.#yielded++;
 		const includeValues = this.#includeValues;
 		const value = {};
 		if (result === ITERATOR_RESULT_FAST) {
@@ -562,7 +658,7 @@ function createFixedBuffer(size) {
 
 //#endregion
 //#region src/store.ts
-const { ONLY_IF_IN_MEMORY_CACHE_FLAG, NOT_IN_MEMORY_CACHE_FLAG, ALWAYS_CREATE_NEW_BUFFER_FLAG, ITERATOR_REVERSE_FLAG, ITERATOR_INCLUSIVE_END_FLAG, ITERATOR_EXCLUSIVE_START_FLAG, ITERATOR_INCLUDE_VALUES_FLAG, ITERATOR_NEEDS_STABLE_VALUE_BUFFER_FLAG, ITERATOR_CONTEXT_IS_TRANSACTION_FLAG } = constants;
+const { ONLY_IF_IN_MEMORY_CACHE_FLAG, NOT_IN_MEMORY_CACHE_FLAG, ALWAYS_CREATE_NEW_BUFFER_FLAG, FRESH_VERSION_FLAG, POPULATE_VERSION_FLAG, ITERATOR_REVERSE_FLAG, ITERATOR_INCLUSIVE_END_FLAG, ITERATOR_EXCLUSIVE_START_FLAG, ITERATOR_INCLUDE_VALUES_FLAG, ITERATOR_NEEDS_STABLE_VALUE_BUFFER_FLAG, ITERATOR_CONTEXT_IS_TRANSACTION_FLAG } = constants;
 const KEY_BUFFER_SIZE = 4096;
 const KEY_BUFFER = createFixedBuffer(KEY_BUFFER_SIZE);
 const VALUE_BUFFER = createFixedBuffer(64 * 1024);
@@ -721,6 +817,10 @@ var Store = class {
 	*/
 	transactionLogsPath;
 	/**
+	* Whether this store's column family participates in the VerificationTable.
+	*/
+	verificationTable;
+	/**
 	* The per-column-family memtable size in bytes at which the memtable is
 	* sealed and flushed.
 	*/
@@ -767,6 +867,7 @@ var Store = class {
 		this.transactionLogMaxSize = options?.transactionLogMaxSize;
 		this.transactionLogRetention = options?.transactionLogRetention;
 		this.transactionLogsPath = options?.transactionLogsPath;
+		this.verificationTable = options?.verificationTable;
 		this.writeBufferSize = options?.writeBufferSize;
 		this.writeKey = writeKey;
 	}
@@ -800,6 +901,20 @@ var Store = class {
 		return new Promise((resolve, reject) => this.db.compact(resolve, reject, startBuffer, endBuffer));
 	}
 	/**
+	* Creates a backup of the entire database (all column families) into the
+	* given directory and resolves with the new backup id. Parent directories
+	* are created as needed. See `backups` for restore and management.
+	*
+	* @example
+	* ```typescript
+	* const id = await db.backup('/path/to/backups');
+	* ```
+	*/
+	async backup(backupDir, options) {
+		await mkdir(backupDir, { recursive: true });
+		return new Promise((resolve, reject) => this.db.backup(resolve, reject, backupDir, options));
+	}
+	/**
 	* Compacts the entire key range of the database synchronously.
 	* This triggers manual compaction which removes tombstones and reclaims space.
 	*
@@ -872,16 +987,19 @@ var Store = class {
 		if (value instanceof Uint8Array) return value;
 		throw new Error(`Invalid value put in database (${typeof value}), consider using an encoder`);
 	}
-	get(context, key, alwaysCreateNewBuffer = false, txnId) {
+	get(context, key, alwaysCreateNewBuffer = false, options) {
 		const keyParam = getKeyParam(this.encodeKey(key));
 		let flags = 0;
 		if (alwaysCreateNewBuffer) flags |= ALWAYS_CREATE_NEW_BUFFER_FLAG;
-		if (this.readOnly) txnId = void 0;
-		const result = context.getSync(keyParam, flags | ONLY_IF_IN_MEMORY_CACHE_FLAG, txnId);
+		if (options?.populateVersion) flags |= POPULATE_VERSION_FLAG;
+		const txnId = this.getTxnId(options);
+		const expectedVersion = options?.expectedVersion;
+		const result = context.getSync(keyParam, flags | ONLY_IF_IN_MEMORY_CACHE_FLAG, txnId, expectedVersion);
 		if (typeof result === "number") {
 			if (result === NOT_IN_MEMORY_CACHE_FLAG) return new Promise((resolve, reject) => {
-				context.get(keyParam, resolve, reject, txnId);
+				context.get(keyParam, resolve, reject, txnId, expectedVersion);
 			});
+			if (result === FRESH_VERSION_FLAG) return result;
 			VALUE_BUFFER.end = result;
 			return VALUE_BUFFER;
 		}
@@ -948,14 +1066,16 @@ var Store = class {
 		if (includeValues && !this.decoderCopies) flags |= ITERATOR_NEEDS_STABLE_VALUE_BUFFER_FLAG;
 		if (context !== this.db) flags |= ITERATOR_CONTEXT_IS_TRANSACTION_FLAG;
 		const advancedOptions = options !== void 0 && (options.adaptiveReadahead !== void 0 || options.asyncIO !== void 0 || options.autoReadaheadSize !== void 0 || options.backgroundPurgeOnIteratorCleanup !== void 0 || options.fillCache !== void 0 || options.readaheadSize !== void 0 || options.tailing !== void 0) ? options : void 0;
-		return new ExtendedIterable(new DBIterator(new NativeIterator(context, flags, startKeyEnd, endKeyStart, endKeyEnd, advancedOptions), this, includeValues));
+		return new ExtendedIterable(new DBIterator(new NativeIterator(context, flags, startKeyEnd, endKeyStart, endKeyEnd, advancedOptions), this, includeValues, options?.limit));
 	}
 	getSync(context, key, alwaysCreateNewBuffer = false, options) {
 		const keyParam = getKeyParam(this.encodeKey(key));
 		let flags = 0;
 		if (alwaysCreateNewBuffer) flags |= ALWAYS_CREATE_NEW_BUFFER_FLAG;
-		const result = context.getSync(keyParam, flags, this.getTxnId(options));
+		if (options?.populateVersion) flags |= POPULATE_VERSION_FLAG;
+		const result = context.getSync(keyParam, flags, this.getTxnId(options), options?.expectedVersion);
 		if (typeof result === "number") {
+			if (result === FRESH_VERSION_FLAG) return result;
 			VALUE_BUFFER.end = result;
 			return VALUE_BUFFER;
 		}
@@ -1044,6 +1164,7 @@ var Store = class {
 			transactionLogMaxSize: this.transactionLogMaxSize,
 			transactionLogRetentionMs: this.transactionLogRetention ? parseDuration(this.transactionLogRetention) : void 0,
 			transactionLogsPath: this.transactionLogsPath,
+			verificationTable: this.verificationTable,
 			writeBufferSize: this.writeBufferSize
 		});
 		return false;
@@ -1093,6 +1214,25 @@ var Store = class {
 		return context.useLog(String(name));
 	}
 	/**
+	* Checks the process-global verification table for a fresh version match
+	* on `key`. Returns `true` when the table currently records `version` for
+	* this database+column-family. Provides a fast cache-freshness check
+	* before falling back to a full read.
+	*/
+	verifyVersion(key, version) {
+		const keyParam = getKeyParam(this.encodeKey(key));
+		return this.db.verifyVersion(keyParam, version);
+	}
+	/**
+	* Seeds the verification-table slot for `key` with `version`. Has no
+	* effect if the slot is currently lock-tagged or the table is disabled.
+	* Useful after a full read where the caller already knows the version.
+	*/
+	populateVersion(key, version) {
+		const keyParam = getKeyParam(this.encodeKey(key));
+		this.db.populateVersion(keyParam, version);
+	}
+	/**
 	* Acquires a lock on the given key and calls the callback.
 	*
 	* @param key - The key to lock.
@@ -1120,6 +1260,13 @@ function getKeyParam(keyBuffer) {
 
 //#endregion
 //#region src/transaction.ts
+/**
+* Sentinel value returned by `commit()` when `coordinatedRetry: true` and the
+* transaction encountered an IsBusy conflict. The native layer parks on VT
+* slots and resolves only after the conflicting transaction releases its write
+* intent, so callers should retry immediately without any backoff delay.
+*/
+const RETRY_NOW = constants.RETRY_NOW_VALUE;
 var TransactionAlreadyAbortedError = class extends Error {
 	code = "ERR_ALREADY_ABORTED";
 };
@@ -1180,13 +1327,16 @@ var Transaction = class extends DBI {
 	}
 	/**
 	* Commit the transaction.
+	*
+	* Returns `RETRY_NOW` when `coordinatedRetry: true` and an IsBusy conflict
+	* was detected. The caller should retry the transaction body immediately.
 	*/
 	async commit() {
 		try {
-			await new Promise((resolve, reject) => {
+			if (await new Promise((resolve, reject) => {
 				this.notify("beforecommit");
 				this.#txn.commit(resolve, reject);
-			});
+			}) === RETRY_NOW) return RETRY_NOW;
 		} catch (err) {
 			throw this.#handleCommitError(err);
 		} finally {
@@ -1332,6 +1482,60 @@ var RocksDatabase = class RocksDatabase extends DBI {
 		return this.store.compact(options);
 	}
 	/**
+	* Creates a backup of the entire database (all column families) into the
+	* given directory, creating parent directories as needed, and resolves with
+	* the new backup id. Use the `backups` namespace to restore, list, delete,
+	* purge, or verify backups.
+	*
+	* When the database was opened with `disableWAL`, the memtable is flushed
+	* before the backup by default so unflushed data is not lost.
+	*
+	* @example
+	* ```typescript
+	* import { backups } from '@harperfast/rocksdb-js';
+	*
+	* const db = RocksDatabase.open('/path/to/database');
+	* const id = await db.backup('/path/to/backups');
+	* ```
+	*/
+	backup(backupDir, options) {
+		return this.store.backup(backupDir, options);
+	}
+	/**
+	* Creates a hardlinked, point-in-time, fully independent copy of the entire
+	* database (all column families) at `targetPath` and resolves once written.
+	*
+	* Unlike a backup, a checkpoint is a normal, writable sibling database: open
+	* it with {@link RocksDatabase.open} and it diverges independently from the
+	* source. SST and blob files are hardlinked when `targetPath` is on the same
+	* filesystem as the database and copied otherwise (other files such as the
+	* MANIFEST are always copied), so the operation is near-instant on the same
+	* filesystem. The memtable is flushed so the checkpoint includes the latest
+	* writes even when the WAL is disabled.
+	*
+	* Parent directories are created as needed. `targetPath` itself must not
+	* already exist (RocksDB creates the checkpoint directory) and rejects with
+	* `Create checkpoint failed: target path exists` if it does. The caller is
+	* responsible for eventual cleanup of the directory.
+	*
+	* @example
+	* ```typescript
+	* const db = RocksDatabase.open('/path/to/database');
+	* await db.createCheckpoint('/path/to/checkpoint');
+	* const branch = RocksDatabase.open('/path/to/checkpoint');
+	* ```
+	*/
+	createCheckpoint(targetPath) {
+		return new Promise((resolve, reject) => {
+			if (existsSync(targetPath)) {
+				reject(/* @__PURE__ */ new Error("Create checkpoint failed: target path exists"));
+				return;
+			}
+			mkdirSync(dirname(targetPath), { recursive: true });
+			this.store.db.createCheckpoint(resolve, reject, targetPath);
+		});
+	}
+	/**
 	* Compacts the entire key range of the database synchronously.
 	* This triggers manual compaction which removes tombstones and reclaims space.
 	*
@@ -1563,6 +1767,30 @@ var RocksDatabase = class RocksDatabase extends DBI {
 	}
 	async ifNoExists(_key) {}
 	/**
+	* Checks the process-global verification table for a fresh version match
+	* on `key`. Returns `true` when the table currently records `version`. Use
+	* this as a fast cache-freshness check before falling back to a full read:
+	*
+	* ```typescript
+	* if (db.verifyVersion(key, cachedEntry.version)) {
+	*   return cachedEntry.value;
+	* }
+	* const value = db.getSync(key);
+	* db.populateVersion(key, extractVersion(value));
+	* ```
+	*/
+	verifyVersion(key, version) {
+		return this.store.verifyVersion(key, version);
+	}
+	/**
+	* Seeds the verification-table slot for `key` with `version`. Has no
+	* effect if the slot is currently lock-tagged or if the verification
+	* table is disabled.
+	*/
+	populateVersion(key, version) {
+		this.store.populateVersion(key, version);
+	}
+	/**
 	* Returns whether the database is open.
 	*
 	* @returns `true` if the database is open, `false` otherwise.
@@ -1702,9 +1930,6 @@ var RocksDatabase = class RocksDatabase extends DBI {
 	get path() {
 		return this.store.path;
 	}
-	/**
-	* Purges transaction logs.
-	*/
 	purgeLogs(options) {
 		return this.store.db.purgeLogs(options);
 	}
@@ -1740,13 +1965,19 @@ var RocksDatabase = class RocksDatabase extends DBI {
 			} catch (callbackErr) {
 				return this.#abortTransaction(txn, callbackErr);
 			}
+			let commitResult;
 			try {
-				await txn.commit();
-				return result;
+				commitResult = await txn.commit();
 			} catch (commitErr) {
 				if (commitErr instanceof TransactionAlreadyAbortedError) return;
-				if (commitErr instanceof TransactionIsBusyError && (options?.retryOnBusy ?? commitErr.hasLog) && attempt <= maxRetries) continue;
+				if (commitErr instanceof TransactionIsBusyError && (options?.retryOnBusy ?? commitErr.hasLog) && attempt < maxRetries) continue;
 				this.#abandonTransaction(txn, commitErr);
+				return;
+			}
+			if (commitResult !== RETRY_NOW) return result;
+			if (attempt >= maxRetries) {
+				this.#abandonTransaction(txn, /* @__PURE__ */ new Error(`Transaction did not commit after ${maxRetries} coordinated retries`));
+				return;
 			}
 		}
 	}
@@ -2059,7 +2290,7 @@ Object.defineProperty(TransactionLog.prototype, "query", { value({ start, end, e
 					done: true,
 					value: void 0
 				};
-				const limit = readUncommitted ? logBuffer.length : size;
+				const limit = readUncommitted ? logBuffer.length : Math.min(size, logBuffer.length);
 				if (position + TRANSACTION_LOG_ENTRY_HEADER_SIZE > limit) throw new RangeError(`Corrupt transaction log: truncated entry header at position ${position.toString(16)} of log ${logBuffer.logId} (available=${limit - position})`);
 				const length = dataView.getUint32(position + 8);
 				if (position + TRANSACTION_LOG_ENTRY_HEADER_SIZE + length > limit) throw new RangeError(`Corrupt transaction log entry at position ${position.toString(16)} of log ${logBuffer.logId}: declared length ${length} overruns the log (limit=${limit})`);
@@ -2153,9 +2384,9 @@ function loadLastPosition(transactionLog, readUncommitted) {
 //#region src/index.ts
 const versions = {
 	rocksdb: version,
-	"rocksdb-js": "2.1.0"
+	"rocksdb-js": "2.3.0"
 };
 
 //#endregion
-export { DBI, DBIterator, RocksDatabase, Store, Transaction, TransactionLog, constants, currentThreadId, parseTransactionLog, registryStatus, shutdown, stats, versions };
+export { DBI, DBIterator, RocksDatabase, Store, Transaction, TransactionLog, backups, constants, coolTransactionLogs, currentThreadId, parseTransactionLog, registryStatus, shutdown, stats, versions };
 //# sourceMappingURL=index.mjs.map