@harperfast/rocksdb-js 2.4.1 → 2.6.0

package/README.md

@@ -10,6 +10,7 @@ A Node.js binding for the RocksDB library.
 - Transaction log system for recording transaction related data
 - Custom stores provide ability to override default database interactions
 - Efficient binary key and value encoding
+- Configurable block/blob compression (LZ4, Zstd, Zlib, and more)
 - Access to internal RocksDB statistics
 - Designed for Node.js and Bun on Linux, macOS, and Windows
 
@@ -43,10 +44,26 @@ Creates a new database instance.
 - `path: string` The path to write the database files to. This path does not need to exist, but the
   parent directories do.
 - `options: object` [optional]
+  - `compression: string | { algorithm: string, level?: number }` The block/blob compression
+    algorithm for this column family. Pass an algorithm name — one of `'none'`, `'snappy'`,
+    `'zlib'`, `'bzip2'`, `'lz4'`, `'lz4hc'`, or `'zstd'` — or an object with an
+    `algorithm` and an optional `level` (forwarded to RocksDB's `compression_opts.level`; the
+    meaning is algorithm-specific). Applies to both SST data blocks and blob files (large values).
+    Defaults to `'lz4'` when the native build supports it, otherwise RocksDB's own default (Snappy
+    when linked, else no compression). See [Compression](#compression). Throws if the algorithm is
+    not compiled into the native build — check [`supportedCompression`](#supportedcompression) for
+    the available list.
   - `disableWAL: boolean` Whether to disable the RocksDB write ahead log. Defaults to `false`.
   - `enableStats: boolean` When `true` and the database is open, RocksDB will captures stats that
     are retrieved by calling `db.getStats()`. Enabling statistics imposes 5-10% in overhead.
     Defaults to `false`.
+  - `maxOpenFiles: number` The maximum number of table files RocksDB keeps open. `0` (the default)
+    derives a budget from the effective per-process open-file limit (an eighth of the limit —
+    several databases can share one process — clamped to `[1024, 262144]`); `-1` holds every table
+    file open (the RocksDB default, which can exhaust the process file-descriptor limit when
+    compaction falls behind under sustained ingest); a positive `int32` is an explicit cap. Reads
+    only pay a reopen cost when the number of live table files exceeds the budget, so raise the
+    process fd limit (and with it the derived budget) for very large databases.
   - `name: string` The column family name. Defaults to `"default"`.
   - `noBlockCache: boolean` When `true`, disables the block cache. Block caching is enabled by
     default and the cache is shared across all database instances.
@@ -60,7 +77,8 @@ Creates a new database instance.
   - `statsLevel: StatsLevel` Controls which type of statistics to skip and reduce statistic
     overhead. Defaults to `StatsLevel.ExceptDetailedTimers`.
   - `store: Store` A custom store that handles all interaction between the `RocksDatabase` or
-    `Transaction` instances and the native database interface. See [Custom Store](#custom-store) for
+    `Transaction` instances and the native database interface. A store is bound to a single
+    `RocksDatabase` instance and cannot be shared between them. See [Custom Store](#custom-store) for
     more information.
   - `transactionLogMaxAgeThreshold: number` The threshold for the transaction log file's last
     modified time to be older than the retention period before it is rotated to the next sequence
@@ -102,6 +120,18 @@ const db2 = new RocksDatabase('path/to/db', { name: 'users' });
 console.log(db.columns); // ['default', 'users']
 ```
 
+### `db.compression: { algorithm: string, level?: number }`
+
+Returns the compression currently in effect for this database's column family, read live from
+RocksDB. `algorithm` is a friendly name (e.g. `'lz4'`, `'zstd'`, `'none'`); `level` is present only
+when a non-default compression level is set. The database must be open. See
+[Compression](#compression).
+
+```typescript
+const db = RocksDatabase.open('path/to/db', { compression: { algorithm: 'zstd', level: 3 } });
+console.log(db.compression); // { algorithm: 'zstd', level: 3 }
+```
+
 ### `db.config(options)`
 
 Sets global database settings.
@@ -904,6 +934,77 @@ An object is a record with the following properties:
 - `standardDeviation: number` A double containing the standard deviation.
 - `sum: number` An unsigned 64-bit integer containing the sum of all values.
 
+## Compression
+
+RocksDB compresses data blocks in SST files (and, in `rocksdb-js`, blob files that hold large
+values) using a configurable algorithm. Compression trades CPU for disk space and, because it
+reduces bytes read from disk, often improves read throughput on I/O-bound workloads.
+
+The following algorithm names are recognized. Which ones are actually usable depends on the native
+build — see **Availability** below.
+
+| Name     | Notes                                                                         |
+| -------- | ----------------------------------------------------------------------------- |
+| `none`   | No compression. Always available.                                             |
+| `snappy` | Fast, modest ratio. RocksDB's own stock default when linked.                  |
+| `zlib`   | Higher ratio, slower. Supports `level` (see zlib's manual).                   |
+| `bzip2`  | High ratio, slow; rarely worth it over zstd.                                  |
+| `lz4`    | Very fast, modest ratio. `rocksdb-js` default when available.                 |
+| `lz4hc`  | LZ4 high-compression variant: better ratio, slower writes, same fast reads.   |
+| `zstd`   | Best ratio-for-speed of the set; supports `level` (higher = smaller, slower). |
+
+Set the algorithm per column family with the `compression` option when opening a database:
+
+```typescript
+// Algorithm name
+const db = RocksDatabase.open('/path/to/db', { compression: 'zstd' });
+
+// Or an object with an explicit level
+const db2 = RocksDatabase.open('/path/to/db2', { compression: { algorithm: 'zstd', level: 3 } });
+
+// Disable compression entirely
+const db3 = RocksDatabase.open('/path/to/db3', { compression: 'none' });
+```
+
+**Default.** When `compression` is omitted, `rocksdb-js` defaults to **LZ4** if the native build
+supports it. LZ4 is fast with a modest compression ratio, making it a good general-purpose default.
+If LZ4 is not compiled in, the default falls back to RocksDB's own default — Snappy when it is
+linked, otherwise no compression. (RocksDB's stock default is Snappy; `rocksdb-js` overrides it to
+LZ4.) Read the algorithm actually in effect with the `db.compression` getter.
+
+**Availability.** The set of algorithms depends on which compression libraries the native binding
+was compiled against, so it varies by build. Always check
+[`supportedCompression`](#supportedcompression) at runtime — opening with an unavailable algorithm
+throws. `'none'` is always available.
+
+**Scope and mutability.** Compression is genuinely **per-column-family**. Each `RocksDatabase`
+targets one column family (`name`), and its `compression` applies only to that CF — opening one CF
+never changes another's algorithm, regardless of open order. A column family keeps its own
+compression across a close/reopen (a plain reopen inherits it); the algorithm is dynamically
+changeable, so an explicit change governs files written afterward while existing SST/blob files keep
+their original compression until rewritten by compaction. It also applies to blob files (large
+values), whose compression otherwise defaults to none.
+
+Because a column family's compression is fixed while it is open, if the same column family is opened
+a second time in the same process (another `RocksDatabase` on the same path/`name`, including from a
+`worker_thread`) with an **explicitly different** algorithm or level, the second open **throws** — a
+plain reopen (no `compression`) instead inherits the live setting.
+
+### `supportedCompression`
+
+A module-level, frozen array of the compression algorithm names compiled into the loaded native
+binding. The set is fixed for a given binary. `'none'` is always present.
+
+```typescript
+import { supportedCompression } from '@harperfast/rocksdb-js';
+
+console.log(supportedCompression); // e.g. ['none', 'snappy', 'lz4', 'zstd']
+
+if (supportedCompression.includes('zstd')) {
+	// safe to open with { compression: 'zstd' }
+}
+```
+
 ## Exclusive Locking
 
 `rocksdb-js` includes a handful of functions for executing thread-safe mutually exclusive functions.
@@ -996,18 +1097,28 @@ Note: If the `callback` throws an error, Node.js suppress the error. Node.js 18.
 `--force-node-api-uncaught-exceptions-policy` flag which will cause errors to emit the
 `'uncaughtException'` event. Future Node.js releases will enable this flag by default.
 
-## Exclusive File Locking
+## File Locking
 
 `rocksdb-js` includes helper functions for creating lock files and releasing them using native APIs.
 This can be used to prevent multiple processes from concurrently accessing a resource. The lock is
 automatically released when the process exits.
 
-### `tryFileLock(file: string): number`
+### `tryFileLock(file: string, shared?: boolean): number`
+
+Attempts to acquire a lock on the given file, creating the file and any missing parent directories.
+The lock is exclusive by default; pass `shared: true` for a shared (reader) lock, which coexists
+with other shared holders but conflicts with an exclusive holder in either direction. Returns a
+non-zero token to pass to `fileLockRelease` if the lock was acquired, or `0` if a conflicting
+holder — in any process, container, or worker thread — currently has it. Throws on a hard error.
 
-Attempts to acquire an exclusive lock on the given file, creating it if it doesn't exist. Returns a
-non-zero token to pass to `fileLockRelease` if the lock was acquired, or `0` if another holder — in
-any process, container, or worker thread — currently has it. Throws if the file's parent directory
-is missing or on a hard error.
+A shared lock needs only read access: it opens the file read-only and does not create it (falling
+back to create only when the file is missing on a writable directory), so it can lock an existing
+lock file on a read-only filesystem. If the media is read-only for every process (`EROFS` /
+`ERROR_WRITE_PROTECT`) so the open fails outright, a shared acquire degrades to a successful no-op
+token — there can be no exclusive holder to exclude on a directory nothing can write. A plain
+permission denial (`EACCES` / `ERROR_ACCESS_DENIED`) is _not_ degraded — only the caller is blocked,
+so a privileged holder may still exist — and throws instead. An exclusive lock always opens
+read-write and creates the file, and so requires a writable location.
 
 ```typescript
 import { tryFileLock } from '@harperfast/rocksdb-js';
@@ -1372,6 +1483,57 @@ Returns an object containing all of the information in the log file.
   - `length: number` The size of the entry data.
   - `timestamp: number` The entry timestamp.
 
+### Transaction Log Validation
+
+#### `validateTransactionLogStore(path, options?)`
+
+Validates a transaction log store directory (`<dbDir>/transaction_logs/<name>`): every
+`<sequence>.txnlog` file is checked for a valid header and intact entry framing (using the same
+scan as open-time crash recovery), file-name and sequence continuity are checked, and the
+`txn.state` side file (when present) is checked for shape and a plausible flushed position.
+
+Validation runs natively on a worker thread and does not require (or open) a database, so it works
+on the store of a closed database and on backup snapshots alike. It resolves with a report rather
+than throwing when the store is invalid — check `result.valid`. It rejects only when `path` does
+not exist or is not a directory.
+
+```typescript
+const result = await validateTransactionLogStore('/path/to/db/transaction_logs/mylog');
+if (!result.valid) {
+	console.error(result.errors);
+	for (const file of result.files) {
+		console.error(file.file, file.errors, file.warnings);
+	}
+}
+```
+
+Returns a `TransactionLogStoreValidation` object:
+
+- `path: string` The validated store directory.
+- `valid: boolean` `true` when there are no store-level errors and every log file is valid.
+- `errors: string[]` Store-level errors (malformed file names, corrupt `txn.state`).
+- `warnings: string[]` Store-level warnings (sequence gaps, unexpected files).
+- `files: TransactionLogFileValidation[]` Per-file results, ordered by sequence number.
+  - `file: string` File name, e.g. `"3.txnlog"`.
+  - `sequence: number` Sequence number parsed from the file name.
+  - `size: number` On-disk file size in bytes.
+  - `entries: number` Number of well-formed entries.
+  - `validBytes: number` End offset of the validly framed data.
+  - `valid: boolean` `true` when the file has no errors.
+  - `errors: string[]` Bad header, unsupported version, or mid-file framing corruption.
+  - `warnings: string[]` Torn tail (recoverable on next open) and per-entry anomalies.
+
+Options:
+
+- `strict?: boolean` Report conditions that indicate an incomplete snapshot — a torn tail, a
+  sequence gap, or a `txn.state` flushed position beyond the newest log file or beyond its file's
+  actual size — as errors instead of warnings. A torn tail on a live store is a normal crash
+  artifact that open-time recovery truncates, but a backup snapshot must be clean end to end —
+  `backups.verify()` uses `strict: true`. Defaults to `false`.
+
+Validating a store that is being actively appended to can spuriously report a torn tail for the
+current log file — the tail of an in-flight append is indistinguishable from a crash artifact.
+
 ### `currentThreadId(): number`
 
 Returns the current thread ID.
@@ -1464,11 +1626,14 @@ grouped under the `backups` namespace export.
 
 > **Only one backup per directory may be in-flight at a time.** RocksDB has no cross-engine lock on
 > a backup directory, so the writing operations — `db.backup()`, `backups.delete()`, and
-> `backups.purge()` — take an on-disk lock (a `.backup.lock` file) for the directory. A second
-> writing operation on the same directory, whether from the same process, a `worker_thread`, or a
-> separate process, **rejects** with a "locked" error rather than corrupting the backup; retry once
-> the in-flight operation finishes. Operations on _different_ directories run in parallel, and the
-> read-only operations (`list`, `verify`, `restore`) are not locked.
+> `backups.purge()` — take an on-disk lock (a `.backup.lock` file) for the directory.
+> `backups.restore()` takes the same lock in **shared** mode: concurrent restores run in parallel,
+> but a writer and a restore on the same directory exclude each other, so a `purge` cannot delete
+> the backup a restore is copying from. A conflicting operation on the same directory, whether from
+> the same process, a `worker_thread`, or a separate process, **rejects** with a "locked" error
+> rather than corrupting the backup; retry once the in-flight operation finishes. Operations on
+> _different_ directories run in parallel, and the read-only operations (`list`, `verify`) are not
+> locked.
 
 ```typescript
 import { RocksDatabase, backups } from '@harperfast/rocksdb-js';
@@ -1491,6 +1656,10 @@ re-copied.
 When the database was opened with `disableWAL`, the memtable is flushed before the backup by
 default so unflushed data is not lost; otherwise flushing follows `options.flushBeforeBackup`.
 
+`backupDir` must not be the database directory itself or a path beneath it — backing up into the
+live database directory would write backup files on top of RocksDB's own files. Such a call rejects
+before anything is written.
+
 ```typescript
 const id = await db.backup('/path/to/backups', { metadata: 'nightly-2026-06-04' });
 ```
@@ -1500,6 +1669,7 @@ const id = await db.backup('/path/to/backups', { metadata: 'nightly-2026-06-04'
 | Option                    | Type      | Default                | Description                                                                          |
 | ------------------------- | --------- | ---------------------- | ------------------------------------------------------------------------------------ |
 | `backupLogFiles`          | `boolean` | `true`                 | Include write-ahead log files in the backup.                                         |
+| `checkDiskSpace`          | `boolean` | `true`                 | Preflight the destination volume and reject if it lacks room for the backup.         |
 | `flushBeforeBackup`       | `boolean` | `true` if WAL disabled | Flush the memtable before backing up.                                                |
 | `maxBackgroundOperations` | `number`  | `1`                    | Number of background threads used to copy files.                                     |
 | `metadata`                | `string`  | `''`                   | Application metadata stored with the backup, returned by `list()`.                   |
@@ -1515,6 +1685,16 @@ backup either has its complete snapshot or none. The snapshot is captured just a
 engine snapshot, so restored logs may run slightly ahead of the restored key-value data (never
 behind it), which is safe for redo-style logs replayed against the restored data.
 
+`checkDiskSpace` (on by default) rejects with an I/O error — its message reports the available and
+required bytes — before any files are copied, so a full destination fails fast instead of partway
+through (a mid-copy failure can leave zero usable backups). The required size is estimated
+conservatively as a full copy of the database's live files, plus the transaction-log snapshot when
+`transactionLogs` is set; since a backup only ever copies files, this can never under-estimate.
+Incremental backups to a right-sized volume may therefore be over-rejected — set `checkDiskSpace:
+false` to skip the check. It also self-disables when free space can't be determined (e.g. some
+network filesystems report `0`), so it never blocks a backup on an untrustworthy number. Only
+directory-target backups are checked; the streaming backup path is unaffected.
+
 ### `db.backup(stream: WritableStream<Uint8Array>, options?: BackupStreamOptions): Promise<void>`
 
 Streams a consistent snapshot of the entire database to `stream` as a tar archive, with **no
@@ -1551,6 +1731,20 @@ Restores a backup from `backupDir` into `dbDir` (creating parent directories as
 database must **not** be open at `dbDir`, and the default restore mode is **destructive** — it
 purges `dbDir` before restoring. Restoring into the backup directory itself is rejected.
 
+The restore holds the backup directory's `.backup.lock` in shared mode for its duration, so it
+rejects with a "locked" error while a writer (`db.backup()`, `backups.delete()`, `backups.purge()`)
+is in flight — and writers reject while the restore runs. Concurrent restores from the same
+directory are allowed.
+
+Because a restore only reads the backup, the shared lock needs no write access to `backupDir`:
+restoring from a **read-only backup directory** (an immutable/WORM store or a read-only-mounted
+volume) works — it locks the existing `.backup.lock` read-only, and if the media is read-only for
+every process (`EROFS`), the lock degrades to a no-op, since nothing can write the directory and so
+no writer can be racing the restore. A mere permission denial is _not_ treated this way — it means
+only the current identity is blocked, so a more-privileged writer could still be running, and the
+restore fails loudly rather than skipping coordination. Writers always need write access and fail on
+a read-only directory.
+
 ```typescript
 // Restore the latest backup.
 await backups.restore('/path/to/backups', '/path/to/restored-db');
@@ -1592,11 +1786,17 @@ backup references them, so this is not equivalent to deleting files manually.
 
 Deletes all but the newest `keepCount` backups.
 
-### `backups.verify(backupDir: string, backupId: number, options?: { verifyWithChecksum?: boolean }): Promise<void>`
+### `backups.verify(backupDir: string, backupId: number, options?: { verifyWithChecksum?: boolean; verifyTransactionLogs?: boolean }): Promise<void>`
 
 Verifies a backup's file sizes, and optionally their checksums (which requires reading all
 backed-up data). Resolves if the backup is intact and rejects otherwise.
 
+When the backup was created with `transactionLogs: true`, the backup's transaction log snapshot
+(`<backupDir>/transaction_logs/<backupId>/`) is also validated with
+[`validateTransactionLogStore`](#validatetransactionlogstorepath-options) in strict mode — every
+log file's header and entry framing must be intact (snapshots are copied on committed entry
+boundaries, so even a torn tail fails verification). Set `verifyTransactionLogs: false` to skip.
+
 ```typescript
 await backups.verify('/path/to/backups', 1, { verifyWithChecksum: true });
 ```
@@ -1661,6 +1861,13 @@ await db.put('foo', 'bar');
 console.log(await db.get('foo'));
 ```
 
+> [!NOTE]
+> A `Store` is bound to a single `RocksDatabase` instance. Passing a store that another
+> `RocksDatabase` already owns throws `Store is already in use by another RocksDatabase instance`.
+> The claim is durable across `db.close()`, so reopening the original owner does not release it.
+> (Sharing a store with the owning database's `Transaction` instances is expected and handled
+> internally.)
+
 > [!IMPORTANT]
 > If your custom store overrides `putSync()` without calling `super.putSync()` and it performs its
 > own `this.encodeKey(key)`, then you MUST encode the VALUE before you encode the KEY.
@@ -1756,8 +1963,9 @@ Available commands:
 - `backups <dir> [subcommand]` Manage database backups; with no subcommand or `ls`/`list`, lists the backups in
   `<dir>`. Subcommands: `backup` (create a backup of the open database), `restore <backup-id>`
   (restore into the open database after confirmation; unavailable in read-only mode),
-  `verify <backup-id>` (checksum verification), `delete <backup-id>`, and `purge <keep-count>`
-  (delete all but the newest backups). `delete` and `purge` report the recovered disk space.
+  `verify <backup-id>` (checksum verification, including the backup's transaction log snapshot),
+  `delete <backup-id>`, and `purge <keep-count>` (delete all but the newest backups). `delete` and
+  `purge` report the recovered disk space.
 - `clear` Clear all data in the current column family
 - `columns` List column families
 - `compact` Compact the current column family
@@ -1775,6 +1983,7 @@ Available commands:
 - `repl` Open a JS sub-REPL; "db" refers to the current column family
 - `stats` Show the statistics for the current column family
 - `use [column]` Create a new column family or switch to an existing one
+- `verify-logs [name]` Validate the transaction log store files (all stores if no name)
 
 ## Development
 

package/bin/rocksdb-js.mjs

@@ -1,6 +1,12 @@
 #!/usr/bin/env node
 
-import { RocksDatabase, backups, parseTransactionLog, versions } from '../dist/index.mjs';
+import {
+	RocksDatabase,
+	backups,
+	parseTransactionLog,
+	validateTransactionLogStore,
+	versions,
+} from '../dist/index.mjs';
 import { existsSync, readdirSync } from 'node:fs';
 import { mkdir, readdir, stat } from 'node:fs/promises';
 import { join, resolve } from 'node:path';
@@ -29,6 +35,7 @@ const COMMANDS = {
 	repl: replCommand,
 	stats: statsCommand,
 	use: useCommand,
+	'verify-logs': verifyLogsCommand,
 };
 
 const BACKUP_SUBCOMMANDS = ['backup', 'restore', 'verify', 'delete', 'purge'];
@@ -73,13 +80,19 @@ function completer(line) {
 		return [hits.length ? hits : columns, arg];
 	}
 
-	if ((command === 'log' || command === 'logs') && currentDB) {
+	if (
+		(command === 'log' ||
+			command === 'logs' ||
+			command === 'verify-logs' ||
+			command === 'purge-logs') &&
+		currentDB
+	) {
 		if (parts.length === 2) {
 			const arg = parts[1];
 			const logs = currentDB.listLogs();
 			const hits = logs.filter((l) => l.startsWith(arg));
 			return [(hits.length ? hits : logs).map((s) => `${s} `), arg];
-		} else if (parts.length === 3) {
+		} else if (parts.length === 3 && (command === 'log' || command === 'logs')) {
 			const [name, store] = parts.slice(1);
 			const logs = currentDB.listLogs();
 			const hit = logs.find((l) => l === name);
@@ -616,6 +629,9 @@ function helpCommand() {
 	console.log(
 		`${hl('use [column]')}               Create a new column family or switch to an existing one`
 	);
+	console.log(
+		`${hl('verify-logs [name]')}         Validate transaction log store files (all stores if no name)`
+	);
 	console.log();
 }
 
@@ -732,6 +748,54 @@ async function purgeLogsCommand(args) {
 	console.log();
 }
 
+async function verifyLogsCommand(args) {
+	const [name] = args;
+	const logs = currentDB.listLogs();
+	if (name && !logs.includes(name)) {
+		console.log(`Log store ${hl(name)} does not exist\n`);
+		return;
+	}
+	if (logs.length === 0) {
+		console.log('No log stores found\n');
+		return;
+	}
+
+	let invalidStores = 0;
+	for (const storeName of name ? [name] : logs) {
+		const storePath = join(currentDB.path, 'transaction_logs', storeName);
+		const { result, time } = await run(() => validateTransactionLogStore(storePath));
+		console.log(`${hl(storeName)}  ${result.valid ? 'OK' : bad('INVALID')} ${note(`(${time}ms)`)}`);
+		if (!result.valid) {
+			invalidStores++;
+		}
+		for (const error of result.errors) {
+			console.log(`  ${bad(`! ${error}`)}`);
+		}
+		for (const warning of result.warnings) {
+			console.log(`  ${warn(`! ${warning}`)}`);
+		}
+		for (const file of result.files) {
+			console.log(
+				`  ${file.file}  ${file.valid ? 'OK' : bad('INVALID')}  ${note(
+					`${file.entries.toLocaleString()} entr${file.entries === 1 ? 'y' : 'ies'}, ${formatBytes(file.size)}`
+				)}`
+			);
+			for (const error of file.errors) {
+				console.log(`    ${bad(`! ${error}`)}`);
+			}
+			for (const warning of file.warnings) {
+				console.log(`    ${warn(`! ${warning}`)}`);
+			}
+		}
+	}
+	if (invalidStores > 0) {
+		console.log(
+			`\n${bad(`${invalidStores} log store${invalidStores === 1 ? ' is' : 's are'} invalid`)}`
+		);
+	}
+	console.log();
+}
+
 async function putCommand(args) {
 	const [key, ...valueParts] = args;
 	if (!key || valueParts.length === 0) {