@harperfast/rocksdb-js 2.3.0 → 2.5.0

package/README.md

@@ -47,6 +47,13 @@ Creates a new database instance.
   - `enableStats: boolean` When `true` and the database is open, RocksDB will captures stats that
     are retrieved by calling `db.getStats()`. Enabling statistics imposes 5-10% in overhead.
     Defaults to `false`.
+  - `maxOpenFiles: number` The maximum number of table files RocksDB keeps open. `0` (the default)
+    derives a budget from the effective per-process open-file limit (an eighth of the limit —
+    several databases can share one process — clamped to `[1024, 262144]`); `-1` holds every table
+    file open (the RocksDB default, which can exhaust the process file-descriptor limit when
+    compaction falls behind under sustained ingest); a positive `int32` is an explicit cap. Reads
+    only pay a reopen cost when the number of live table files exceeds the budget, so raise the
+    process fd limit (and with it the derived budget) for very large databases.
   - `name: string` The column family name. Defaults to `"default"`.
   - `noBlockCache: boolean` When `true`, disables the block cache. Block caching is enabled by
     default and the cache is shared across all database instances.
@@ -996,6 +1003,50 @@ Note: If the `callback` throws an error, Node.js suppress the error. Node.js 18.
 `--force-node-api-uncaught-exceptions-policy` flag which will cause errors to emit the
 `'uncaughtException'` event. Future Node.js releases will enable this flag by default.
 
+## File Locking
+
+`rocksdb-js` includes helper functions for creating lock files and releasing them using native APIs.
+This can be used to prevent multiple processes from concurrently accessing a resource. The lock is
+automatically released when the process exits.
+
+### `tryFileLock(file: string, shared?: boolean): number`
+
+Attempts to acquire a lock on the given file, creating the file and any missing parent directories.
+The lock is exclusive by default; pass `shared: true` for a shared (reader) lock, which coexists
+with other shared holders but conflicts with an exclusive holder in either direction. Returns a
+non-zero token to pass to `fileLockRelease` if the lock was acquired, or `0` if a conflicting
+holder — in any process, container, or worker thread — currently has it. Throws on a hard error.
+
+A shared lock needs only read access: it opens the file read-only and does not create it (falling
+back to create only when the file is missing on a writable directory), so it can lock an existing
+lock file on a read-only filesystem. If the media is read-only for every process (`EROFS` /
+`ERROR_WRITE_PROTECT`) so the open fails outright, a shared acquire degrades to a successful no-op
+token — there can be no exclusive holder to exclude on a directory nothing can write. A plain
+permission denial (`EACCES` / `ERROR_ACCESS_DENIED`) is _not_ degraded — only the caller is blocked,
+so a privileged holder may still exist — and throws instead. An exclusive lock always opens
+read-write and creates the file, and so requires a writable location.
+
+```typescript
+import { tryFileLock } from '@harperfast/rocksdb-js';
+
+const token = tryFileLock('/path/to/lock');
+if (token) {
+	console.log('lock acquired');
+} else {
+	console.log('lock not available, another process is holding it');
+}
+```
+
+### `fileLockRelease(token: number): void`
+
+Releases the file lock for the given token.
+
+```typescript
+import { fileLockRelease } from '@harperfast/rocksdb-js';
+
+fileLockRelease(token);
+```
+
 ## Verification Table
 
 The verification table is a process-global, fixed-size structure that lets an application cheaply
@@ -1338,6 +1389,57 @@ Returns an object containing all of the information in the log file.
   - `length: number` The size of the entry data.
   - `timestamp: number` The entry timestamp.
 
+### Transaction Log Validation
+
+#### `validateTransactionLogStore(path, options?)`
+
+Validates a transaction log store directory (`<dbDir>/transaction_logs/<name>`): every
+`<sequence>.txnlog` file is checked for a valid header and intact entry framing (using the same
+scan as open-time crash recovery), file-name and sequence continuity are checked, and the
+`txn.state` side file (when present) is checked for shape and a plausible flushed position.
+
+Validation runs natively on a worker thread and does not require (or open) a database, so it works
+on the store of a closed database and on backup snapshots alike. It resolves with a report rather
+than throwing when the store is invalid — check `result.valid`. It rejects only when `path` does
+not exist or is not a directory.
+
+```typescript
+const result = await validateTransactionLogStore('/path/to/db/transaction_logs/mylog');
+if (!result.valid) {
+	console.error(result.errors);
+	for (const file of result.files) {
+		console.error(file.file, file.errors, file.warnings);
+	}
+}
+```
+
+Returns a `TransactionLogStoreValidation` object:
+
+- `path: string` The validated store directory.
+- `valid: boolean` `true` when there are no store-level errors and every log file is valid.
+- `errors: string[]` Store-level errors (malformed file names, corrupt `txn.state`).
+- `warnings: string[]` Store-level warnings (sequence gaps, unexpected files).
+- `files: TransactionLogFileValidation[]` Per-file results, ordered by sequence number.
+  - `file: string` File name, e.g. `"3.txnlog"`.
+  - `sequence: number` Sequence number parsed from the file name.
+  - `size: number` On-disk file size in bytes.
+  - `entries: number` Number of well-formed entries.
+  - `validBytes: number` End offset of the validly framed data.
+  - `valid: boolean` `true` when the file has no errors.
+  - `errors: string[]` Bad header, unsupported version, or mid-file framing corruption.
+  - `warnings: string[]` Torn tail (recoverable on next open) and per-entry anomalies.
+
+Options:
+
+- `strict?: boolean` Report conditions that indicate an incomplete snapshot — a torn tail, a
+  sequence gap, or a `txn.state` flushed position beyond the newest log file or beyond its file's
+  actual size — as errors instead of warnings. A torn tail on a live store is a normal crash
+  artifact that open-time recovery truncates, but a backup snapshot must be clean end to end —
+  `backups.verify()` uses `strict: true`. Defaults to `false`.
+
+Validating a store that is being actively appended to can spuriously report a torn tail for the
+current log file — the tail of an in-flight append is indistinguishable from a crash artifact.
+
 ### `currentThreadId(): number`
 
 Returns the current thread ID.
@@ -1419,10 +1521,26 @@ Backups use RocksDB's `BackupEngine` to capture consistent, incremental, checksu
 snapshots of a database. A backup covers the **entire database** — every column family, the
 manifest, and (by default) the write-ahead log — so it is not scoped to an individual `Store`.
 
+A backup can be written to a local **directory** (incremental, with a management API) or streamed to
+a **`WritableStream`** as a tar archive with no intermediate copy on disk. See
+[docs/backups.md](docs/backups.md) for a full guide covering both modes, restore, checkpoints, and
+caveats.
+
 Creating a backup is an instance method (`db.backup()`) because it needs a live database. The
 remaining operations act on a backup directory and do not require an open database, so they are
 grouped under the `backups` namespace export.
 
+> **Only one backup per directory may be in-flight at a time.** RocksDB has no cross-engine lock on
+> a backup directory, so the writing operations — `db.backup()`, `backups.delete()`, and
+> `backups.purge()` — take an on-disk lock (a `.backup.lock` file) for the directory.
+> `backups.restore()` takes the same lock in **shared** mode: concurrent restores run in parallel,
+> but a writer and a restore on the same directory exclude each other, so a `purge` cannot delete
+> the backup a restore is copying from. A conflicting operation on the same directory, whether from
+> the same process, a `worker_thread`, or a separate process, **rejects** with a "locked" error
+> rather than corrupting the backup; retry once the in-flight operation finishes. Operations on
+> _different_ directories run in parallel, and the read-only operations (`list`, `verify`) are not
+> locked.
+
 ```typescript
 import { RocksDatabase, backups } from '@harperfast/rocksdb-js';
 
@@ -1444,21 +1562,74 @@ re-copied.
 When the database was opened with `disableWAL`, the memtable is flushed before the backup by
 default so unflushed data is not lost; otherwise flushing follows `options.flushBeforeBackup`.
 
+`backupDir` must not be the database directory itself or a path beneath it — backing up into the
+live database directory would write backup files on top of RocksDB's own files. Such a call rejects
+before anything is written.
+
 ```typescript
 const id = await db.backup('/path/to/backups', { metadata: 'nightly-2026-06-04' });
 ```
 
 `BackupOptions`:
 
-| Option                    | Type      | Default                | Description                                                           |
-| ------------------------- | --------- | ---------------------- | --------------------------------------------------------------------- |
-| `flushBeforeBackup`       | `boolean` | `true` if WAL disabled | Flush the memtable before backing up.                                 |
-| `metadata`                | `string`  | `''`                   | Application metadata stored with the backup, returned by `list()`.    |
-| `shareTableFiles`         | `boolean` | `true`                 | Share files between backups to enable incremental backups.            |
-| `shareFilesWithChecksum`  | `boolean` | `true`                 | Distinguish shared files by checksum to avoid cross-database clashes. |
-| `backupLogFiles`          | `boolean` | `true`                 | Include write-ahead log files in the backup.                          |
-| `sync`                    | `boolean` | `true`                 | `fsync` backup files for crash consistency.                           |
-| `maxBackgroundOperations` | `number`  | `1`                    | Number of background threads used to copy files.                      |
+| Option                    | Type      | Default                | Description                                                                          |
+| ------------------------- | --------- | ---------------------- | ------------------------------------------------------------------------------------ |
+| `backupLogFiles`          | `boolean` | `true`                 | Include write-ahead log files in the backup.                                         |
+| `checkDiskSpace`          | `boolean` | `true`                 | Preflight the destination volume and reject if it lacks room for the backup.         |
+| `flushBeforeBackup`       | `boolean` | `true` if WAL disabled | Flush the memtable before backing up.                                                |
+| `maxBackgroundOperations` | `number`  | `1`                    | Number of background threads used to copy files.                                     |
+| `metadata`                | `string`  | `''`                   | Application metadata stored with the backup, returned by `list()`.                   |
+| `shareFilesWithChecksum`  | `boolean` | `true`                 | Distinguish shared files by checksum to avoid cross-database clashes.                |
+| `shareTableFiles`         | `boolean` | `true`                 | Share files between backups to enable incremental backups.                           |
+| `sync`                    | `boolean` | `true`                 | `fsync` backup files (including the transaction log snapshot) for crash consistency. |
+| `transactionLogs`         | `boolean` | `false`                | Snapshot the transaction log store into `<backupDir>/transaction_logs/<backupId>/`.  |
+
+When `transactionLogs` is enabled, the log snapshot is staged and atomically renamed into
+`<backupDir>/transaction_logs/<backupId>/` only after every file has been copied (and fsynced, per
+`sync`), so a crash mid-backup can never leave a partial log snapshot for a listed backup id — a
+backup either has its complete snapshot or none. The snapshot is captured just after the RocksDB
+engine snapshot, so restored logs may run slightly ahead of the restored key-value data (never
+behind it), which is safe for redo-style logs replayed against the restored data.
+
+`checkDiskSpace` (on by default) rejects with an I/O error — its message reports the available and
+required bytes — before any files are copied, so a full destination fails fast instead of partway
+through (a mid-copy failure can leave zero usable backups). The required size is estimated
+conservatively as a full copy of the database's live files, plus the transaction-log snapshot when
+`transactionLogs` is set; since a backup only ever copies files, this can never under-estimate.
+Incremental backups to a right-sized volume may therefore be over-rejected — set `checkDiskSpace:
+false` to skip the check. It also self-disables when free space can't be determined (e.g. some
+network filesystems report `0`), so it never blocks a backup on an untrustworthy number. Only
+directory-target backups are checked; the streaming backup path is unaffected.
+
+### `db.backup(stream: WritableStream<Uint8Array>, options?: BackupStreamOptions): Promise<void>`
+
+Streams a consistent snapshot of the entire database to `stream` as a tar archive, with **no
+intermediate copy written to disk**, and resolves once the stream has been fully written and closed.
+Backpressure is honored end to end, so a slow consumer (e.g. an upload) paces the backup rather than
+buffering it in memory. The archive unpacks with any tar tool into a directory that opens as a
+RocksDB database.
+
+```typescript
+import { createWriteStream } from 'node:fs';
+import { Writable } from 'node:stream';
+
+await db.backup(Writable.toWeb(createWriteStream('/path/to/backup.tar')));
+// Restore: `tar -xf backup.tar -C /restored`, then open '/restored'.
+
+// Or gzip it (`tar -xzf backup.tar.gz` to restore):
+await db.backup(Writable.toWeb(createWriteStream('/path/to/backup.tar.gz')), { gzip: true });
+```
+
+`BackupStreamOptions`:
+
+| Option              | Type      | Default                | Description                                                         |
+| ------------------- | --------- | ---------------------- | ------------------------------------------------------------------- |
+| `flushBeforeBackup` | `boolean` | `true` if WAL disabled | Flush the memtable before streaming.                                |
+| `gzip`              | `boolean` | `false`                | Gzip-compress the archive, producing a `.tar.gz` instead of `.tar`. |
+
+Stream backups are always full snapshots (no incremental sharing), have no `backups.*` management
+API, and **cannot be resumed** — a failed transfer must be restarted from the beginning. See
+[docs/backups.md](docs/backups.md#stream-backups) for details.
 
 ### `backups.restore(backupDir: string, dbDir: string, options?: RestoreOptions): Promise<void>`
 
@@ -1466,6 +1637,20 @@ Restores a backup from `backupDir` into `dbDir` (creating parent directories as
 database must **not** be open at `dbDir`, and the default restore mode is **destructive** — it
 purges `dbDir` before restoring. Restoring into the backup directory itself is rejected.
 
+The restore holds the backup directory's `.backup.lock` in shared mode for its duration, so it
+rejects with a "locked" error while a writer (`db.backup()`, `backups.delete()`, `backups.purge()`)
+is in flight — and writers reject while the restore runs. Concurrent restores from the same
+directory are allowed.
+
+Because a restore only reads the backup, the shared lock needs no write access to `backupDir`:
+restoring from a **read-only backup directory** (an immutable/WORM store or a read-only-mounted
+volume) works — it locks the existing `.backup.lock` read-only, and if the media is read-only for
+every process (`EROFS`), the lock degrades to a no-op, since nothing can write the directory and so
+no writer can be racing the restore. A mere permission denial is _not_ treated this way — it means
+only the current identity is blocked, so a more-privileged writer could still be running, and the
+restore fails loudly rather than skipping coordination. Writers always need write access and fail on
+a read-only directory.
+
 ```typescript
 // Restore the latest backup.
 await backups.restore('/path/to/backups', '/path/to/restored-db');
@@ -1507,11 +1692,17 @@ backup references them, so this is not equivalent to deleting files manually.
 
 Deletes all but the newest `keepCount` backups.
 
-### `backups.verify(backupDir: string, backupId: number, options?: { verifyWithChecksum?: boolean }): Promise<void>`
+### `backups.verify(backupDir: string, backupId: number, options?: { verifyWithChecksum?: boolean; verifyTransactionLogs?: boolean }): Promise<void>`
 
 Verifies a backup's file sizes, and optionally their checksums (which requires reading all
 backed-up data). Resolves if the backup is intact and rejects otherwise.
 
+When the backup was created with `transactionLogs: true`, the backup's transaction log snapshot
+(`<backupDir>/transaction_logs/<backupId>/`) is also validated with
+[`validateTransactionLogStore`](#validatetransactionlogstorepath-options) in strict mode — every
+log file's header and entry framing must be intact (snapshots are copied on committed entry
+boundaries, so even a torn tail fails verification). Set `verifyTransactionLogs: false` to skip.
+
 ```typescript
 await backups.verify('/path/to/backups', 1, { verifyWithChecksum: true });
 ```
@@ -1671,8 +1862,9 @@ Available commands:
 - `backups <dir> [subcommand]` Manage database backups; with no subcommand or `ls`/`list`, lists the backups in
   `<dir>`. Subcommands: `backup` (create a backup of the open database), `restore <backup-id>`
   (restore into the open database after confirmation; unavailable in read-only mode),
-  `verify <backup-id>` (checksum verification), `delete <backup-id>`, and `purge <keep-count>`
-  (delete all but the newest backups). `delete` and `purge` report the recovered disk space.
+  `verify <backup-id>` (checksum verification, including the backup's transaction log snapshot),
+  `delete <backup-id>`, and `purge <keep-count>` (delete all but the newest backups). `delete` and
+  `purge` report the recovered disk space.
 - `clear` Clear all data in the current column family
 - `columns` List column families
 - `compact` Compact the current column family
@@ -1690,6 +1882,7 @@ Available commands:
 - `repl` Open a JS sub-REPL; "db" refers to the current column family
 - `stats` Show the statistics for the current column family
 - `use [column]` Create a new column family or switch to an existing one
+- `verify-logs [name]` Validate the transaction log store files (all stores if no name)
 
 ## Development
 

package/bin/rocksdb-js.mjs

@@ -1,6 +1,12 @@
 #!/usr/bin/env node
 
-import { RocksDatabase, backups, parseTransactionLog, versions } from '../dist/index.mjs';
+import {
+	RocksDatabase,
+	backups,
+	parseTransactionLog,
+	validateTransactionLogStore,
+	versions,
+} from '../dist/index.mjs';
 import { existsSync, readdirSync } from 'node:fs';
 import { mkdir, readdir, stat } from 'node:fs/promises';
 import { join, resolve } from 'node:path';
@@ -29,6 +35,7 @@ const COMMANDS = {
 	repl: replCommand,
 	stats: statsCommand,
 	use: useCommand,
+	'verify-logs': verifyLogsCommand,
 };
 
 const BACKUP_SUBCOMMANDS = ['backup', 'restore', 'verify', 'delete', 'purge'];
@@ -73,13 +80,19 @@ function completer(line) {
 		return [hits.length ? hits : columns, arg];
 	}
 
-	if ((command === 'log' || command === 'logs') && currentDB) {
+	if (
+		(command === 'log' ||
+			command === 'logs' ||
+			command === 'verify-logs' ||
+			command === 'purge-logs') &&
+		currentDB
+	) {
 		if (parts.length === 2) {
 			const arg = parts[1];
 			const logs = currentDB.listLogs();
 			const hits = logs.filter((l) => l.startsWith(arg));
 			return [(hits.length ? hits : logs).map((s) => `${s} `), arg];
-		} else if (parts.length === 3) {
+		} else if (parts.length === 3 && (command === 'log' || command === 'logs')) {
 			const [name, store] = parts.slice(1);
 			const logs = currentDB.listLogs();
 			const hit = logs.find((l) => l === name);
@@ -616,6 +629,9 @@ function helpCommand() {
 	console.log(
 		`${hl('use [column]')}               Create a new column family or switch to an existing one`
 	);
+	console.log(
+		`${hl('verify-logs [name]')}         Validate transaction log store files (all stores if no name)`
+	);
 	console.log();
 }
 
@@ -732,6 +748,54 @@ async function purgeLogsCommand(args) {
 	console.log();
 }
 
+async function verifyLogsCommand(args) {
+	const [name] = args;
+	const logs = currentDB.listLogs();
+	if (name && !logs.includes(name)) {
+		console.log(`Log store ${hl(name)} does not exist\n`);
+		return;
+	}
+	if (logs.length === 0) {
+		console.log('No log stores found\n');
+		return;
+	}
+
+	let invalidStores = 0;
+	for (const storeName of name ? [name] : logs) {
+		const storePath = join(currentDB.path, 'transaction_logs', storeName);
+		const { result, time } = await run(() => validateTransactionLogStore(storePath));
+		console.log(`${hl(storeName)}  ${result.valid ? 'OK' : bad('INVALID')} ${note(`(${time}ms)`)}`);
+		if (!result.valid) {
+			invalidStores++;
+		}
+		for (const error of result.errors) {
+			console.log(`  ${bad(`! ${error}`)}`);
+		}
+		for (const warning of result.warnings) {
+			console.log(`  ${warn(`! ${warning}`)}`);
+		}
+		for (const file of result.files) {
+			console.log(
+				`  ${file.file}  ${file.valid ? 'OK' : bad('INVALID')}  ${note(
+					`${file.entries.toLocaleString()} entr${file.entries === 1 ? 'y' : 'ies'}, ${formatBytes(file.size)}`
+				)}`
+			);
+			for (const error of file.errors) {
+				console.log(`    ${bad(`! ${error}`)}`);
+			}
+			for (const warning of file.warnings) {
+				console.log(`    ${warn(`! ${warning}`)}`);
+			}
+		}
+	}
+	if (invalidStores > 0) {
+		console.log(
+			`\n${bad(`${invalidStores} log store${invalidStores === 1 ? ' is' : 's are'} invalid`)}`
+		);
+	}
+	console.log();
+}
+
 async function putCommand(args) {
 	const [key, ...valueParts] = args;
 	if (!key || valueParts.length === 0) {