@harperfast/rocksdb-js 1.2.0 → 1.4.0

package/README.md

@@ -106,11 +106,26 @@ Sets global database settings.
     Defaults to 32MB. Set to `0` (zero) disables block cache for future opened databases. Existing
     block cache for any opened databases is resized immediately. Negative values throw an error.
   - `compactOnClose: boolean` When `true`, compacts the database on close. Defaults to `false`.
+  - `writeBufferManagerAllowStall: boolean` When `true`, writes are stalled once the manager's
+    `buffer_size` is exceeded, providing a hard cap on memtable memory. When `false`, memtables are
+    allowed to grow past the limit and flushes are simply scheduled more aggressively. Off by
+    default to favor write throughput over hard memory bounding. Defaults to `false`.
+  - `writeBufferManagerCostToCache: boolean` When `true`, memtable memory is "charged" against the
+    shared block cache so the block cache and write buffers draw from a single pool. During write
+    bursts the cache shrinks to make room for memtables; once memtables flush, the cache can grow
+    back into the reclaimed space. Defaults to `false`.
+  - `writeBufferManagerSize: number` Total memtable memory limit (bytes) shared across every
+    database opened in this process. When set, RocksDB uses a single `WriteBufferManager` so write
+    buffers are bounded process-wide rather than per database. A value of `0` disables the manager.
+    Defaults to `0`.
 
 ```typescript
 RocksDatabase.config({
 	blockCacheSize: 100 * 1024 * 1024, // 100MB
 	compactOnClose: true,
+	writeBufferManagerAllowStall: false,
+	writeBufferManagerCostToCache: false,
+	writeBufferManagerSize: 64 * 1024 * 1024, // 64MB
 });
 ```
 
@@ -1421,6 +1436,24 @@ To run the tests without code coverage, run:
 pnpm test
 ```
 
+#### Native C++ unit tests (GoogleTest)
+
+Build and run standalone native tests (RocksDB + binding `core/` helpers, no Node in the test
+binary):
+
+```bash
+pnpm test:native
+```
+
+Coverage report for native tests (macOS/Linux; requires `lcov`):
+
+```bash
+pnpm coverage:native
+# open coverage/native/html/index.html
+```
+
+Note: Code coverage is not supported on Windows. Tests are run without coverage.
+
 To run a specific test suite, for example `"ranges"`, run:
 
 ```bash

package/bin/rocksdb-js.mjs

@@ -2,7 +2,7 @@
 
 import { RocksDatabase, parseTransactionLog, versions } from '../dist/index.mjs';
 import { existsSync, readdirSync } from 'node:fs';
-import { mkdir, readdir } from 'node:fs/promises';
+import { mkdir, readdir, stat } from 'node:fs/promises';
 import { join, resolve } from 'node:path';
 import { stdin, stdout } from 'node:process';
 import { createInterface } from 'node:readline/promises';
@@ -19,6 +19,7 @@ const COMMANDS = {
 	get: getCommand,
 	help: helpCommand,
 	log: logCommand,
+	logs: logCommand, // alias for log
 	prop: propCommand,
 	'purge-logs': purgeLogsCommand,
 	put: putCommand,
@@ -66,12 +67,12 @@ function completer(line) {
 		return [hits.length ? hits : columns, arg];
 	}
 
-	if (command === 'log' && currentDB) {
+	if ((command === 'log' || command === 'logs') && currentDB) {
 		if (parts.length === 2) {
 			const arg = parts[1];
 			const logs = currentDB.listLogs();
 			const hits = logs.filter((l) => l.startsWith(arg));
-			return [hits.length ? hits : logs, arg];
+			return [(hits.length ? hits : logs).map((s) => `${s} `), arg];
 		} else if (parts.length === 3) {
 			const [name, store] = parts.slice(1);
 			const logs = currentDB.listLogs();
@@ -81,7 +82,7 @@ function completer(line) {
 				const logFiles = readdirSync(logPath).filter(
 					(f) => (!store || f.startsWith(store)) && f.endsWith('.txnlog')
 				);
-				return [logFiles.length ? logFiles : [], store, name];
+				return [logFiles.length ? logFiles.map((f) => `${f} `) : [], store, name];
 			}
 			return [[], store, name];
 		}
@@ -149,6 +150,34 @@ function wrapColumns(columns) {
 	return prefix + lines.join('\n' + indent);
 }
 
+function formatHexDump(data) {
+	const bytesPerRow = 16;
+	const lines = [];
+	for (let offset = 0; offset < data.length; offset += bytesPerRow) {
+		const chunk = data.subarray(offset, offset + bytesPerRow);
+		const hexParts = [];
+		for (let i = 0; i < bytesPerRow; i++) {
+			if (i === 8) hexParts.push('');
+			hexParts.push(i < chunk.length ? chunk[i].toString(16).padStart(2, '0') : '  ');
+		}
+		let ascii = '';
+		for (const byte of chunk) {
+			ascii += byte >= 0x20 && byte <= 0x7e ? String.fromCharCode(byte) : '.';
+		}
+		lines.push(
+			`${note(offset.toString(16).padStart(8, '0'))}  ${hl(hexParts.join(' '))}  ${note('|')}${hl(ascii)}${note('|')}`
+		);
+	}
+	return lines.join('\n');
+}
+
+function formatBytes(bytes) {
+	if (bytes < 1024) return `${bytes} B`;
+	if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(2)} KB`;
+	if (bytes < 1024 * 1024 * 1024) return `${(bytes / 1024 / 1024).toFixed(2)} MB`;
+	return `${(bytes / 1024 / 1024 / 1024).toFixed(2)} GB`;
+}
+
 async function run(fn) {
 	const start = Date.now();
 	const result = await fn();
@@ -320,40 +349,72 @@ async function logCommand(args) {
 				}
 				const { data } = log.entries[entry - 1];
 				console.log(`Entry ${entry} (${data.length.toLocaleString()} bytes)\n`);
-				console.log(
-					hl(
-						data
-							.toString('hex')
-							.match(/.{1,2}/g)
-							.join(' ')
-					)
-				);
+				console.log(formatHexDump(data));
 				console.log();
 				return;
 			}
 
-			console.log(`Size      = ${hl(`${log.size.toLocaleString()} bytes`)}`);
-			console.log(`Version   = ${hl(log.version)}`);
-			console.log(`Timestamp = ${hl(new Date(log.timestamp).toISOString())}`);
-			console.log(`Entries   = ${hl(log.entries.length)}`);
+			for (const anomaly of log.anomalies) {
+				console.log(bad(`!  ${anomaly}`));
+			}
 			let i = 1;
 			for (const entry of log.entries) {
+				const ts =
+					Number.isFinite(entry.timestamp) && entry.timestamp > 0
+						? new Date(entry.timestamp).toISOString()
+						: String(entry.timestamp);
 				console.log(
-					`${String(i++).padStart(3)})  ${hl(new Date(entry.timestamp).toISOString())}  ${entry.length.toLocaleString()} bytes  ${entry.flags & 0x01 ? '' : note('(continued in next file)')}`
+					`${String(i++).padStart(3)})  ${hl(ts)}  ${entry.length.toLocaleString()} bytes  ${entry.flags & 0x01 ? note('(end of transaction)') : ''}`
 				);
+				if (entry.anomalies) {
+					for (const anomaly of entry.anomalies) {
+						console.log(`     ${bad(`! ${anomaly}`)}`);
+					}
+				}
 			}
 			console.log();
 			return;
 		}
+
 		const logPath = join(currentDB.path, 'transaction_logs', name);
 		const logFiles = await readdir(logPath);
 		for (const logFile of logFiles) {
 			if (!logFile.endsWith('.txnlog')) continue;
 			console.log(hl(logFile));
+
+			const log = parseTransactionLog(join(logPath, logFile), { skipData: true });
+			const ts =
+				Number.isFinite(log.timestamp) && log.timestamp > 0
+					? new Date(log.timestamp).toISOString()
+					: String(log.timestamp);
+			console.log(`  Size      = ${hl(`${log.size.toLocaleString()} bytes`)}`);
+			console.log(`  Version   = ${hl(log.version)}`);
+			console.log(`  Timestamp = ${hl(ts)}`);
+			console.log(`  Entries   = ${hl(log.entries.length)}`);
+			const totalAnomalies = log.anomalies.length + log.entryAnomalyCount;
+			if (totalAnomalies > 0) {
+				console.log(`  Anomalies = ${bad(totalAnomalies)}`);
+				for (const anomaly of log.anomalies) {
+					console.log(`    ${bad(`! ${anomaly}`)}`);
+				}
+				if (log.entryAnomalyCount > 0) {
+					console.log(
+						`    ${bad(`! ${log.entryAnomalyCount} entr${log.entryAnomalyCount === 1 ? 'y has' : 'ies have'} anomalies (run "log ${name} ${logFile}" for details)`)}`
+					);
+				}
+			}
 		}
 	} else if (logs.length > 0) {
-		for (const log of logs) {
-			console.log(hl(log));
+		for (const name of logs) {
+			const files = (await readdir(join(currentDB.path, 'transaction_logs', name))).filter((f) =>
+				f.endsWith('.txnlog')
+			);
+			const maxlen = files.reduce((max, file) => Math.max(max, file.length), 0);
+			console.log(hl(name));
+			for (const file of files) {
+				const { size } = await stat(join(currentDB.path, 'transaction_logs', name, file));
+				console.log(`  ${file.padEnd(maxlen)} ${note(`(${formatBytes(size)})`)}`);
+			}
 		}
 	} else {
 		console.log('No log stores found');
@@ -533,6 +594,7 @@ async function main() {
 			const command = line[0];
 			if (!command) continue;
 			const commandFn = COMMANDS[command];
+			console.log();
 			if (commandFn) {
 				try {
 					await commandFn(line.slice(1));

package/dist/index.cjs

@@ -658,6 +658,24 @@ var Store = class {
 	*/
 	maxKeySize;
 	/**
+	* The maximum number of memtables that can be queued per column family
+	* before writes stall. Higher values absorb write bursts while flushes catch
+	* up, at the cost of memory.
+	*/
+	maxWriteBufferNumber;
+	/**
+	* The bytes of recent memtable history to retain in memory for transaction
+	* conflict checking. `-1` derives the value from
+	* `maxWriteBufferNumber * writeBufferSize`.
+	*/
+	maxWriteBufferSizeToMaintain;
+	/**
+	* The total memtable budget in bytes across all column families. When the
+	* sum of memtables reaches this size, RocksDB flushes the largest one. `0`
+	* disables the global trigger so per-CF `writeBufferSize` drives flushing.
+	*/
+	dbWriteBufferSize;
+	/**
 	* The name of the store (e.g. the column family). Defaults to `'default'`.
 	*/
 	name;
@@ -667,7 +685,8 @@ var Store = class {
 	noBlockCache;
 	/**
 	* The number of threads to use for parallel operations. This is a RocksDB
-	* option.
+	* option. When undefined, the native layer picks
+	* `max(1, hardware_concurrency() / 2)`.
 	*/
 	parallelismThreads;
 	/**
@@ -725,6 +744,11 @@ var Store = class {
 	*/
 	transactionLogsPath;
 	/**
+	* The per-column-family memtable size in bytes at which the memtable is
+	* sealed and flushed.
+	*/
+	writeBufferSize;
+	/**
 	* The function used to encode keys using the shared `keyBuffer`.
 	*/
 	writeKey;
@@ -739,6 +763,7 @@ var Store = class {
 		if (options !== void 0 && options !== null && typeof options !== "object") throw new TypeError("Database options must be an object");
 		const { keyEncoding, readKey, writeKey } = initKeyEncoder(options?.keyEncoding, options?.keyEncoder);
 		this.db = new NativeDatabase();
+		this.dbWriteBufferSize = options?.dbWriteBufferSize;
 		this.decoder = options?.decoder ?? null;
 		this.disableWAL = options?.disableWAL ?? false;
 		this.enableStats = options?.enableStats ?? false;
@@ -749,9 +774,11 @@ var Store = class {
 		this.keyBuffer = KEY_BUFFER;
 		this.keyEncoding = keyEncoding;
 		this.maxKeySize = options?.maxKeySize ?? MAX_KEY_SIZE;
+		this.maxWriteBufferNumber = options?.maxWriteBufferNumber;
+		this.maxWriteBufferSizeToMaintain = options?.maxWriteBufferSizeToMaintain;
 		this.name = options?.name ?? "default";
 		this.noBlockCache = options?.noBlockCache;
-		this.parallelismThreads = options?.parallelismThreads ?? 1;
+		this.parallelismThreads = options?.parallelismThreads;
 		this.path = path;
 		this.pessimistic = options?.pessimistic ?? false;
 		this.readOnly = options?.readOnly ?? false;
@@ -763,6 +790,7 @@ var Store = class {
 		this.transactionLogMaxSize = options?.transactionLogMaxSize;
 		this.transactionLogRetention = options?.transactionLogRetention;
 		this.transactionLogsPath = options?.transactionLogsPath;
+		this.writeBufferSize = options?.writeBufferSize;
 		this.writeKey = writeKey;
 	}
 	/**
@@ -1024,8 +1052,11 @@ var Store = class {
 	open() {
 		if (this.db.opened) return true;
 		this.db.open(this.path, {
+			dbWriteBufferSize: this.dbWriteBufferSize,
 			disableWAL: this.disableWAL,
 			enableStats: this.enableStats,
+			maxWriteBufferNumber: this.maxWriteBufferNumber,
+			maxWriteBufferSizeToMaintain: this.maxWriteBufferSizeToMaintain,
 			mode: this.pessimistic ? "pessimistic" : "optimistic",
 			name: this.name,
 			noBlockCache: this.noBlockCache,
@@ -1035,7 +1066,8 @@ var Store = class {
 			transactionLogMaxAgeThreshold: this.transactionLogMaxAgeThreshold,
 			transactionLogMaxSize: this.transactionLogMaxSize,
 			transactionLogRetentionMs: this.transactionLogRetention ? parseDuration(this.transactionLogRetention) : void 0,
-			transactionLogsPath: this.transactionLogsPath
+			transactionLogsPath: this.transactionLogsPath,
+			writeBufferSize: this.writeBufferSize
 		});
 		return false;
 	}
@@ -1828,12 +1860,14 @@ var RocksDatabase = class RocksDatabase extends DBI {
 //#endregion
 //#region src/parse-transaction-log.ts
 const { TRANSACTION_LOG_TOKEN } = constants;
+const MIN_VALID_TIMESTAMP = Date.UTC(2026, 0, 27);
+const VALID_FLAGS_MASK = 1;
 /**
 * Loads an entire transaction log file into memory.
 * @param path - The path to the transaction log file.
 * @returns The transaction log.
 */
-function parseTransactionLog(path) {
+function parseTransactionLog(path, options = {}) {
 	let stats;
 	try {
 		stats = (0, node_fs.statSync)(path);
@@ -1857,7 +1891,10 @@ function parseTransactionLog(path) {
 		const version = read(1).readUInt8(0);
 		if (version !== 1) throw new Error(`Unsupported transaction log file version: ${version}`);
 		const timestamp = read(8).readDoubleBE(0);
+		const anomalies = [];
+		if (!Number.isFinite(timestamp) || timestamp < MIN_VALID_TIMESTAMP) anomalies.push(`Header timestamp ${timestamp} predates 2026-01-27 (possible corruption)`);
 		const entries = [];
+		let entryAnomalyCount = 0;
 		while (fileOffset < size) {
 			const timestamp = read(8).readDoubleBE(0);
 			if (timestamp === 0) {
@@ -1867,15 +1904,25 @@ function parseTransactionLog(path) {
 			const length = read(4).readUInt32BE(0);
 			const flags = read(1).readUInt8(0);
 			const data = read(length);
-			entries.push({
+			const entryAnomalies = [];
+			if (!Number.isFinite(timestamp) || timestamp < MIN_VALID_TIMESTAMP) entryAnomalies.push(`timestamp ${timestamp} predates 2026-01-27 (possible corruption)`);
+			if ((flags & ~VALID_FLAGS_MASK) !== 0) entryAnomalies.push(`flags 0x${flags.toString(16).padStart(2, "0")} contains undefined bits (expected 0x00 or 0x01)`);
+			const entry = {
 				timestamp,
 				length,
 				flags,
-				data
-			});
+				data: options.skipData ? void 0 : Buffer.from(data)
+			};
+			if (entryAnomalies.length > 0) {
+				entry.anomalies = entryAnomalies;
+				entryAnomalyCount += entryAnomalies.length;
+			}
+			entries.push(entry);
 		}
 		return {
+			anomalies,
 			entries,
+			entryAnomalyCount,
 			timestamp,
 			size,
 			version
@@ -2068,7 +2115,7 @@ function loadLastPosition(transactionLog, readUncommitted) {
 //#region src/index.ts
 const versions = {
 	rocksdb: version,
-	"rocksdb-js": "1.2.0"
+	"rocksdb-js": "1.4.0"
 };
 
 //#endregion