@harperfast/rocksdb-js 1.2.0 → 1.4.0

package/dist/index.d.cts

@@ -172,6 +172,24 @@ declare class Store {
   */
   maxKeySize: number;
   /**
+  * The maximum number of memtables that can be queued per column family
+  * before writes stall. Higher values absorb write bursts while flushes catch
+  * up, at the cost of memory.
+  */
+  maxWriteBufferNumber?: number;
+  /**
+  * The bytes of recent memtable history to retain in memory for transaction
+  * conflict checking. `-1` derives the value from
+  * `maxWriteBufferNumber * writeBufferSize`.
+  */
+  maxWriteBufferSizeToMaintain?: number;
+  /**
+  * The total memtable budget in bytes across all column families. When the
+  * sum of memtables reaches this size, RocksDB flushes the largest one. `0`
+  * disables the global trigger so per-CF `writeBufferSize` drives flushing.
+  */
+  dbWriteBufferSize?: number;
+  /**
   * The name of the store (e.g. the column family). Defaults to `'default'`.
   */
   name: string;
@@ -181,9 +199,10 @@ declare class Store {
   noBlockCache?: boolean;
   /**
   * The number of threads to use for parallel operations. This is a RocksDB
-  * option.
+  * option. When undefined, the native layer picks
+  * `max(1, hardware_concurrency() / 2)`.
   */
-  parallelismThreads: number;
+  parallelismThreads?: number;
   /**
   * The path to the database.
   */
@@ -239,6 +258,11 @@ declare class Store {
   */
   transactionLogsPath?: string;
   /**
+  * The per-column-family memtable size in bytes at which the memtable is
+  * sealed and flushed.
+  */
+  writeBufferSize?: number;
+  /**
   * The function used to encode keys using the shared `keyBuffer`.
   */
   writeKey: WriteKeyFunction;
@@ -492,8 +516,11 @@ declare class NativeIteratorCls {
 }
 type NativeDatabaseMode = "optimistic" | "pessimistic";
 type NativeDatabaseOptions = {
+  dbWriteBufferSize?: number;
   disableWAL?: boolean;
   enableStats?: boolean;
+  maxWriteBufferNumber?: number;
+  maxWriteBufferSizeToMaintain?: number;
   mode?: NativeDatabaseMode;
   name?: string;
   noBlockCache?: boolean;
@@ -504,6 +531,7 @@ type NativeDatabaseOptions = {
   transactionLogMaxSize?: number;
   transactionLogRetentionMs?: number;
   transactionLogsPath?: string;
+  writeBufferSize?: number;
 };
 type ResolveCallback<T> = (value: T) => void;
 type RejectCallback = (err: Error) => void;
@@ -569,6 +597,40 @@ type NativeDatabase = {
 type RocksDatabaseConfig = {
   blockCacheSize?: number;
   compactOnClose?: boolean;
+  /**
+  * Total memtable memory limit (bytes) shared across every database opened
+  * in this process. When set, RocksDB uses a single `WriteBufferManager` so
+  * write buffers are bounded process-wide rather than per database. 0 (the
+  * default) disables the manager.
+  *
+  * Can be updated at runtime; the new size takes effect on the existing
+  * manager via `SetBufferSize`.
+  */
+  writeBufferManagerSize?: number;
+  /**
+  * When `true`, memtable memory is "charged" against the shared block cache
+  * so the block cache and write buffers draw from a single pool. During
+  * write bursts the cache shrinks to make room for memtables; once
+  * memtables flush, the cache can grow back into the reclaimed space.
+  *
+  * Has no effect when the block cache is disabled (size 0) or
+  * `writeBufferManagerSize` is 0. Must be set on the same `config()` call
+  * that first enables the manager — changing it after the manager has been
+  * created has no effect on the running instance.
+  *
+  * @default false
+  */
+  writeBufferManagerCostToCache?: boolean;
+  /**
+  * When `true`, writes are stalled once the manager's `buffer_size` is
+  * exceeded, providing a hard cap on memtable memory. When `false`,
+  * memtables are allowed to grow past the limit and flushes are simply
+  * scheduled more aggressively. Off by default to favor write throughput
+  * over hard memory bounding.
+  *
+  * @default false
+  */
+  writeBufferManagerAllowStall?: boolean;
 };
 type RegistryStatusDB = {
   path: string;
@@ -1353,13 +1415,16 @@ declare class RocksDatabase extends DBI<DBITransactional> {
 //#endregion
 //#region src/parse-transaction-log.d.ts
 interface LogEntry {
-  data: Buffer;
+  anomalies?: string[];
+  data?: Buffer;
   flags: number;
   length: number;
   timestamp?: number;
 }
 interface TransactionLog$1 {
+  anomalies: string[];
   entries: LogEntry[];
+  entryAnomalyCount: number;
   timestamp: number;
   size: number;
   version: number;
@@ -1369,7 +1434,9 @@ interface TransactionLog$1 {
 * @param path - The path to the transaction log file.
 * @returns The transaction log.
 */
-declare function parseTransactionLog(path: string): TransactionLog$1;
+declare function parseTransactionLog(path: string, options?: {
+  skipData?: boolean;
+}): TransactionLog$1;
 //#endregion
 //#region src/index.d.ts
 declare const versions: {

package/dist/index.d.mts

@@ -172,6 +172,24 @@ declare class Store {
   */
   maxKeySize: number;
   /**
+  * The maximum number of memtables that can be queued per column family
+  * before writes stall. Higher values absorb write bursts while flushes catch
+  * up, at the cost of memory.
+  */
+  maxWriteBufferNumber?: number;
+  /**
+  * The bytes of recent memtable history to retain in memory for transaction
+  * conflict checking. `-1` derives the value from
+  * `maxWriteBufferNumber * writeBufferSize`.
+  */
+  maxWriteBufferSizeToMaintain?: number;
+  /**
+  * The total memtable budget in bytes across all column families. When the
+  * sum of memtables reaches this size, RocksDB flushes the largest one. `0`
+  * disables the global trigger so per-CF `writeBufferSize` drives flushing.
+  */
+  dbWriteBufferSize?: number;
+  /**
   * The name of the store (e.g. the column family). Defaults to `'default'`.
   */
   name: string;
@@ -181,9 +199,10 @@ declare class Store {
   noBlockCache?: boolean;
   /**
   * The number of threads to use for parallel operations. This is a RocksDB
-  * option.
+  * option. When undefined, the native layer picks
+  * `max(1, hardware_concurrency() / 2)`.
   */
-  parallelismThreads: number;
+  parallelismThreads?: number;
   /**
   * The path to the database.
   */
@@ -239,6 +258,11 @@ declare class Store {
   */
   transactionLogsPath?: string;
   /**
+  * The per-column-family memtable size in bytes at which the memtable is
+  * sealed and flushed.
+  */
+  writeBufferSize?: number;
+  /**
   * The function used to encode keys using the shared `keyBuffer`.
   */
   writeKey: WriteKeyFunction;
@@ -492,8 +516,11 @@ declare class NativeIteratorCls {
 }
 type NativeDatabaseMode = "optimistic" | "pessimistic";
 type NativeDatabaseOptions = {
+  dbWriteBufferSize?: number;
   disableWAL?: boolean;
   enableStats?: boolean;
+  maxWriteBufferNumber?: number;
+  maxWriteBufferSizeToMaintain?: number;
   mode?: NativeDatabaseMode;
   name?: string;
   noBlockCache?: boolean;
@@ -504,6 +531,7 @@ type NativeDatabaseOptions = {
   transactionLogMaxSize?: number;
   transactionLogRetentionMs?: number;
   transactionLogsPath?: string;
+  writeBufferSize?: number;
 };
 type ResolveCallback<T> = (value: T) => void;
 type RejectCallback = (err: Error) => void;
@@ -569,6 +597,40 @@ type NativeDatabase = {
 type RocksDatabaseConfig = {
   blockCacheSize?: number;
   compactOnClose?: boolean;
+  /**
+  * Total memtable memory limit (bytes) shared across every database opened
+  * in this process. When set, RocksDB uses a single `WriteBufferManager` so
+  * write buffers are bounded process-wide rather than per database. 0 (the
+  * default) disables the manager.
+  *
+  * Can be updated at runtime; the new size takes effect on the existing
+  * manager via `SetBufferSize`.
+  */
+  writeBufferManagerSize?: number;
+  /**
+  * When `true`, memtable memory is "charged" against the shared block cache
+  * so the block cache and write buffers draw from a single pool. During
+  * write bursts the cache shrinks to make room for memtables; once
+  * memtables flush, the cache can grow back into the reclaimed space.
+  *
+  * Has no effect when the block cache is disabled (size 0) or
+  * `writeBufferManagerSize` is 0. Must be set on the same `config()` call
+  * that first enables the manager — changing it after the manager has been
+  * created has no effect on the running instance.
+  *
+  * @default false
+  */
+  writeBufferManagerCostToCache?: boolean;
+  /**
+  * When `true`, writes are stalled once the manager's `buffer_size` is
+  * exceeded, providing a hard cap on memtable memory. When `false`,
+  * memtables are allowed to grow past the limit and flushes are simply
+  * scheduled more aggressively. Off by default to favor write throughput
+  * over hard memory bounding.
+  *
+  * @default false
+  */
+  writeBufferManagerAllowStall?: boolean;
 };
 type RegistryStatusDB = {
   path: string;
@@ -1353,13 +1415,16 @@ declare class RocksDatabase extends DBI<DBITransactional> {
 //#endregion
 //#region src/parse-transaction-log.d.ts
 interface LogEntry {
-  data: Buffer;
+  anomalies?: string[];
+  data?: Buffer;
   flags: number;
   length: number;
   timestamp?: number;
 }
 interface TransactionLog$1 {
+  anomalies: string[];
   entries: LogEntry[];
+  entryAnomalyCount: number;
   timestamp: number;
   size: number;
   version: number;
@@ -1369,7 +1434,9 @@ interface TransactionLog$1 {
 * @param path - The path to the transaction log file.
 * @returns The transaction log.
 */
-declare function parseTransactionLog(path: string): TransactionLog$1;
+declare function parseTransactionLog(path: string, options?: {
+  skipData?: boolean;
+}): TransactionLog$1;
 //#endregion
 //#region src/index.d.ts
 declare const versions: {

package/dist/index.mjs

@@ -633,6 +633,24 @@ var Store = class {
 	*/
 	maxKeySize;
 	/**
+	* The maximum number of memtables that can be queued per column family
+	* before writes stall. Higher values absorb write bursts while flushes catch
+	* up, at the cost of memory.
+	*/
+	maxWriteBufferNumber;
+	/**
+	* The bytes of recent memtable history to retain in memory for transaction
+	* conflict checking. `-1` derives the value from
+	* `maxWriteBufferNumber * writeBufferSize`.
+	*/
+	maxWriteBufferSizeToMaintain;
+	/**
+	* The total memtable budget in bytes across all column families. When the
+	* sum of memtables reaches this size, RocksDB flushes the largest one. `0`
+	* disables the global trigger so per-CF `writeBufferSize` drives flushing.
+	*/
+	dbWriteBufferSize;
+	/**
 	* The name of the store (e.g. the column family). Defaults to `'default'`.
 	*/
 	name;
@@ -642,7 +660,8 @@ var Store = class {
 	noBlockCache;
 	/**
 	* The number of threads to use for parallel operations. This is a RocksDB
-	* option.
+	* option. When undefined, the native layer picks
+	* `max(1, hardware_concurrency() / 2)`.
 	*/
 	parallelismThreads;
 	/**
@@ -700,6 +719,11 @@ var Store = class {
 	*/
 	transactionLogsPath;
 	/**
+	* The per-column-family memtable size in bytes at which the memtable is
+	* sealed and flushed.
+	*/
+	writeBufferSize;
+	/**
 	* The function used to encode keys using the shared `keyBuffer`.
 	*/
 	writeKey;
@@ -714,6 +738,7 @@ var Store = class {
 		if (options !== void 0 && options !== null && typeof options !== "object") throw new TypeError("Database options must be an object");
 		const { keyEncoding, readKey, writeKey } = initKeyEncoder(options?.keyEncoding, options?.keyEncoder);
 		this.db = new NativeDatabase();
+		this.dbWriteBufferSize = options?.dbWriteBufferSize;
 		this.decoder = options?.decoder ?? null;
 		this.disableWAL = options?.disableWAL ?? false;
 		this.enableStats = options?.enableStats ?? false;
@@ -724,9 +749,11 @@ var Store = class {
 		this.keyBuffer = KEY_BUFFER;
 		this.keyEncoding = keyEncoding;
 		this.maxKeySize = options?.maxKeySize ?? MAX_KEY_SIZE;
+		this.maxWriteBufferNumber = options?.maxWriteBufferNumber;
+		this.maxWriteBufferSizeToMaintain = options?.maxWriteBufferSizeToMaintain;
 		this.name = options?.name ?? "default";
 		this.noBlockCache = options?.noBlockCache;
-		this.parallelismThreads = options?.parallelismThreads ?? 1;
+		this.parallelismThreads = options?.parallelismThreads;
 		this.path = path;
 		this.pessimistic = options?.pessimistic ?? false;
 		this.readOnly = options?.readOnly ?? false;
@@ -738,6 +765,7 @@ var Store = class {
 		this.transactionLogMaxSize = options?.transactionLogMaxSize;
 		this.transactionLogRetention = options?.transactionLogRetention;
 		this.transactionLogsPath = options?.transactionLogsPath;
+		this.writeBufferSize = options?.writeBufferSize;
 		this.writeKey = writeKey;
 	}
 	/**
@@ -999,8 +1027,11 @@ var Store = class {
 	open() {
 		if (this.db.opened) return true;
 		this.db.open(this.path, {
+			dbWriteBufferSize: this.dbWriteBufferSize,
 			disableWAL: this.disableWAL,
 			enableStats: this.enableStats,
+			maxWriteBufferNumber: this.maxWriteBufferNumber,
+			maxWriteBufferSizeToMaintain: this.maxWriteBufferSizeToMaintain,
 			mode: this.pessimistic ? "pessimistic" : "optimistic",
 			name: this.name,
 			noBlockCache: this.noBlockCache,
@@ -1010,7 +1041,8 @@ var Store = class {
 			transactionLogMaxAgeThreshold: this.transactionLogMaxAgeThreshold,
 			transactionLogMaxSize: this.transactionLogMaxSize,
 			transactionLogRetentionMs: this.transactionLogRetention ? parseDuration(this.transactionLogRetention) : void 0,
-			transactionLogsPath: this.transactionLogsPath
+			transactionLogsPath: this.transactionLogsPath,
+			writeBufferSize: this.writeBufferSize
 		});
 		return false;
 	}
@@ -1803,12 +1835,14 @@ var RocksDatabase = class RocksDatabase extends DBI {
 //#endregion
 //#region src/parse-transaction-log.ts
 const { TRANSACTION_LOG_TOKEN } = constants;
+const MIN_VALID_TIMESTAMP = Date.UTC(2026, 0, 27);
+const VALID_FLAGS_MASK = 1;
 /**
 * Loads an entire transaction log file into memory.
 * @param path - The path to the transaction log file.
 * @returns The transaction log.
 */
-function parseTransactionLog(path) {
+function parseTransactionLog(path, options = {}) {
 	let stats;
 	try {
 		stats = statSync(path);
@@ -1832,7 +1866,10 @@ function parseTransactionLog(path) {
 		const version = read(1).readUInt8(0);
 		if (version !== 1) throw new Error(`Unsupported transaction log file version: ${version}`);
 		const timestamp = read(8).readDoubleBE(0);
+		const anomalies = [];
+		if (!Number.isFinite(timestamp) || timestamp < MIN_VALID_TIMESTAMP) anomalies.push(`Header timestamp ${timestamp} predates 2026-01-27 (possible corruption)`);
 		const entries = [];
+		let entryAnomalyCount = 0;
 		while (fileOffset < size) {
 			const timestamp = read(8).readDoubleBE(0);
 			if (timestamp === 0) {
@@ -1842,15 +1879,25 @@ function parseTransactionLog(path) {
 			const length = read(4).readUInt32BE(0);
 			const flags = read(1).readUInt8(0);
 			const data = read(length);
-			entries.push({
+			const entryAnomalies = [];
+			if (!Number.isFinite(timestamp) || timestamp < MIN_VALID_TIMESTAMP) entryAnomalies.push(`timestamp ${timestamp} predates 2026-01-27 (possible corruption)`);
+			if ((flags & ~VALID_FLAGS_MASK) !== 0) entryAnomalies.push(`flags 0x${flags.toString(16).padStart(2, "0")} contains undefined bits (expected 0x00 or 0x01)`);
+			const entry = {
 				timestamp,
 				length,
 				flags,
-				data
-			});
+				data: options.skipData ? void 0 : Buffer.from(data)
+			};
+			if (entryAnomalies.length > 0) {
+				entry.anomalies = entryAnomalies;
+				entryAnomalyCount += entryAnomalies.length;
+			}
+			entries.push(entry);
 		}
 		return {
+			anomalies,
 			entries,
+			entryAnomalyCount,
 			timestamp,
 			size,
 			version
@@ -2043,7 +2090,7 @@ function loadLastPosition(transactionLog, readUncommitted) {
 //#region src/index.ts
 const versions = {
 	rocksdb: version,
-	"rocksdb-js": "1.2.0"
+	"rocksdb-js": "1.4.0"
 };
 
 //#endregion