@harperfast/rocksdb-js 1.1.1 → 1.3.0

package/dist/index.cjs

@@ -453,42 +453,62 @@ var DBI = class DBI {
 
 //#endregion
 //#region src/dbi-iterator.ts
+const { ITERATOR_RESULT_DONE, ITERATOR_RESULT_FAST } = constants;
+const DONE_RESULT = Object.freeze({
+	done: true,
+	value: void 0
+});
 /**
-* Wraps an iterator, namely the `NativeIterator` class, and decodes the key
-* and value.
+* Wraps the `NativeIterator` C++ binding, decoding keys and values from the
+* shared key/value buffers (fast path) or from per-iteration buffers (slow
+* fallback path used for oversized data or stable-buffer decoders).
+*
+* The native `next()` returns a primitive signal (and writes lengths to the
+* shared `ITERATOR_STATE` buffer) instead of constructing a JS result object,
+* so this class is responsible for building the `IteratorResult`.
 */
 var DBIterator = class {
 	iterator;
 	store;
 	#includeValues;
-	constructor(iterator, store, options) {
+	constructor(iterator, store, includeValues) {
 		this.iterator = iterator;
 		this.store = store;
-		this.#includeValues = options?.values ?? true;
+		this.#includeValues = includeValues;
 	}
 	[Symbol.iterator]() {
 		return this;
 	}
-	next(...[_value]) {
+	next() {
 		const result = this.iterator.next();
-		if (result.done) return result;
+		if (result === ITERATOR_RESULT_DONE) return DONE_RESULT;
+		const includeValues = this.#includeValues;
 		const value = {};
-		value.key = this.store.decodeKey(result.value.key);
-		if (this.#includeValues) value.value = this.store.decodeValue(result.value.value);
+		if (result === ITERATOR_RESULT_FAST) {
+			value.key = this.store.readKey(KEY_BUFFER, 0, ITERATOR_STATE[0]);
+			if (includeValues) {
+				VALUE_BUFFER.end = ITERATOR_STATE[1];
+				value.value = this.store.decodeValue(VALUE_BUFFER);
+			}
+		} else {
+			const slow = result;
+			value.key = this.store.decodeKey(slow.key);
+			if (includeValues && slow.value !== void 0) value.value = this.store.decodeValue(slow.value);
+		}
 		return {
 			done: false,
 			value
 		};
 	}
 	return(value) {
-		if (this.iterator.return) return this.iterator.return(value);
+		this.iterator.return?.();
 		return {
 			done: true,
 			value
 		};
 	}
 	throw(err) {
-		if (this.iterator.throw) return this.iterator.throw(err);
+		this.iterator.throw?.(err);
 		throw err;
 	}
 };
@@ -563,13 +583,18 @@ function createFixedBuffer(size) {
 
 //#endregion
 //#region src/store.ts
-const { ONLY_IF_IN_MEMORY_CACHE_FLAG, NOT_IN_MEMORY_CACHE_FLAG, ALWAYS_CREATE_NEW_BUFFER_FLAG } = constants;
+const { ONLY_IF_IN_MEMORY_CACHE_FLAG, NOT_IN_MEMORY_CACHE_FLAG, ALWAYS_CREATE_NEW_BUFFER_FLAG, ITERATOR_REVERSE_FLAG, ITERATOR_INCLUSIVE_END_FLAG, ITERATOR_EXCLUSIVE_START_FLAG, ITERATOR_INCLUDE_VALUES_FLAG, ITERATOR_NEEDS_STABLE_VALUE_BUFFER_FLAG, ITERATOR_CONTEXT_IS_TRANSACTION_FLAG } = constants;
 const KEY_BUFFER_SIZE = 4096;
 const KEY_BUFFER = createFixedBuffer(KEY_BUFFER_SIZE);
 const VALUE_BUFFER = createFixedBuffer(64 * 1024);
+/**
+* Backing buffer for the shared iterator state. Layout (Uint32Array view):
+*   [0] = key length written into KEY_BUFFER by the most recent iterator step
+*   [1] = value length written into VALUE_BUFFER by the most recent step
+*/
+const ITERATOR_STATE_BUFFER = Buffer.allocUnsafeSlow(8);
+const ITERATOR_STATE = new Uint32Array(ITERATOR_STATE_BUFFER.buffer, ITERATOR_STATE_BUFFER.byteOffset, 2);
 const MAX_KEY_SIZE = 1024 * 1024;
-const RESET_BUFFER_MODE = 1024;
-const REUSE_BUFFER_MODE = 512;
 const SAVE_BUFFER_SIZE = 8192;
 /**
 * A store wraps the `NativeDatabase` binding and database settings so that a
@@ -631,6 +656,24 @@ var Store = class {
 	*/
 	maxKeySize;
 	/**
+	* The maximum number of memtables that can be queued per column family
+	* before writes stall. Higher values absorb write bursts while flushes catch
+	* up, at the cost of memory.
+	*/
+	maxWriteBufferNumber;
+	/**
+	* The bytes of recent memtable history to retain in memory for transaction
+	* conflict checking. `-1` derives the value from
+	* `maxWriteBufferNumber * writeBufferSize`.
+	*/
+	maxWriteBufferSizeToMaintain;
+	/**
+	* The total memtable budget in bytes across all column families. When the
+	* sum of memtables reaches this size, RocksDB flushes the largest one. `0`
+	* disables the global trigger so per-CF `writeBufferSize` drives flushing.
+	*/
+	dbWriteBufferSize;
+	/**
 	* The name of the store (e.g. the column family). Defaults to `'default'`.
 	*/
 	name;
@@ -640,7 +683,8 @@ var Store = class {
 	noBlockCache;
 	/**
 	* The number of threads to use for parallel operations. This is a RocksDB
-	* option.
+	* option. When undefined, the native layer picks
+	* `max(1, hardware_concurrency() / 2)`.
 	*/
 	parallelismThreads;
 	/**
@@ -698,6 +742,11 @@ var Store = class {
 	*/
 	transactionLogsPath;
 	/**
+	* The per-column-family memtable size in bytes at which the memtable is
+	* sealed and flushed.
+	*/
+	writeBufferSize;
+	/**
 	* The function used to encode keys using the shared `keyBuffer`.
 	*/
 	writeKey;
@@ -712,6 +761,7 @@ var Store = class {
 		if (options !== void 0 && options !== null && typeof options !== "object") throw new TypeError("Database options must be an object");
 		const { keyEncoding, readKey, writeKey } = initKeyEncoder(options?.keyEncoding, options?.keyEncoder);
 		this.db = new NativeDatabase();
+		this.dbWriteBufferSize = options?.dbWriteBufferSize;
 		this.decoder = options?.decoder ?? null;
 		this.disableWAL = options?.disableWAL ?? false;
 		this.enableStats = options?.enableStats ?? false;
@@ -722,9 +772,11 @@ var Store = class {
 		this.keyBuffer = KEY_BUFFER;
 		this.keyEncoding = keyEncoding;
 		this.maxKeySize = options?.maxKeySize ?? MAX_KEY_SIZE;
+		this.maxWriteBufferNumber = options?.maxWriteBufferNumber;
+		this.maxWriteBufferSizeToMaintain = options?.maxWriteBufferSizeToMaintain;
 		this.name = options?.name ?? "default";
 		this.noBlockCache = options?.noBlockCache;
-		this.parallelismThreads = options?.parallelismThreads ?? 1;
+		this.parallelismThreads = options?.parallelismThreads;
 		this.path = path;
 		this.pessimistic = options?.pessimistic ?? false;
 		this.readOnly = options?.readOnly ?? false;
@@ -736,6 +788,7 @@ var Store = class {
 		this.transactionLogMaxSize = options?.transactionLogMaxSize;
 		this.transactionLogRetention = options?.transactionLogRetention;
 		this.transactionLogsPath = options?.transactionLogsPath;
+		this.writeBufferSize = options?.writeBufferSize;
 		this.writeKey = writeKey;
 	}
 	/**
@@ -745,6 +798,52 @@ var Store = class {
 		this.db.close();
 	}
 	/**
+	* Compacts the entire key range of the database asynchronously.
+	* This triggers manual compaction which removes tombstones and reclaims space.
+	*
+	* @example
+	* ```typescript
+	* const db = RocksDatabase.open('/path/to/database');
+	* await db.compact();
+	* ```
+	*/
+	compact(options) {
+		let startBuffer;
+		let endBuffer;
+		if (options?.start !== void 0) {
+			const start = this.encodeKey(options.start);
+			startBuffer = Buffer.from(start.subarray(start.start, start.end));
+		}
+		if (options?.end !== void 0) {
+			const end = this.encodeKey(options.end);
+			endBuffer = Buffer.from(end.subarray(end.start, end.end));
+		}
+		return new Promise((resolve, reject) => this.db.compact(resolve, reject, startBuffer, endBuffer));
+	}
+	/**
+	* Compacts the entire key range of the database synchronously.
+	* This triggers manual compaction which removes tombstones and reclaims space.
+	*
+	* @example
+	* ```typescript
+	* const db = RocksDatabase.open('/path/to/database');
+	* db.compactSync();
+	* ```
+	*/
+	compactSync(options) {
+		let startBuffer;
+		let endBuffer;
+		if (options?.start !== void 0) {
+			const start = this.encodeKey(options.start);
+			startBuffer = Buffer.from(start.subarray(start.start, start.end));
+		}
+		if (options?.end !== void 0) {
+			const end = this.encodeKey(options.end);
+			endBuffer = Buffer.from(end.subarray(end.start, end.end));
+		}
+		this.db.compactSync(startBuffer, endBuffer);
+	}
+	/**
 	* Decodes a key from the database.
 	*
 	* @param key - The key to decode.
@@ -785,7 +884,7 @@ var Store = class {
 	encodeValue(value) {
 		if (value && value["binary-data"]) return value["binary-data"];
 		if (typeof this.encoder?.encode === "function") {
-			if (this.encoder.copyBuffers) return this.encoder.encode(value, REUSE_BUFFER_MODE | RESET_BUFFER_MODE);
+			if (this.encoder.copyBuffers) return this.encoder.encode(value, 1536);
 			const valueBuffer = this.encoder.encode(value);
 			if (typeof valueBuffer === "string") return Buffer.from(valueBuffer);
 			return valueBuffer;
@@ -832,27 +931,45 @@ var Store = class {
 	}
 	getRange(context, options) {
 		if (!this.db.opened) throw new Error("Database not open");
-		options = { ...options };
-		const unencodedStartKey = options.key ?? options.start;
-		if (unencodedStartKey !== void 0) {
-			const start = this.encodeKey(unencodedStartKey);
-			options.start = Buffer.from(start.subarray(start.start, start.end));
+		let startUnencoded = options?.key ?? options?.start;
+		let endUnencoded = options?.key ?? options?.end;
+		const includeValues = options?.values ?? true;
+		const reverse = options?.reverse ?? false;
+		let exclusiveStart = options?.exclusiveStart ?? false;
+		let inclusiveEnd = options?.inclusiveEnd ?? false;
+		if (options?.key !== void 0) inclusiveEnd = true;
+		if (reverse) {
+			const tmp = startUnencoded;
+			startUnencoded = endUnencoded;
+			endUnencoded = tmp;
+			exclusiveStart = options?.exclusiveStart ?? true;
+			inclusiveEnd = options?.inclusiveEnd ?? true;
 		}
-		if (options.key !== void 0) {
-			options.end = options.start;
-			options.inclusiveEnd = true;
-		} else if (options.end !== void 0) {
-			const end = this.encodeKey(options.end);
-			options.end = Buffer.from(end.subarray(end.start, end.end));
+		const keyBuffer = this.keyBuffer;
+		let startKeyEnd = 0;
+		let endKeyStart = 0;
+		let endKeyEnd = 0;
+		if (startUnencoded !== void 0) {
+			startKeyEnd = this.writeKey(startUnencoded, keyBuffer, 0);
+			if (startKeyEnd === 0) throw new Error("Zero length key is not allowed");
 		}
-		if (options.reverse) {
-			const start = options.start;
-			options.start = options.end;
-			options.end = start;
-			options.exclusiveStart = options.exclusiveStart ?? true;
-			options.inclusiveEnd = options.inclusiveEnd ?? true;
+		if (endUnencoded !== void 0) if (endUnencoded === startUnencoded) {
+			endKeyStart = 0;
+			endKeyEnd = startKeyEnd;
+		} else {
+			endKeyStart = startKeyEnd;
+			endKeyEnd = this.writeKey(endUnencoded, keyBuffer, endKeyStart);
+			if (endKeyEnd === endKeyStart) throw new Error("Zero length key is not allowed");
 		}
-		return new _harperfast_extended_iterable.ExtendedIterable(new DBIterator(new NativeIterator(context, options), this, options));
+		let flags = 0;
+		if (reverse) flags |= ITERATOR_REVERSE_FLAG;
+		if (exclusiveStart) flags |= ITERATOR_EXCLUSIVE_START_FLAG;
+		if (inclusiveEnd) flags |= ITERATOR_INCLUSIVE_END_FLAG;
+		if (includeValues) flags |= ITERATOR_INCLUDE_VALUES_FLAG;
+		if (includeValues && !this.decoderCopies) flags |= ITERATOR_NEEDS_STABLE_VALUE_BUFFER_FLAG;
+		if (context !== this.db) flags |= ITERATOR_CONTEXT_IS_TRANSACTION_FLAG;
+		const advancedOptions = options !== void 0 && (options.adaptiveReadahead !== void 0 || options.asyncIO !== void 0 || options.autoReadaheadSize !== void 0 || options.backgroundPurgeOnIteratorCleanup !== void 0 || options.fillCache !== void 0 || options.readaheadSize !== void 0 || options.tailing !== void 0) ? options : void 0;
+		return new _harperfast_extended_iterable.ExtendedIterable(new DBIterator(new NativeIterator(context, flags, startKeyEnd, endKeyStart, endKeyEnd, advancedOptions), this, includeValues));
 	}
 	getSync(context, key, alwaysCreateNewBuffer = false, options) {
 		const keyParam = getKeyParam(this.encodeKey(key));
@@ -933,8 +1050,11 @@ var Store = class {
 	open() {
 		if (this.db.opened) return true;
 		this.db.open(this.path, {
+			dbWriteBufferSize: this.dbWriteBufferSize,
 			disableWAL: this.disableWAL,
 			enableStats: this.enableStats,
+			maxWriteBufferNumber: this.maxWriteBufferNumber,
+			maxWriteBufferSizeToMaintain: this.maxWriteBufferSizeToMaintain,
 			mode: this.pessimistic ? "pessimistic" : "optimistic",
 			name: this.name,
 			noBlockCache: this.noBlockCache,
@@ -944,7 +1064,8 @@ var Store = class {
 			transactionLogMaxAgeThreshold: this.transactionLogMaxAgeThreshold,
 			transactionLogMaxSize: this.transactionLogMaxSize,
 			transactionLogRetentionMs: this.transactionLogRetention ? parseDuration(this.transactionLogRetention) : void 0,
-			transactionLogsPath: this.transactionLogsPath
+			transactionLogsPath: this.transactionLogsPath,
+			writeBufferSize: this.writeBufferSize
 		});
 		return false;
 	}
@@ -1219,6 +1340,32 @@ var RocksDatabase = class RocksDatabase extends DBI {
 		this.store.close();
 	}
 	/**
+	* Compacts the entire key range of the database asynchronously.
+	* This triggers manual compaction which removes tombstones and reclaims space.
+	*
+	* @example
+	* ```typescript
+	* const db = RocksDatabase.open('/path/to/database');
+	* await db.compact();
+	* ```
+	*/
+	compact(options) {
+		return this.store.compact(options);
+	}
+	/**
+	* Compacts the entire key range of the database synchronously.
+	* This triggers manual compaction which removes tombstones and reclaims space.
+	*
+	* @example
+	* ```typescript
+	* const db = RocksDatabase.open('/path/to/database');
+	* db.compactSync();
+	* ```
+	*/
+	compactSync(options) {
+		return this.store.compactSync(options);
+	}
+	/**
 	* Returns the list of column families in the RocksDB database.
 	*/
 	get columns() {
@@ -1300,6 +1447,19 @@ var RocksDatabase = class RocksDatabase extends DBI {
 		return this.store.db.getDBProperty(propertyName);
 	}
 	/**
+	* Retrieves the estimated number of keys in the database.
+	*
+	* @example
+	* ```typescript
+	* const db = RocksDatabase.open('/path/to/database');
+	* const estimated = db.getEstimatedKeyCount();
+	* console.log(estimated);
+	* ```
+	*/
+	getEstimatedKeyCount() {
+		return this.getDBIntProperty("rocksdb.estimate-num-keys") ?? 0;
+	}
+	/**
 	* Returns the current timestamp as a monotonically increasing timestamp in
 	* milliseconds represented as a decimal number.
 	*
@@ -1438,6 +1598,7 @@ var RocksDatabase = class RocksDatabase extends DBI {
 		if (store.open()) return this;
 		store.db.setDefaultValueBuffer(VALUE_BUFFER);
 		store.db.setDefaultKeyBuffer(KEY_BUFFER);
+		store.db.setIteratorState(ITERATOR_STATE_BUFFER);
 		/**
 		* The encoder initialization precedence is:
 		* 1. encoder.Encoder
@@ -1809,6 +1970,7 @@ Object.defineProperty(TransactionLog.prototype, "query", { value({ start, end, e
 			logBuffer.logId = 0;
 			logBuffer.size = 0;
 			logBuffer.dataView = new DataView(logBuffer.buffer);
+			size = 0;
 		}
 	}
 	dataView = logBuffer.dataView;
@@ -1936,7 +2098,7 @@ function loadLastPosition(transactionLog, readUncommitted) {
 //#region src/index.ts
 const versions = {
 	rocksdb: version,
-	"rocksdb-js": "1.1.1"
+	"rocksdb-js": "1.3.0"
 };
 
 //#endregion