@harperfast/rocksdb-js 1.1.1 → 1.3.0

package/dist/index.d.cts

@@ -36,16 +36,21 @@ interface DBIteratorValue<T> {
   value: T;
 }
 /**
-* Wraps an iterator, namely the `NativeIterator` class, and decodes the key
-* and value.
+* Wraps the `NativeIterator` C++ binding, decoding keys and values from the
+* shared key/value buffers (fast path) or from per-iteration buffers (slow
+* fallback path used for oversized data or stable-buffer decoders).
+*
+* The native `next()` returns a primitive signal (and writes lengths to the
+* shared `ITERATOR_STATE` buffer) instead of constructing a JS result object,
+* so this class is responsible for building the `IteratorResult`.
 */
 declare class DBIterator<T> implements Iterator<DBIteratorValue<T>> {
   #private;
-  iterator: Iterator<DBIteratorValue<T>>;
+  iterator: InstanceType<typeof NativeIteratorCls>;
   store: Store;
-  constructor(iterator: Iterator<DBIteratorValue<T>>, store: Store, options?: IteratorOptions & T);
+  constructor(iterator: InstanceType<typeof NativeIteratorCls>, store: Store, includeValues: boolean);
   [Symbol.iterator](): Iterator<DBIteratorValue<T>>;
-  next(...[_value]: [] | [any]): IteratorResult<DBIteratorValue<T>>;
+  next(): IteratorResult<DBIteratorValue<T>>;
   return(value?: any): IteratorResult<DBIteratorValue<T>, any>;
   throw(err: unknown): IteratorResult<DBIteratorValue<T>, any>;
 }
@@ -57,6 +62,10 @@ type StoreIteratorOptions = IteratorOptions & DBITransactional;
 type StorePutOptions = PutOptions & DBITransactional;
 type StoreRangeOptions = RangeOptions & DBITransactional;
 type StoreRemoveOptions = DBITransactional | unknown;
+type CompactOptions = {
+  start?: Key;
+  end?: Key;
+};
 /**
 * Options for the `Store` class.
 */
@@ -163,6 +172,24 @@ declare class Store {
   */
   maxKeySize: number;
   /**
+  * The maximum number of memtables that can be queued per column family
+  * before writes stall. Higher values absorb write bursts while flushes catch
+  * up, at the cost of memory.
+  */
+  maxWriteBufferNumber?: number;
+  /**
+  * The bytes of recent memtable history to retain in memory for transaction
+  * conflict checking. `-1` derives the value from
+  * `maxWriteBufferNumber * writeBufferSize`.
+  */
+  maxWriteBufferSizeToMaintain?: number;
+  /**
+  * The total memtable budget in bytes across all column families. When the
+  * sum of memtables reaches this size, RocksDB flushes the largest one. `0`
+  * disables the global trigger so per-CF `writeBufferSize` drives flushing.
+  */
+  dbWriteBufferSize?: number;
+  /**
   * The name of the store (e.g. the column family). Defaults to `'default'`.
   */
   name: string;
@@ -172,9 +199,10 @@ declare class Store {
   noBlockCache?: boolean;
   /**
   * The number of threads to use for parallel operations. This is a RocksDB
-  * option.
+  * option. When undefined, the native layer picks
+  * `max(1, hardware_concurrency() / 2)`.
   */
-  parallelismThreads: number;
+  parallelismThreads?: number;
   /**
   * The path to the database.
   */
@@ -230,6 +258,11 @@ declare class Store {
   */
   transactionLogsPath?: string;
   /**
+  * The per-column-family memtable size in bytes at which the memtable is
+  * sealed and flushed.
+  */
+  writeBufferSize?: number;
+  /**
   * The function used to encode keys using the shared `keyBuffer`.
   */
   writeKey: WriteKeyFunction;
@@ -245,6 +278,28 @@ declare class Store {
   */
   close(): void;
   /**
+  * Compacts the entire key range of the database asynchronously.
+  * This triggers manual compaction which removes tombstones and reclaims space.
+  *
+  * @example
+  * ```typescript
+  * const db = RocksDatabase.open('/path/to/database');
+  * await db.compact();
+  * ```
+  */
+  compact(options?: CompactOptions): Promise<void>;
+  /**
+  * Compacts the entire key range of the database synchronously.
+  * This triggers manual compaction which removes tombstones and reclaims space.
+  *
+  * @example
+  * ```typescript
+  * const db = RocksDatabase.open('/path/to/database');
+  * db.compactSync();
+  * ```
+  */
+  compactSync(options?: CompactOptions): void;
+  /**
   * Decodes a key from the database.
   *
   * @param key - The key to decode.
@@ -429,10 +484,43 @@ type TransactionLog = {
   _lastCommittedPosition: Float64Array;
   _logBuffers: Map<number, WeakRef<LogBuffer>>;
 };
+/**
+* Shape of options that can be passed to the native iterator constructor for
+* the rare case of advanced RocksDB ReadOptions overrides. Common iterator
+* options are passed via the bitmask `flags` argument instead.
+*/
+type NativeIteratorAdvancedOptions = {
+  adaptiveReadahead?: boolean;
+  asyncIO?: boolean;
+  autoReadaheadSize?: boolean;
+  backgroundPurgeOnIteratorCleanup?: boolean;
+  fillCache?: boolean;
+  readaheadSize?: number;
+  tailing?: boolean;
+};
+/**
+* The result of a single native iterator step. A number value matches one of
+* the `ITERATOR_RESULT_*` constants. The slow-path object is returned when
+* the data does not fit in the shared key/value buffers, or when the decoder
+* needs a stable value buffer.
+*/
+type NativeIteratorResult = number | {
+  key: Buffer;
+  value?: Buffer;
+};
+declare class NativeIteratorCls {
+  constructor(context: StoreContext, flags: number, startKeyEnd: number, endKeyStart: number, endKeyEnd: number, options?: NativeIteratorAdvancedOptions);
+  next(): NativeIteratorResult;
+  return(): void;
+  throw(err?: unknown): void;
+}
 type NativeDatabaseMode = "optimistic" | "pessimistic";
 type NativeDatabaseOptions = {
+  dbWriteBufferSize?: number;
   disableWAL?: boolean;
   enableStats?: boolean;
+  maxWriteBufferNumber?: number;
+  maxWriteBufferSizeToMaintain?: number;
   mode?: NativeDatabaseMode;
   name?: string;
   noBlockCache?: boolean;
@@ -443,6 +531,7 @@ type NativeDatabaseOptions = {
   transactionLogMaxSize?: number;
   transactionLogRetentionMs?: number;
   transactionLogsPath?: string;
+  writeBufferSize?: number;
 };
 type ResolveCallback<T> = (value: T) => void;
 type RejectCallback = (err: Error) => void;
@@ -469,6 +558,8 @@ type NativeDatabase = {
   clear(resolve: ResolveCallback<void>, reject: RejectCallback): void;
   clearSync(): void;
   close(): void;
+  compact(resolve: ResolveCallback<void>, reject: RejectCallback, start?: Key, end?: Key): void;
+  compactSync(start?: Key, end?: Key): void;
   columns: string[];
   destroy(): void;
   drop(resolve: ResolveCallback<void>, reject: RejectCallback): void;
@@ -497,6 +588,7 @@ type NativeDatabase = {
   removeSync(key: BufferWithDataView, txnId?: number): void;
   setDefaultKeyBuffer(buffer: Buffer | Uint8Array | null): void;
   setDefaultValueBuffer(buffer: Buffer | Uint8Array | null): void;
+  setIteratorState(buffer: Buffer | Uint8Array): void;
   tryLock(key: BufferWithDataView, callback?: () => void): boolean;
   unlock(key: BufferWithDataView): void;
   useLog(name: string): TransactionLog;
@@ -504,6 +596,41 @@ type NativeDatabase = {
 };
 type RocksDatabaseConfig = {
   blockCacheSize?: number;
+  compactOnClose?: boolean;
+  /**
+  * Total memtable memory limit (bytes) shared across every database opened
+  * in this process. When set, RocksDB uses a single `WriteBufferManager` so
+  * write buffers are bounded process-wide rather than per database. 0 (the
+  * default) disables the manager.
+  *
+  * Can be updated at runtime; the new size takes effect on the existing
+  * manager via `SetBufferSize`.
+  */
+  writeBufferManagerSize?: number;
+  /**
+  * When `true`, memtable memory is "charged" against the shared block cache
+  * so the block cache and write buffers draw from a single pool. During
+  * write bursts the cache shrinks to make room for memtables; once
+  * memtables flush, the cache can grow back into the reclaimed space.
+  *
+  * Has no effect when the block cache is disabled (size 0) or
+  * `writeBufferManagerSize` is 0. Must be set on the same `config()` call
+  * that first enables the manager — changing it after the manager has been
+  * created has no effect on the running instance.
+  *
+  * @default false
+  */
+  writeBufferManagerCostToCache?: boolean;
+  /**
+  * When `true`, writes are stalled once the manager's `buffer_size` is
+  * exceeded, providing a hard cap on memtable memory. When `false`,
+  * memtables are allowed to grow past the limit and flushes are simply
+  * scheduled more aggressively. Off by default to favor write throughput
+  * over hard memory bounding.
+  *
+  * @default false
+  */
+  writeBufferManagerAllowStall?: boolean;
 };
 type RegistryStatusDB = {
   path: string;
@@ -523,6 +650,14 @@ declare const constants: {
   TRANSACTION_LOG_TOKEN: number;
   TRANSACTION_LOG_ENTRY_HEADER_SIZE: number;
   TRANSACTION_LOG_FILE_HEADER_SIZE: number;
+  ITERATOR_REVERSE_FLAG: number;
+  ITERATOR_INCLUSIVE_END_FLAG: number;
+  ITERATOR_EXCLUSIVE_START_FLAG: number;
+  ITERATOR_INCLUDE_VALUES_FLAG: number;
+  ITERATOR_NEEDS_STABLE_VALUE_BUFFER_FLAG: number;
+  ITERATOR_CONTEXT_IS_TRANSACTION_FLAG: number;
+  ITERATOR_RESULT_DONE: number;
+  ITERATOR_RESULT_FAST: number;
 };
 declare const NativeDatabase: NativeDatabase;
 declare const NativeTransaction: NativeTransaction;
@@ -975,6 +1110,28 @@ declare class RocksDatabase extends DBI<DBITransactional> {
   */
   close(): void;
   /**
+  * Compacts the entire key range of the database asynchronously.
+  * This triggers manual compaction which removes tombstones and reclaims space.
+  *
+  * @example
+  * ```typescript
+  * const db = RocksDatabase.open('/path/to/database');
+  * await db.compact();
+  * ```
+  */
+  compact(options?: CompactOptions): Promise<void>;
+  /**
+  * Compacts the entire key range of the database synchronously.
+  * This triggers manual compaction which removes tombstones and reclaims space.
+  *
+  * @example
+  * ```typescript
+  * const db = RocksDatabase.open('/path/to/database');
+  * db.compactSync();
+  * ```
+  */
+  compactSync(options?: CompactOptions): void;
+  /**
   * Returns the list of column families in the RocksDB database.
   */
   get columns(): string[];
@@ -1034,6 +1191,17 @@ declare class RocksDatabase extends DBI<DBITransactional> {
   */
   getDBProperty(propertyName: string): string | undefined;
   /**
+  * Retrieves the estimated number of keys in the database.
+  *
+  * @example
+  * ```typescript
+  * const db = RocksDatabase.open('/path/to/database');
+  * const estimated = db.getEstimatedKeyCount();
+  * console.log(estimated);
+  * ```
+  */
+  getEstimatedKeyCount(): number;
+  /**
   * Returns the current timestamp as a monotonically increasing timestamp in
   * milliseconds represented as a decimal number.
   *

package/dist/index.d.mts

@@ -36,16 +36,21 @@ interface DBIteratorValue<T> {
   value: T;
 }
 /**
-* Wraps an iterator, namely the `NativeIterator` class, and decodes the key
-* and value.
+* Wraps the `NativeIterator` C++ binding, decoding keys and values from the
+* shared key/value buffers (fast path) or from per-iteration buffers (slow
+* fallback path used for oversized data or stable-buffer decoders).
+*
+* The native `next()` returns a primitive signal (and writes lengths to the
+* shared `ITERATOR_STATE` buffer) instead of constructing a JS result object,
+* so this class is responsible for building the `IteratorResult`.
 */
 declare class DBIterator<T> implements Iterator<DBIteratorValue<T>> {
   #private;
-  iterator: Iterator<DBIteratorValue<T>>;
+  iterator: InstanceType<typeof NativeIteratorCls>;
   store: Store;
-  constructor(iterator: Iterator<DBIteratorValue<T>>, store: Store, options?: IteratorOptions & T);
+  constructor(iterator: InstanceType<typeof NativeIteratorCls>, store: Store, includeValues: boolean);
   [Symbol.iterator](): Iterator<DBIteratorValue<T>>;
-  next(...[_value]: [] | [any]): IteratorResult<DBIteratorValue<T>>;
+  next(): IteratorResult<DBIteratorValue<T>>;
   return(value?: any): IteratorResult<DBIteratorValue<T>, any>;
   throw(err: unknown): IteratorResult<DBIteratorValue<T>, any>;
 }
@@ -57,6 +62,10 @@ type StoreIteratorOptions = IteratorOptions & DBITransactional;
 type StorePutOptions = PutOptions & DBITransactional;
 type StoreRangeOptions = RangeOptions & DBITransactional;
 type StoreRemoveOptions = DBITransactional | unknown;
+type CompactOptions = {
+  start?: Key;
+  end?: Key;
+};
 /**
 * Options for the `Store` class.
 */
@@ -163,6 +172,24 @@ declare class Store {
   */
   maxKeySize: number;
   /**
+  * The maximum number of memtables that can be queued per column family
+  * before writes stall. Higher values absorb write bursts while flushes catch
+  * up, at the cost of memory.
+  */
+  maxWriteBufferNumber?: number;
+  /**
+  * The bytes of recent memtable history to retain in memory for transaction
+  * conflict checking. `-1` derives the value from
+  * `maxWriteBufferNumber * writeBufferSize`.
+  */
+  maxWriteBufferSizeToMaintain?: number;
+  /**
+  * The total memtable budget in bytes across all column families. When the
+  * sum of memtables reaches this size, RocksDB flushes the largest one. `0`
+  * disables the global trigger so per-CF `writeBufferSize` drives flushing.
+  */
+  dbWriteBufferSize?: number;
+  /**
   * The name of the store (e.g. the column family). Defaults to `'default'`.
   */
   name: string;
@@ -172,9 +199,10 @@ declare class Store {
   noBlockCache?: boolean;
   /**
   * The number of threads to use for parallel operations. This is a RocksDB
-  * option.
+  * option. When undefined, the native layer picks
+  * `max(1, hardware_concurrency() / 2)`.
   */
-  parallelismThreads: number;
+  parallelismThreads?: number;
   /**
   * The path to the database.
   */
@@ -230,6 +258,11 @@ declare class Store {
   */
   transactionLogsPath?: string;
   /**
+  * The per-column-family memtable size in bytes at which the memtable is
+  * sealed and flushed.
+  */
+  writeBufferSize?: number;
+  /**
   * The function used to encode keys using the shared `keyBuffer`.
   */
   writeKey: WriteKeyFunction;
@@ -245,6 +278,28 @@ declare class Store {
   */
   close(): void;
   /**
+  * Compacts the entire key range of the database asynchronously.
+  * This triggers manual compaction which removes tombstones and reclaims space.
+  *
+  * @example
+  * ```typescript
+  * const db = RocksDatabase.open('/path/to/database');
+  * await db.compact();
+  * ```
+  */
+  compact(options?: CompactOptions): Promise<void>;
+  /**
+  * Compacts the entire key range of the database synchronously.
+  * This triggers manual compaction which removes tombstones and reclaims space.
+  *
+  * @example
+  * ```typescript
+  * const db = RocksDatabase.open('/path/to/database');
+  * db.compactSync();
+  * ```
+  */
+  compactSync(options?: CompactOptions): void;
+  /**
   * Decodes a key from the database.
   *
   * @param key - The key to decode.
@@ -429,10 +484,43 @@ type TransactionLog = {
   _lastCommittedPosition: Float64Array;
   _logBuffers: Map<number, WeakRef<LogBuffer>>;
 };
+/**
+* Shape of options that can be passed to the native iterator constructor for
+* the rare case of advanced RocksDB ReadOptions overrides. Common iterator
+* options are passed via the bitmask `flags` argument instead.
+*/
+type NativeIteratorAdvancedOptions = {
+  adaptiveReadahead?: boolean;
+  asyncIO?: boolean;
+  autoReadaheadSize?: boolean;
+  backgroundPurgeOnIteratorCleanup?: boolean;
+  fillCache?: boolean;
+  readaheadSize?: number;
+  tailing?: boolean;
+};
+/**
+* The result of a single native iterator step. A number value matches one of
+* the `ITERATOR_RESULT_*` constants. The slow-path object is returned when
+* the data does not fit in the shared key/value buffers, or when the decoder
+* needs a stable value buffer.
+*/
+type NativeIteratorResult = number | {
+  key: Buffer;
+  value?: Buffer;
+};
+declare class NativeIteratorCls {
+  constructor(context: StoreContext, flags: number, startKeyEnd: number, endKeyStart: number, endKeyEnd: number, options?: NativeIteratorAdvancedOptions);
+  next(): NativeIteratorResult;
+  return(): void;
+  throw(err?: unknown): void;
+}
 type NativeDatabaseMode = "optimistic" | "pessimistic";
 type NativeDatabaseOptions = {
+  dbWriteBufferSize?: number;
   disableWAL?: boolean;
   enableStats?: boolean;
+  maxWriteBufferNumber?: number;
+  maxWriteBufferSizeToMaintain?: number;
   mode?: NativeDatabaseMode;
   name?: string;
   noBlockCache?: boolean;
@@ -443,6 +531,7 @@ type NativeDatabaseOptions = {
   transactionLogMaxSize?: number;
   transactionLogRetentionMs?: number;
   transactionLogsPath?: string;
+  writeBufferSize?: number;
 };
 type ResolveCallback<T> = (value: T) => void;
 type RejectCallback = (err: Error) => void;
@@ -469,6 +558,8 @@ type NativeDatabase = {
   clear(resolve: ResolveCallback<void>, reject: RejectCallback): void;
   clearSync(): void;
   close(): void;
+  compact(resolve: ResolveCallback<void>, reject: RejectCallback, start?: Key, end?: Key): void;
+  compactSync(start?: Key, end?: Key): void;
   columns: string[];
   destroy(): void;
   drop(resolve: ResolveCallback<void>, reject: RejectCallback): void;
@@ -497,6 +588,7 @@ type NativeDatabase = {
   removeSync(key: BufferWithDataView, txnId?: number): void;
   setDefaultKeyBuffer(buffer: Buffer | Uint8Array | null): void;
   setDefaultValueBuffer(buffer: Buffer | Uint8Array | null): void;
+  setIteratorState(buffer: Buffer | Uint8Array): void;
   tryLock(key: BufferWithDataView, callback?: () => void): boolean;
   unlock(key: BufferWithDataView): void;
   useLog(name: string): TransactionLog;
@@ -504,6 +596,41 @@ type NativeDatabase = {
 };
 type RocksDatabaseConfig = {
   blockCacheSize?: number;
+  compactOnClose?: boolean;
+  /**
+  * Total memtable memory limit (bytes) shared across every database opened
+  * in this process. When set, RocksDB uses a single `WriteBufferManager` so
+  * write buffers are bounded process-wide rather than per database. 0 (the
+  * default) disables the manager.
+  *
+  * Can be updated at runtime; the new size takes effect on the existing
+  * manager via `SetBufferSize`.
+  */
+  writeBufferManagerSize?: number;
+  /**
+  * When `true`, memtable memory is "charged" against the shared block cache
+  * so the block cache and write buffers draw from a single pool. During
+  * write bursts the cache shrinks to make room for memtables; once
+  * memtables flush, the cache can grow back into the reclaimed space.
+  *
+  * Has no effect when the block cache is disabled (size 0) or
+  * `writeBufferManagerSize` is 0. Must be set on the same `config()` call
+  * that first enables the manager — changing it after the manager has been
+  * created has no effect on the running instance.
+  *
+  * @default false
+  */
+  writeBufferManagerCostToCache?: boolean;
+  /**
+  * When `true`, writes are stalled once the manager's `buffer_size` is
+  * exceeded, providing a hard cap on memtable memory. When `false`,
+  * memtables are allowed to grow past the limit and flushes are simply
+  * scheduled more aggressively. Off by default to favor write throughput
+  * over hard memory bounding.
+  *
+  * @default false
+  */
+  writeBufferManagerAllowStall?: boolean;
 };
 type RegistryStatusDB = {
   path: string;
@@ -523,6 +650,14 @@ declare const constants: {
   TRANSACTION_LOG_TOKEN: number;
   TRANSACTION_LOG_ENTRY_HEADER_SIZE: number;
   TRANSACTION_LOG_FILE_HEADER_SIZE: number;
+  ITERATOR_REVERSE_FLAG: number;
+  ITERATOR_INCLUSIVE_END_FLAG: number;
+  ITERATOR_EXCLUSIVE_START_FLAG: number;
+  ITERATOR_INCLUDE_VALUES_FLAG: number;
+  ITERATOR_NEEDS_STABLE_VALUE_BUFFER_FLAG: number;
+  ITERATOR_CONTEXT_IS_TRANSACTION_FLAG: number;
+  ITERATOR_RESULT_DONE: number;
+  ITERATOR_RESULT_FAST: number;
 };
 declare const NativeDatabase: NativeDatabase;
 declare const NativeTransaction: NativeTransaction;
@@ -975,6 +1110,28 @@ declare class RocksDatabase extends DBI<DBITransactional> {
   */
   close(): void;
   /**
+  * Compacts the entire key range of the database asynchronously.
+  * This triggers manual compaction which removes tombstones and reclaims space.
+  *
+  * @example
+  * ```typescript
+  * const db = RocksDatabase.open('/path/to/database');
+  * await db.compact();
+  * ```
+  */
+  compact(options?: CompactOptions): Promise<void>;
+  /**
+  * Compacts the entire key range of the database synchronously.
+  * This triggers manual compaction which removes tombstones and reclaims space.
+  *
+  * @example
+  * ```typescript
+  * const db = RocksDatabase.open('/path/to/database');
+  * db.compactSync();
+  * ```
+  */
+  compactSync(options?: CompactOptions): void;
+  /**
   * Returns the list of column families in the RocksDB database.
   */
   get columns(): string[];
@@ -1034,6 +1191,17 @@ declare class RocksDatabase extends DBI<DBITransactional> {
   */
   getDBProperty(propertyName: string): string | undefined;
   /**
+  * Retrieves the estimated number of keys in the database.
+  *
+  * @example
+  * ```typescript
+  * const db = RocksDatabase.open('/path/to/database');
+  * const estimated = db.getEstimatedKeyCount();
+  * console.log(estimated);
+  * ```
+  */
+  getEstimatedKeyCount(): number;
+  /**
   * Returns the current timestamp as a monotonically increasing timestamp in
   * milliseconds represented as a decimal number.
   *