@harperfast/rocksdb-js 2.10.0 → 2.12.0

package/dist/index.d.cts

@@ -178,6 +178,14 @@ export declare const backups: {
    * Verifies a backup's file sizes, and optionally their checksums (which
    * requires reading all backed-up data).
    *
+   * Set `verifyTables: true` to also prove this build can read the backup
+   * before restoring it: every table file is opened, one at a time, and its
+   * footer (including `format_version`) and properties must be readable, and
+   * every compression algorithm it records must be supported by this build.
+   * A checksum-valid table written by a newer or differently built RocksDB
+   * otherwise passes and fails on the first read after a restore. Blob files
+   * are not checked.
+   *
    * When the backup was created with `transactionLogs: true`, the backup's
    * transaction log snapshot (`<backupDir>/transaction_logs/<backupId>/`) is
    * also validated — every log file's header and entry framing must be intact
@@ -186,6 +194,7 @@ export declare const backups: {
    */
   verify(backupDir: string, backupId: number, options?: {
     verifyWithChecksum?: boolean;
+    verifyTables?: boolean;
     verifyTransactionLogs?: boolean;
   }): Promise<void>;
 };
@@ -648,6 +657,7 @@ export declare class Store {
   /**
    * The target maximum size of a transaction log before rotation. Transactions
    * are never split across files, so one transaction may exceed this target.
+   * Defaults to 16 MiB. `0` (formerly unlimited) is rejected at open.
    */
   transactionLogMaxSize?: number;
   /**
@@ -661,6 +671,11 @@ export declare class Store {
    * The path to the transaction logs directory.
    */
   transactionLogsPath?: string;
+  /**
+   * The transaction log whose batch keys this process originates; seeds the
+   * monotonic timestamp floor at open. See `StoreOptions.timestampFloorLog`.
+   */
+  timestampFloorLog?: string;
   /**
    * Whether this store's column family participates in the VerificationTable.
    */
@@ -1378,6 +1393,7 @@ type StatsBasics = {
   "txnlog.replayGapBytes": number;
   "commitPipeline.logQueueDepth": number;
   "commitPipeline.commitQueueDepth": number;
+  "commitPipeline.commitThreads": number;
   "columnFamily.pendingReclaims": number;
   "writeBufferManager.bufferSize": number;
   "writeBufferManager.memoryUsage": number;
@@ -1952,7 +1968,8 @@ type NativeTransaction = {
 type LogBuffer = Buffer & {
   dataView: DataView;
   logId: number;
-  size: number;
+  readonly readableExtent: number;
+  size?: number;
 };
 type TransactionLogQueryOptions = {
   start?: number;
@@ -2008,6 +2025,7 @@ type TransactionLogStats = {
     oldestFileAgeMs: number;
     purgeableFiles: number;
     retainedUnflushedFiles: number;
+    retentionPins: number;
     lastPurgeMs: number;
   };
   totals: {
@@ -2035,12 +2053,13 @@ export type TransactionLog = {
   name: string;
   path: string;
   query(options?: TransactionLogQueryOptions): IterableIterator<TransactionEntry>;
-  _currentLogBuffer: LogBuffer;
+  _currentLogBuffer: WeakRef<LogBuffer> | undefined;
   _findPosition(timestamp: number): number;
   _getLastCommittedPosition(): Buffer;
   _getLastFlushed(): number;
   _getMemoryMapOfFile(sequenceId: number): LogBuffer | undefined;
   _lastCommittedPosition: Float64Array;
+  _nextLogId(sequenceId: number): number;
   _logBuffers: Map<number, WeakRef<LogBuffer>>;
 };
 /**
@@ -2130,9 +2149,31 @@ type NativeDatabaseOptions = {
   secondaryPath?: string;
   statsLevel?: (typeof stats.StatsLevel)[keyof typeof stats.StatsLevel];
   transactionLogMaxAgeThreshold?: number;
+  /** Target segment size in bytes; defaults to 16 MiB. `0` is rejected. */
   transactionLogMaxSize?: number;
   transactionLogRetentionMs?: number;
   transactionLogsPath?: string;
+  /**
+   * The name of the transaction log whose batch keys this process originates
+   * (the log it passes to `useLog()` for its own writes).
+   *
+   * At open, the process-wide monotonic timestamp floor is raised above every
+   * batch key still durable in that log, so a backward wall-clock step between
+   * runs cannot reissue a transaction timestamp that is already a key in it.
+   *
+   * Name only a log this process originates. A log written under timestamps
+   * adopted from another node (a replication receiver calling
+   * `transaction.setTimestamp()`) is keyed by that node's clock, and seeding
+   * from it would ratchet this process's clock to the fastest of those nodes
+   * on every restart. Unset (the default), the floor is left alone; native
+   * code cannot tell the two kinds of log apart.
+   *
+   * Safety check: an incomplete scan or a key implausibly far ahead of the
+   * wall clock rejects this open rather than risk issuing a duplicate key.
+   * The floor is process-wide, so it is shared by every database open in the
+   * process.
+   */
+  timestampFloorLog?: string;
   /**
    * When true, transaction writes to this column family invalidate the
    * VerificationTable slot for each written key at write time (not at
@@ -2191,9 +2232,9 @@ type FlushOptions = {
    * the descriptor is process-global and shared across `worker_threads`, so the stall lands on
    * every other column family and every other handle that opened the same path — not just the
    * one you called. It also relocates the hang rather than removing it: a stalled `db->Write()`
-   * blocks the database's single `CommitWorker` thread, which dispatches every
-   * `Transaction.commit()` in order, so the stall queues up every commit behind it — including
-   * ones from callers that never touched flush — until the stall clears.
+   * blocks each of the database's `CommitWorker` threads that reaches it, so the stall queues up
+   * every async `Transaction.commit()` behind them — including ones from callers that never
+   * touched flush — until the stall clears.
    *
    * Note this is a *different* knob from the `writeBufferManagerAllowStall` config, and their
    * polarity is nearly opposite: that one decides whether the WriteBufferManager may stall
@@ -2271,6 +2312,7 @@ type NativeDatabase = {
   removeListener(event: string | BufferWithDataView, callback: () => void): boolean;
   removeSync(key: BufferWithDataView, txnId?: number): void;
   resume(): void;
+  setCompression(algorithm: string, level?: number): void;
   setDefaultKeyBuffer(buffer: Buffer | Uint8Array | null): void;
   setDefaultValueBuffer(buffer: Buffer | Uint8Array | null): void;
   setIteratorState(buffer: Buffer | Uint8Array): void;
@@ -2282,6 +2324,38 @@ type NativeDatabase = {
 };
 type RocksDatabaseConfig = {
   blockCacheSize?: number;
+  /**
+   * Number of commit lock buckets each writable optimistic database allocates
+   * when opened with parallel validation. Defaults to 65,536 (about 2.5 MiB per
+   * database on Linux x64; RocksDB's own default is 2^20); accepts integers
+   * from 16 through 16,777,216. Applies to databases opened after the call; open
+   * databases keep their count. Smaller pools only cost throughput when several
+   * commits to the same database overlap (`commitThreads`, `commitSync()`
+   * callers) and their keys collide, which large transactions make likely:
+   * raise the count for databases that commit hundreds of keys at a time. This
+   * does not limit transaction size.
+   */
+  occLockBuckets?: number;
+  /**
+   * How writable optimistic databases opened after the call validate commits.
+   * `'parallel'` (default, RocksDB's default) uses the `occLockBuckets` pool and
+   * lets concurrent commits share a RocksDB write. `'serial'` validates inside
+   * the write group and allocates no bucket pool, but RocksDB never batches its
+   * commits with other writers, so a database's concurrent commits (its
+   * `commitThreads` and any `commitSync()` callers) lose most of their
+   * throughput. Open databases keep their policy.
+   */
+  occValidation?: "parallel" | "serial";
+  /**
+   * Maximum number of dedicated threads each database opened after the call uses
+   * to commit async transactions. Threads start only when commits overlap, so a
+   * database committed to one transaction at a time owns one. Defaults to
+   * `min(4, hardware concurrency)`; accepts integers from 1 through 64. With 1,
+   * async commits to a database run one at a time in dispatch order; with more,
+   * concurrent commits finish (and their promises resolve) in any order, as
+   * concurrent `commitSync()` calls always have. Open databases keep their limit.
+   */
+  commitThreads?: number;
   /**
    * Number of slots in the process-global verification table. Each slot is
    * 8 bytes; the default of 128K slots is 1 MB. Set to 0 to disable.
@@ -2496,6 +2570,16 @@ export declare const supportedCompression: readonly string[];
 declare const NativeDatabase: NativeDatabase;
 declare const NativeTransaction: NativeTransaction;
 export declare const TransactionLog: TransactionLog;
+/**
+ * Returns the status of every database currently held by the native registry.
+ * Walks every registered descriptor, so it samples live state rather than
+ * reading a stored property.
+ */
+export declare const getRegistryStatus: () => RegistryStatus;
+/**
+ * @deprecated Use {@link getRegistryStatus} instead. This alias will be removed
+ * in v3.0.0.
+ */
 export declare const registryStatus: () => RegistryStatus;
 /**
  * Flushes every open database and waits for outstanding compactions, then
@@ -2844,6 +2928,51 @@ export declare class RocksDatabase extends DBI<DBITransactional> {
    * ```
    */
   get compression(): CompressionInfo;
+  /**
+   * Dynamically changes the compression algorithm (and optional level) for
+   * this database's column family on an already-open database — no close, no
+   * reopen, and it does not conflict with another handle that already has
+   * this column family open (unlike an explicit open-time `compression`
+   * mismatch). That other handle's live options change too, silently,
+   * without notice; a later explicit reopen of it is compared against the
+   * new value, so a stale request it still holds can then conflict. Backed
+   * by RocksDB's `DB::SetOptions()`, which documents both `compression` and
+   * `blob_compression_type` as mutable column family options.
+   *
+   * This governs only *newly written* files (the next flush and any future
+   * compaction output) going forward; SST and blob files already on disk
+   * keep their existing compression until they are rewritten by a later
+   * compaction. `compact()`'s default options can skip already-bottommost
+   * files; use `compact({ bottommost: true })` to rewrite them under the new
+   * codec. This is the live-mutation counterpart to the open-time
+   * `compression` option — see the README's Compression section for when to
+   * use each.
+   *
+   * @example
+   * ```typescript
+   * const db = RocksDatabase.open('/path/to/db', { compression: 'none' });
+   * db.setCompression({ algorithm: 'zstd', level: 19 });
+   * // new writes are now compressed with zstd; existing SSTs are untouched
+   * // until compacted.
+   * ```
+   */
+  setCompression(compression: CompressionOption): void;
+  /**
+   * Resyncs the store's cached open-time compression option from the live
+   * database state after `setCompression()`. Also clears
+   * `compressionForAllColumnFamilies`: that flag means "apply this option to
+   * every column family opened for this database", which no longer describes
+   * a live change scoped to a single column family — leaving it set would
+   * restamp every other family with this algorithm on the next
+   * same-instance `close()` + `open()`.
+   *
+   * The flag clear is unconditional (a plain assignment, so it cannot itself
+   * fail) and comes first: a concurrent close racing the live-state read
+   * below throws "Database not open" from the getter, and that read failing
+   * must not also skip the flag clear -- otherwise the same restamp this
+   * method exists to prevent survives on the next open().
+   */
+  private syncStoreCompressionFromLive;
   /**
    * The informational-log settings currently in effect for this database,
    * read live from RocksDB, as `{ maxLogFileSize, infoLogLevel }`. These are

package/dist/index.d.mts

@@ -178,6 +178,14 @@ export declare const backups: {
    * Verifies a backup's file sizes, and optionally their checksums (which
    * requires reading all backed-up data).
    *
+   * Set `verifyTables: true` to also prove this build can read the backup
+   * before restoring it: every table file is opened, one at a time, and its
+   * footer (including `format_version`) and properties must be readable, and
+   * every compression algorithm it records must be supported by this build.
+   * A checksum-valid table written by a newer or differently built RocksDB
+   * otherwise passes and fails on the first read after a restore. Blob files
+   * are not checked.
+   *
    * When the backup was created with `transactionLogs: true`, the backup's
    * transaction log snapshot (`<backupDir>/transaction_logs/<backupId>/`) is
    * also validated — every log file's header and entry framing must be intact
@@ -186,6 +194,7 @@ export declare const backups: {
    */
   verify(backupDir: string, backupId: number, options?: {
     verifyWithChecksum?: boolean;
+    verifyTables?: boolean;
     verifyTransactionLogs?: boolean;
   }): Promise<void>;
 };
@@ -648,6 +657,7 @@ export declare class Store {
   /**
    * The target maximum size of a transaction log before rotation. Transactions
    * are never split across files, so one transaction may exceed this target.
+   * Defaults to 16 MiB. `0` (formerly unlimited) is rejected at open.
    */
   transactionLogMaxSize?: number;
   /**
@@ -661,6 +671,11 @@ export declare class Store {
    * The path to the transaction logs directory.
    */
   transactionLogsPath?: string;
+  /**
+   * The transaction log whose batch keys this process originates; seeds the
+   * monotonic timestamp floor at open. See `StoreOptions.timestampFloorLog`.
+   */
+  timestampFloorLog?: string;
   /**
    * Whether this store's column family participates in the VerificationTable.
    */
@@ -1378,6 +1393,7 @@ type StatsBasics = {
   "txnlog.replayGapBytes": number;
   "commitPipeline.logQueueDepth": number;
   "commitPipeline.commitQueueDepth": number;
+  "commitPipeline.commitThreads": number;
   "columnFamily.pendingReclaims": number;
   "writeBufferManager.bufferSize": number;
   "writeBufferManager.memoryUsage": number;
@@ -1952,7 +1968,8 @@ type NativeTransaction = {
 type LogBuffer = Buffer & {
   dataView: DataView;
   logId: number;
-  size: number;
+  readonly readableExtent: number;
+  size?: number;
 };
 type TransactionLogQueryOptions = {
   start?: number;
@@ -2008,6 +2025,7 @@ type TransactionLogStats = {
     oldestFileAgeMs: number;
     purgeableFiles: number;
     retainedUnflushedFiles: number;
+    retentionPins: number;
     lastPurgeMs: number;
   };
   totals: {
@@ -2035,12 +2053,13 @@ export type TransactionLog = {
   name: string;
   path: string;
   query(options?: TransactionLogQueryOptions): IterableIterator<TransactionEntry>;
-  _currentLogBuffer: LogBuffer;
+  _currentLogBuffer: WeakRef<LogBuffer> | undefined;
   _findPosition(timestamp: number): number;
   _getLastCommittedPosition(): Buffer;
   _getLastFlushed(): number;
   _getMemoryMapOfFile(sequenceId: number): LogBuffer | undefined;
   _lastCommittedPosition: Float64Array;
+  _nextLogId(sequenceId: number): number;
   _logBuffers: Map<number, WeakRef<LogBuffer>>;
 };
 /**
@@ -2130,9 +2149,31 @@ type NativeDatabaseOptions = {
   secondaryPath?: string;
   statsLevel?: (typeof stats.StatsLevel)[keyof typeof stats.StatsLevel];
   transactionLogMaxAgeThreshold?: number;
+  /** Target segment size in bytes; defaults to 16 MiB. `0` is rejected. */
   transactionLogMaxSize?: number;
   transactionLogRetentionMs?: number;
   transactionLogsPath?: string;
+  /**
+   * The name of the transaction log whose batch keys this process originates
+   * (the log it passes to `useLog()` for its own writes).
+   *
+   * At open, the process-wide monotonic timestamp floor is raised above every
+   * batch key still durable in that log, so a backward wall-clock step between
+   * runs cannot reissue a transaction timestamp that is already a key in it.
+   *
+   * Name only a log this process originates. A log written under timestamps
+   * adopted from another node (a replication receiver calling
+   * `transaction.setTimestamp()`) is keyed by that node's clock, and seeding
+   * from it would ratchet this process's clock to the fastest of those nodes
+   * on every restart. Unset (the default), the floor is left alone; native
+   * code cannot tell the two kinds of log apart.
+   *
+   * Safety check: an incomplete scan or a key implausibly far ahead of the
+   * wall clock rejects this open rather than risk issuing a duplicate key.
+   * The floor is process-wide, so it is shared by every database open in the
+   * process.
+   */
+  timestampFloorLog?: string;
   /**
    * When true, transaction writes to this column family invalidate the
    * VerificationTable slot for each written key at write time (not at
@@ -2191,9 +2232,9 @@ type FlushOptions = {
    * the descriptor is process-global and shared across `worker_threads`, so the stall lands on
    * every other column family and every other handle that opened the same path — not just the
    * one you called. It also relocates the hang rather than removing it: a stalled `db->Write()`
-   * blocks the database's single `CommitWorker` thread, which dispatches every
-   * `Transaction.commit()` in order, so the stall queues up every commit behind it — including
-   * ones from callers that never touched flush — until the stall clears.
+   * blocks each of the database's `CommitWorker` threads that reaches it, so the stall queues up
+   * every async `Transaction.commit()` behind them — including ones from callers that never
+   * touched flush — until the stall clears.
    *
    * Note this is a *different* knob from the `writeBufferManagerAllowStall` config, and their
    * polarity is nearly opposite: that one decides whether the WriteBufferManager may stall
@@ -2271,6 +2312,7 @@ type NativeDatabase = {
   removeListener(event: string | BufferWithDataView, callback: () => void): boolean;
   removeSync(key: BufferWithDataView, txnId?: number): void;
   resume(): void;
+  setCompression(algorithm: string, level?: number): void;
   setDefaultKeyBuffer(buffer: Buffer | Uint8Array | null): void;
   setDefaultValueBuffer(buffer: Buffer | Uint8Array | null): void;
   setIteratorState(buffer: Buffer | Uint8Array): void;
@@ -2282,6 +2324,38 @@ type NativeDatabase = {
 };
 type RocksDatabaseConfig = {
   blockCacheSize?: number;
+  /**
+   * Number of commit lock buckets each writable optimistic database allocates
+   * when opened with parallel validation. Defaults to 65,536 (about 2.5 MiB per
+   * database on Linux x64; RocksDB's own default is 2^20); accepts integers
+   * from 16 through 16,777,216. Applies to databases opened after the call; open
+   * databases keep their count. Smaller pools only cost throughput when several
+   * commits to the same database overlap (`commitThreads`, `commitSync()`
+   * callers) and their keys collide, which large transactions make likely:
+   * raise the count for databases that commit hundreds of keys at a time. This
+   * does not limit transaction size.
+   */
+  occLockBuckets?: number;
+  /**
+   * How writable optimistic databases opened after the call validate commits.
+   * `'parallel'` (default, RocksDB's default) uses the `occLockBuckets` pool and
+   * lets concurrent commits share a RocksDB write. `'serial'` validates inside
+   * the write group and allocates no bucket pool, but RocksDB never batches its
+   * commits with other writers, so a database's concurrent commits (its
+   * `commitThreads` and any `commitSync()` callers) lose most of their
+   * throughput. Open databases keep their policy.
+   */
+  occValidation?: "parallel" | "serial";
+  /**
+   * Maximum number of dedicated threads each database opened after the call uses
+   * to commit async transactions. Threads start only when commits overlap, so a
+   * database committed to one transaction at a time owns one. Defaults to
+   * `min(4, hardware concurrency)`; accepts integers from 1 through 64. With 1,
+   * async commits to a database run one at a time in dispatch order; with more,
+   * concurrent commits finish (and their promises resolve) in any order, as
+   * concurrent `commitSync()` calls always have. Open databases keep their limit.
+   */
+  commitThreads?: number;
   /**
    * Number of slots in the process-global verification table. Each slot is
    * 8 bytes; the default of 128K slots is 1 MB. Set to 0 to disable.
@@ -2496,6 +2570,16 @@ export declare const supportedCompression: readonly string[];
 declare const NativeDatabase: NativeDatabase;
 declare const NativeTransaction: NativeTransaction;
 export declare const TransactionLog: TransactionLog;
+/**
+ * Returns the status of every database currently held by the native registry.
+ * Walks every registered descriptor, so it samples live state rather than
+ * reading a stored property.
+ */
+export declare const getRegistryStatus: () => RegistryStatus;
+/**
+ * @deprecated Use {@link getRegistryStatus} instead. This alias will be removed
+ * in v3.0.0.
+ */
 export declare const registryStatus: () => RegistryStatus;
 /**
  * Flushes every open database and waits for outstanding compactions, then
@@ -2844,6 +2928,51 @@ export declare class RocksDatabase extends DBI<DBITransactional> {
    * ```
    */
   get compression(): CompressionInfo;
+  /**
+   * Dynamically changes the compression algorithm (and optional level) for
+   * this database's column family on an already-open database — no close, no
+   * reopen, and it does not conflict with another handle that already has
+   * this column family open (unlike an explicit open-time `compression`
+   * mismatch). That other handle's live options change too, silently,
+   * without notice; a later explicit reopen of it is compared against the
+   * new value, so a stale request it still holds can then conflict. Backed
+   * by RocksDB's `DB::SetOptions()`, which documents both `compression` and
+   * `blob_compression_type` as mutable column family options.
+   *
+   * This governs only *newly written* files (the next flush and any future
+   * compaction output) going forward; SST and blob files already on disk
+   * keep their existing compression until they are rewritten by a later
+   * compaction. `compact()`'s default options can skip already-bottommost
+   * files; use `compact({ bottommost: true })` to rewrite them under the new
+   * codec. This is the live-mutation counterpart to the open-time
+   * `compression` option — see the README's Compression section for when to
+   * use each.
+   *
+   * @example
+   * ```typescript
+   * const db = RocksDatabase.open('/path/to/db', { compression: 'none' });
+   * db.setCompression({ algorithm: 'zstd', level: 19 });
+   * // new writes are now compressed with zstd; existing SSTs are untouched
+   * // until compacted.
+   * ```
+   */
+  setCompression(compression: CompressionOption): void;
+  /**
+   * Resyncs the store's cached open-time compression option from the live
+   * database state after `setCompression()`. Also clears
+   * `compressionForAllColumnFamilies`: that flag means "apply this option to
+   * every column family opened for this database", which no longer describes
+   * a live change scoped to a single column family — leaving it set would
+   * restamp every other family with this algorithm on the next
+   * same-instance `close()` + `open()`.
+   *
+   * The flag clear is unconditional (a plain assignment, so it cannot itself
+   * fail) and comes first: a concurrent close racing the live-state read
+   * below throws "Database not open" from the getter, and that read failing
+   * must not also skip the flag clear -- otherwise the same restamp this
+   * method exists to prevent survives on the next open().
+   */
+  private syncStoreCompressionFromLive;
   /**
    * The informational-log settings currently in effect for this database,
    * read live from RocksDB, as `{ maxLogFileSize, infoLogLevel }`. These are