@harperfast/rocksdb-js 2.11.0 → 2.13.0

package/dist/index.d.mts

@@ -1,5 +1,4 @@
 import { ExtendedIterable } from "@harperfast/extended-iterable";
-//#region src/backup.d.ts
 /**
  * Options for creating a backup via `db.backup()`.
  *
@@ -178,6 +177,14 @@ export declare const backups: {
    * Verifies a backup's file sizes, and optionally their checksums (which
    * requires reading all backed-up data).
    *
+   * Set `verifyTables: true` to also prove this build can read the backup
+   * before restoring it: every table file is opened, one at a time, and its
+   * footer (including `format_version`) and properties must be readable, and
+   * every compression algorithm it records must be supported by this build.
+   * A checksum-valid table written by a newer or differently built RocksDB
+   * otherwise passes and fails on the first read after a restore. Blob files
+   * are not checked.
+   *
    * When the backup was created with `transactionLogs: true`, the backup's
    * transaction log snapshot (`<backupDir>/transaction_logs/<backupId>/`) is
    * also validated — every log file's header and entry framing must be intact
@@ -186,12 +193,11 @@ export declare const backups: {
    */
   verify(backupDir: string, backupId: number, options?: {
     verifyWithChecksum?: boolean;
+    verifyTables?: boolean;
     verifyTransactionLogs?: boolean;
   }): Promise<void>;
 };
-//#endregion
-//#region src/encoding.d.ts
-type Key = Key[] | string | symbol | number | boolean | Uint8Array | Buffer | null;
+type Key = Key[] | string | symbol | number | bigint | boolean | Uint8Array | Buffer | null;
 interface BufferWithDataView extends Buffer {
   dataView: DataView;
   start: number;
@@ -219,8 +225,6 @@ type Encoding = "binary" | "ordered-binary" | "msgpack" | false;
 type KeyEncoding = "binary" | "ordered-binary" | "uint32";
 type ReadKeyFunction<T> = (source: BufferWithDataView, start: number, end?: number) => T;
 type WriteKeyFunction = (key: Key, target: BufferWithDataView, start: number) => number;
-//#endregion
-//#region src/dbi-iterator.d.ts
 interface DBIteratorValue<T> {
   key: Key;
   value: T;
@@ -238,14 +242,12 @@ export declare class DBIterator<T> implements Iterator<DBIteratorValue<T>> {
   #private;
   iterator: InstanceType<typeof NativeIteratorCls>;
   store: Store;
-  constructor(iterator: InstanceType<typeof NativeIteratorCls>, store: Store, includeValues: boolean, limit?: number);
+  constructor(iterator: InstanceType<typeof NativeIteratorCls>, store: Store, includeValues: boolean, limit?: number, transaction?: NativeTransaction);
   [Symbol.iterator](): Iterator<DBIteratorValue<T>>;
   next(): IteratorResult<DBIteratorValue<T>>;
   return(value?: any): IteratorResult<DBIteratorValue<T>, any>;
   throw(err: unknown): IteratorResult<DBIteratorValue<T>, any>;
 }
-//#endregion
-//#region src/store.d.ts
 type StoreContext = NativeDatabase | NativeTransaction;
 type StoreGetOptions = GetOptions & DBITransactional;
 type StoreIteratorOptions = IteratorOptions & DBITransactional;
@@ -676,6 +678,10 @@ export declare class Store {
    * sealed and flushed.
    */
   writeBufferSize?: number;
+  /**
+   * The memtable arena block size in bytes; `0` or unset is automatic.
+   */
+  arenaBlockSize?: number;
   /**
    * The function used to encode keys using the shared `keyBuffer`.
    */
@@ -912,8 +918,6 @@ interface PutOptions {
   noDupData?: boolean;
   noOverwrite?: boolean;
 }
-//#endregion
-//#region src/transaction.d.ts
 /**
  * Sentinel value returned by `commit()` when `coordinatedRetry: true` and the
  * transaction encountered an IsBusy conflict. The native layer parks on VT
@@ -921,6 +925,35 @@ interface PutOptions {
  * intent, so callers should retry immediately without any backoff delay.
  */
 declare const RETRY_NOW: number;
+/**
+ * `abort()` of a transaction whose transaction-log batch could not be marked aborted
+ * (`transactionLogAbortMarkers`). The batch stays unresolved, so committed readers do not pass it and
+ * the log store refuses appends, until a retry on the next append, database flush or store close
+ * marks it.
+ */
+export declare class TransactionAbortUnresolvedError extends Error {
+  readonly code = "ERR_TRANSACTION_ABORT_UNRESOLVED";
+  readonly txn: Transaction;
+  constructor(error: Error, txn: Transaction);
+}
+/**
+ * Why a transaction's most recent attempt was refused or retried, from {@link Transaction.lastConflictKind}.
+ */
+export declare const ConflictKind: {
+  readonly none: 0;
+  /** Refused on an older transaction's write intent before writing its log batch. */
+  readonly intentBeforeLog: 1;
+  /** Refused on an older transaction's write intent after its log batch was written. */
+  readonly intentAfterLog: 2;
+  /** RocksDB found a conflict at commit. */
+  readonly atCommit: 3;
+  /**
+   * RocksDB's conflict check, run before the log write, found a write that already committed, or,
+   * as at commit, could not validate because the memtable history it needs was flushed (needs a
+   * RocksDB build that exports the check).
+   */
+  readonly checkBeforeLog: 4;
+};
 /**
  * Provides transaction level operations to a transaction callback.
  */
@@ -966,6 +999,22 @@ export declare class Transaction extends DBI {
    * @returns The transaction timestamp in milliseconds.
    */
   getTimestamp(): number;
+  /**
+   * Whether this transaction's batch is in the transaction log. A commit conflict after the batch
+   * was written keeps it for the retry, which must not stage it again; an attempt refused before
+   * writing it (`refusedBeforeLog`) leaves nothing.
+   */
+  logWritten(): boolean;
+  /**
+   * Why the most recent attempt was refused or retried; one of {@link ConflictKind}.
+   */
+  lastConflictKind(): number;
+  /**
+   * Bounds coordinated-retry parks: past `deadline` (epoch milliseconds) a parked commit resolves
+   * `RETRY_NOW` at once, so a caller enforcing its own deadline can abort instead of waiting out
+   * the default park bound. 0 clears it.
+   */
+  setParkDeadline(deadline: number): void;
   /**
    * Get the transaction id.
    */
@@ -982,11 +1031,7 @@ export declare class Transaction extends DBI {
    */
   setTimestamp(timestamp?: number): void;
 }
-//#endregion
-//#region src/util.d.ts
 type MaybePromise<T> = T | Promise<T>;
-//#endregion
-//#region src/dbi.d.ts
 interface RocksDBOptions {
   /**
    * When `true`, RocksDB will do some enhancements for prefetching the data.
@@ -1324,8 +1369,6 @@ export declare class DBI<T extends DBITransactional | unknown = unknown> {
    */
   useLog(name: string | number): TransactionLog;
 }
-//#endregion
-//#region src/stats.d.ts
 /**
  * Database-level RocksDB statistics types for `db.getStats()` / `db.getStat()`.
  *
@@ -1666,6 +1709,10 @@ type StatsAllExtras = {
   "rocksdb.checkpoint.read.bytes": number;
   "rocksdb.checkpoint.write.bytes": number;
   "rocksdb.file.submit.async.read.fallback": number;
+  "rocksdb.blobdb.lazy.read.count": number;
+  "rocksdb.blobdb.lazy.read.bytes": number;
+  "rocksdb.blobdb.lazy.partial.read.count": number;
+  "rocksdb.blobdb.lazy.partial.bytes.saved": number;
   "rocksdb.compaction.times.cpu_micros": StatsHistogramData;
   "rocksdb.subcompaction.setup.times.micros": StatsHistogramData;
   "rocksdb.table.sync.micros": StatsHistogramData;
@@ -1748,8 +1795,6 @@ interface GetStatsMethod {
   getStats(all?: false): StatsDefault;
   getStats(all: true): StatsAll;
 }
-//#endregion
-//#region src/validate-transaction-log.d.ts
 /**
  * The validation result for a single transaction log file within a store, as
  * returned by {@link validateTransactionLogStore}.
@@ -1856,8 +1901,6 @@ interface ValidateTransactionLogStoreOptions {
  * ```
  */
 export declare function validateTransactionLogStore(path: string, options?: ValidateTransactionLogStoreOptions): Promise<TransactionLogStoreValidation>;
-//#endregion
-//#region src/load-binding.d.ts
 /**
  * The `Error` subclass passed to `'error'` event listeners and returned by
  * `db.getLastError()` when RocksDB reports a background error (e.g. a write
@@ -1951,9 +1994,13 @@ type NativeTransaction = {
   getCount(options?: RangeOptions): number;
   getSync(keyLengthOrKeyBuffer: number | Buffer): Buffer | number | undefined;
   getTimestamp(): number;
+  lastConflictKind(): number;
+  logWritten(): boolean;
   putSync(key: Key, value: Buffer | Uint8Array, txnId?: number): void;
   removeSync(key: Key): void;
+  setParkDeadline(deadline: number): void;
   setTimestamp(timestamp?: number): void;
+  trimIterators(): void;
   useLog(name: string | number, jsDatabase: NativeDatabase): TransactionLog;
 };
 type LogBuffer = Buffer & {
@@ -2016,6 +2063,7 @@ type TransactionLogStats = {
     oldestFileAgeMs: number;
     purgeableFiles: number;
     retainedUnflushedFiles: number;
+    retentionPins: number;
     lastPurgeMs: number;
   };
   totals: {
@@ -2028,6 +2076,16 @@ type TransactionLogStats = {
     purgeRuns: number;
     databaseFlushes: number;
     writeFailures: number;
+    /** Batches marked aborted (`transactionLogAbortMarkers`). */
+    abortsMarked: number;
+    /** Abort marker writes or syncs that failed; each leaves its batch unresolved until a retry. */
+    abortMarkFailures: number;
+    /**
+     * Microseconds logged batches held the committed-read watermark back after their first commit
+     * conflict, until they committed or ended; `pinnedTransactions` counts those batches.
+     */
+    watermarkPinnedUs: number;
+    pinnedTransactions: number;
   };
   config: {
     maxFileSize: number;
@@ -2045,11 +2103,13 @@ export type TransactionLog = {
   query(options?: TransactionLogQueryOptions): IterableIterator<TransactionEntry>;
   _currentLogBuffer: WeakRef<LogBuffer> | undefined;
   _findPosition(timestamp: number): number;
+  _findExactStart(timestamp: number, span: Float64Array): number;
   _getLastCommittedPosition(): Buffer;
   _getLastFlushed(): number;
   _getMemoryMapOfFile(sequenceId: number): LogBuffer | undefined;
   _lastCommittedPosition: Float64Array;
   _nextLogId(sequenceId: number): number;
+  _segmentRemovals(): number;
   _logBuffers: Map<number, WeakRef<LogBuffer>>;
 };
 /**
@@ -2079,11 +2139,22 @@ type NativeIteratorResult = number | {
 declare class NativeIteratorCls {
   constructor(context: StoreContext, flags: number, startKeyEnd: number, endKeyStart: number, endKeyEnd: number, options?: NativeIteratorAdvancedOptions, transactionId?: number);
   next(): NativeIteratorResult;
-  return(): void;
-  throw(err?: unknown): void;
+  /** Returns whether the native iterator went back to its transaction's pool. */
+  return(): boolean;
+  throw(err?: unknown): boolean;
 }
 type NativeDatabaseMode = "optimistic" | "pessimistic";
 type NativeDatabaseOptions = {
+  /**
+   * Memtable arena block size in bytes. Every column family that has been
+   * written holds at least one block charged to the WriteBufferManager, so
+   * with many column families this sets a floor on the manager's usage.
+   * `0` (the default) is automatic: with a WriteBufferManager attached,
+   * `writeBufferManagerSize / 2048` clamped to between 64 KiB and
+   * `min(1 MiB, writeBufferSize / 8)`; without one, RocksDB's own
+   * `min(1 MiB, writeBufferSize / 8)`.
+   */
+  arenaBlockSize?: number;
   /**
    * The friendly name of a compression algorithm compiled into this RocksDB
    * build (see `supportedCompression`), e.g. `'lz4'`, `'zstd'`, `'none'`. The
@@ -2257,6 +2328,7 @@ type NativeDatabase = {
   dropSync(): void;
   flush(resolve: ResolveCallback<void>, reject: RejectCallback, options?: FlushOptions): void;
   flushSync(options?: FlushOptions): void;
+  syncWal(resolve: ResolveCallback<void>, reject: RejectCallback): void;
   notify(event: string | BufferWithDataView, args?: any[]): boolean;
   get(keyLengthOrKeyBuffer: number | Buffer, resolve: ResolveCallback<Buffer | number>, reject: RejectCallback, txnId?: number, expectedVersion?: number): number;
   estimateCount(startKey?: Buffer, endKey?: Buffer): {
@@ -2346,6 +2418,24 @@ type RocksDatabaseConfig = {
    * concurrent `commitSync()` calls always have. Open databases keep their limit.
    */
   commitThreads?: number;
+  /**
+   * Marks a transaction-log batch aborted in place (`TRANSACTION_LOG_ENTRY_ABORTED_FLAG` on its
+   * first entry) when its transaction ends without committing after the batch was written, so
+   * readers skip it. Off by default: with it off, behavior is unchanged. Turn it on only once every
+   * reader of these logs (replicas, restores, repair tools) skips aborted batches; an older reader
+   * shows an aborted batch as live. Read when a transaction starts and when it is reset for a
+   * retry, so it can change at runtime.
+   */
+  transactionLogAbortMarkers?: boolean;
+  /**
+   * With `transactionLogAbortMarkers` on, refuses a conflicting attempt before it writes its
+   * transaction-log batch when an older transaction holds a write intent on one of its keys, and
+   * (with `coordinatedRetry`) parks it until that holder ends. This bounds how long the losers of
+   * heavily overlapping transactions wait, at the cost of more retries and less throughput when
+   * transactions are large or contention is low. Off by default; no effect with abort markers off
+   * or the verification table disabled. Read with `transactionLogAbortMarkers`.
+   */
+  transactionLogWriteIntents?: boolean;
   /**
    * Number of slots in the process-global verification table. Each slot is
    * 8 bytes; the default of 128K slots is 1 MB. Set to 0 to disable.
@@ -2394,7 +2484,27 @@ type RocksDatabaseConfig = {
    * @default false
    */
   writeBufferManagerAllowStall?: boolean;
+  /**
+   * Which memtable the WriteBufferManager flushes when it is over budget:
+   *
+   * - `'oldest'`: the oldest memtable of the database being written.
+   * - `'largest'`: the largest memtable of the database being written. Because
+   *   this library opens databases with atomic flush, this currently behaves
+   *   like `'oldest'`.
+   * - `'largestAcrossDBs'`: a background thread flushes whichever database
+   *   holds the most memtable memory, whether or not it is being written.
+   *
+   * When unset, `'largestAcrossDBs'` while `writeBufferManagerAllowStall` is
+   * on (and following later changes to it), otherwise `'oldest'`. With stalls
+   * allowed, `'oldest'` can stall writes permanently: a writer to one database
+   * waits for memory held by other databases that nothing is writing to, so
+   * nothing flushes them. Without stalls, `'largestAcrossDBs'` has measured
+   * slower. Can be changed while databases are open; an explicit value stays
+   * in effect for the life of the process.
+   */
+  writeBufferManagerFlushPolicy?: WriteBufferManagerFlushPolicy;
 };
+type WriteBufferManagerFlushPolicy = "oldest" | "largest" | "largestAcrossDBs";
 type RegistryStatusTransaction = {
   /** The transaction id assigned by the database descriptor. */
   id: number;
@@ -2455,6 +2565,11 @@ type WriteBufferManagerStats = {
   mutableMemoryUsage: number;
   allowStall: boolean;
   costToCache: boolean;
+  /**
+   * The flush policy in effect: the live manager's, or the one a manager would
+   * be created with (see `writeBufferManagerFlushPolicy`).
+   */
+  flushPolicy: WriteBufferManagerFlushPolicy;
   stallActive: boolean;
   /**
    * How long the current stall has been active, in milliseconds; 0 when not
@@ -2537,6 +2652,10 @@ export declare const constants: {
   RETRY_NOW_VALUE: number;
   TRANSACTION_LOG_TOKEN: number;
   TRANSACTION_LOG_ENTRY_HEADER_SIZE: number;
+  TRANSACTION_LOG_ENTRY_LAST_FLAG: number;
+  TRANSACTION_LOG_ENTRY_ABORTED_FLAG: number;
+  /** 1 when the linked RocksDB exports the conflict check run before the log write. */
+  PRELOG_CONFLICT_CHECK: number;
   TRANSACTION_LOG_FILE_HEADER_SIZE: number;
   ITERATOR_REVERSE_FLAG: number;
   ITERATOR_INCLUSIVE_END_FLAG: number;
@@ -2547,6 +2666,7 @@ export declare const constants: {
   ITERATOR_HAS_TRANSACTION_ID_FLAG: number;
   ITERATOR_RESULT_DONE: number;
   ITERATOR_RESULT_FAST: number;
+  ITERATOR_RESULT_RELEASED: number;
 };
 /**
  * The friendly names of every compression algorithm compiled into the loaded
@@ -2602,7 +2722,7 @@ export declare const currentThreadId: () => number;
  * Reads the process-wide steady clock: `std::chrono::steady_clock` as
  * fractional milliseconds from an unspecified origin that is fixed for the life
  * of the process. Every sample in the process — main thread and every
- * `worker_threads` worker, whenever started or restarted, on Node, Bun and Deno
+ * `worker_threads` worker, whenever started or restarted, on Node and Bun
  * — is in one domain, and the difference between two samples is real elapsed
  * time unaffected by wall-clock steps. Non-decreasing but not unique (two
  * samples may be equal; treat `==` as "not later"). Windows QPC samples from
@@ -2658,8 +2778,6 @@ export declare const stats: {
     All: number;
   };
 };
-//#endregion
-//#region src/backup-stream.d.ts
 /**
  * Options for streaming a backup via `db.backup(stream)`.
  */
@@ -2689,8 +2807,6 @@ interface BackupStreamOptions {
    */
   transactionLogs?: boolean;
 }
-//#endregion
-//#region src/count-estimator.d.ts
 interface CountEstimatorOptions extends CountEstimateOptions {}
 /**
  * Progressively refines a range key-count estimate as an iterator traverses
@@ -2740,8 +2856,6 @@ export declare class CountEstimator {
    */
   estimate(): CountEstimate;
 }
-//#endregion
-//#region src/database.d.ts
 type TransactionCallback<T> = (txn: Transaction, attempt: number) => T | PromiseLike<T>;
 interface RocksDatabaseOptions extends StoreOptions {
   /**
@@ -2761,7 +2875,8 @@ interface TransactionOptions extends NativeTransactionOptions {
   /**
    * Whether to retry the transaction if it fails with `IsBusy`.
    *
-   * @default `true` when the transaction is bound to a transaction log, otherwise `false`
+   * @default `true` when the transaction wrote a transaction-log batch or was refused before
+   * writing one (`refusedBeforeLog`), otherwise `false`
    */
   retryOnBusy?: boolean;
 }
@@ -3094,6 +3209,17 @@ export declare class RocksDatabase extends DBI<DBITransactional> {
    * @return {void} Does not return a value.
    */
   flushSync(options?: FlushOptions): void;
+  /**
+   * Syncs the write-ahead log to stable storage (`DB::SyncWAL`), so every WAL-backed write
+   * committed before the call survives a power loss afterwards. Unlike {@link flush} it writes
+   * no SST file and never stalls writers. Not covered: writes that skipped the WAL
+   * (`disableWAL`), and transaction-log files, which keep their own durability. Runs on the
+   * libuv threadpool; RocksDB serializes concurrent syncs of one WAL, so keep one in flight
+   * per database rather than parking pool workers on a queue of them.
+   *
+   * On a read-only database the returned promise resolves without syncing.
+   */
+  syncWal(): Promise<void>;
   /**
    * Returns the most recent {@link BackgroundError} observed on this database, or
    * `null` when none has occurred. The `'error'` event is the push counterpart;
@@ -3564,8 +3690,6 @@ export declare class RocksDatabase extends DBI<DBITransactional> {
    */
   withLock(key: Key, callback: () => void | Promise<void>): Promise<void> | undefined;
 }
-//#endregion
-//#region src/parse-transaction-log.d.ts
 interface LogEntry {
   anomalies?: string[];
   data?: Buffer;
@@ -3589,8 +3713,6 @@ interface TransactionLog$1 {
 export declare function parseTransactionLog(path: string, options?: {
   skipData?: boolean;
 }): TransactionLog$1;
-//#endregion
-//#region src/transaction-log-reader.d.ts
 /**
  * Thrown when an entry's framing is broken. `resyncPosition` is the offset where valid framing
  * resumes, when one was found: the frames after a mid-log break are intact and reachable, and
@@ -3611,12 +3733,29 @@ export declare class CorruptFrameError extends RangeError {
   unreadableBytes: number;
   constructor(message: string, logId: number, position: number, resyncPosition: number | undefined);
 }
-//#endregion
-//#region src/index.d.ts
+/**
+ * Thrown when an iterator advances past segments it never read and that can no longer be read —
+ * typically ones retention purged while the reader lagged. The entries they held cannot be
+ * recovered from this log, so a consumer replicating it must not treat the stream as contiguous.
+ * Missing sequence numbers prove only that continuity is unknown, not that each held entries.
+ *
+ * The iterator is left positioned at the start of `resumeLogId`, so a caller that accepts the
+ * discontinuity calls `next()` again to continue from there.
+ *
+ * Not a `RangeError`: this is not broken framing, and consumers classify those by class.
+ */
+export declare class TransactionLogGapError extends Error {
+  /** Sequence number of the last segment this iterator read up to. */
+  logId: number;
+  /** Sequence number of the segment iteration resumes in. */
+  resumeLogId: number;
+  /** Segments between `logId` and `resumeLogId` that could not be read. Empty segments are not counted. */
+  unavailableLogCount: number;
+  constructor(logId: number, resumeLogId: number, unavailableLogCount: number);
+}
 export declare const versions: {
   rocksdb: string;
   "rocksdb-js": string;
 };
-//#endregion
-export type { BackgroundErrorOptions, BackupInfo, BackupOptions, BackupStreamOptions, CompressionAlgorithm, CompressionInfo, CompressionOption, CountEstimate, CountEstimateOptions, CountEstimatorOptions, FlushOptions, GetStatsMethod, IteratorOptions, Key, LogOptions, RestoreMode, RestoreOptions, RocksDBStat, RocksDBStats, RocksDatabaseOptions, StatsAll, StatsAllExtras, StatsBasics, StatsCurated, StatsCuratedExtras, StatsDefault, StatsHistogramData, StatsValue, StoreContext, StoreGetOptions, StoreIteratorOptions, StorePutOptions, StoreRangeOptions, StoreRemoveOptions, TransactionEntry, TransactionLogFileValidation, TransactionLogPosition, TransactionLogStats, TransactionLogStoreValidation, ValidateTransactionLogStoreOptions, WriteBufferManagerStats };
+export type { BackgroundErrorOptions, BackupInfo, BackupOptions, BackupStreamOptions, CompressionAlgorithm, CompressionInfo, CompressionOption, CountEstimate, CountEstimateOptions, CountEstimatorOptions, FlushOptions, GetStatsMethod, IteratorOptions, Key, LogOptions, RestoreMode, RestoreOptions, RocksDBStat, RocksDBStats, RocksDatabaseOptions, StatsAll, StatsAllExtras, StatsBasics, StatsCurated, StatsCuratedExtras, StatsDefault, StatsHistogramData, StatsValue, StoreContext, StoreGetOptions, StoreIteratorOptions, StorePutOptions, StoreRangeOptions, StoreRemoveOptions, TransactionEntry, TransactionLogFileValidation, TransactionLogPosition, TransactionLogStats, TransactionLogStoreValidation, ValidateTransactionLogStoreOptions, WriteBufferManagerFlushPolicy, WriteBufferManagerStats };
 //# sourceMappingURL=index.d.mts.map