@harperfast/rocksdb-js 2.8.0 → 2.9.1

package/docs/stats.md

@@ -73,6 +73,28 @@ By default, `db.getStats()` returns properties only.
 | `txnlog.totalSizeBytes`                     | Total on-disk size in bytes of all transaction log files, summed across all logs.                                                                                                                                             | gauge  |
 | `txnlog.transactionsWritten`                | Cumulative number of transactions successfully written to the logs (lifetime total), summed across all logs.                                                                                                                  | ticker |
 | `txnlog.uncommittedTransactions`            | Number of transactions written to a log but not yet committed to RocksDB, summed across all logs.                                                                                                                             | gauge  |
+| `writeBufferManager.bufferSize`             | The `WriteBufferManager`'s memtable-memory budget in bytes, read live. `0` when no manager is configured. **Process-wide**, not per-database — see below.                                                                     | gauge  |
+| `writeBufferManager.memoryUsage`            | Total memtable memory in bytes charged against the manager. **Process-wide.**                                                                                                                                                 | gauge  |
+| `writeBufferManager.mutableMemoryUsage`     | The share of `writeBufferManager.memoryUsage` held by active (mutable) memtables; the rest is awaiting flush or retained write history. **Process-wide.**                                                                     | gauge  |
+| `writeBufferManager.stallActive`            | `1` while the manager is stalling writes, else `0`. **Process-wide.**                                                                                                                                                         | gauge  |
+| `writeBufferManager.stallActiveMs`          | How long the current stall has been active, in milliseconds; `0` when not stalled. Sampled once a second by the stall watchdog. **Process-wide.**                                                                             | gauge  |
+
+#### The `writeBufferManager.*` keys are process-wide
+
+The `WriteBufferManager` is a **singleton shared by every database opened in this process**,
+`worker_threads` included. These five keys therefore describe the whole process no matter which
+database's `getStats()` returned them — two databases in one process always report identical
+values, and the numbers do not attribute memory to the database you asked. They appear regardless
+of `enableStats`, like the `txnlog.*` and `commitPipeline.*` keys.
+`getWriteBufferManagerStats()` returns the same values plus the manager's
+configuration and its live column-family inventory.
+
+`writeBufferManager.stallActive` is the only signal that distinguishes a `WriteBufferManager`
+stall from an idle database. RocksDB's own stall accounting does not cover it:
+`rocksdb.stall.micros`, `rocksdb.db.write.stall` and the `'writeStall'` event all track the
+`WriteController` (L0 and pending-compaction triggers), while a `WriteBufferManager` stall blocks
+writers on a separate queue and touches none of them. Through an eight-hour production wedge all
+three read `0` (HarperFast/rocksdb-js#822).
 
 ### Basic + Curated Stats
 
@@ -433,6 +455,27 @@ which includes all of the stats above plus the following:
 | `rocksdb.write.self`                                               | Number of writes processed by the calling thread itself (as the write group leader).                                                          | ticker    |
 | `rocksdb.write.wal`                                                | Number of writes that were written to the WAL.                                                                                                | ticker    |
 
+#### `flush.reason.*` vs `atomic_flush.request.reason.*`
+
+This library always opens databases with RocksDB's `atomic_flush`. That does **not** move
+`WriteBufferManager`-pressure flushes off `rocksdb.flush.reason.write_buffer_manager` and onto
+`rocksdb.atomic_flush.request.reason.write_buffer_manager` — measured on this library, both counters
+move together (HarperFast/rocksdb-js#822).
+
+They count different events, though, and neither magnitude can be derived from the other:
+
+- `rocksdb.atomic_flush.request.reason.<reason>` counts flush **requests** — scheduling decisions.
+- `rocksdb.flush.reason.<reason>` counts the flushes that actually **ran**.
+
+Measured ratios in one process ranged from 1:1 (a single column family) to about 1:2 (two to six
+column families), and requests can outnumber executed flushes when several databases share one
+manager. Treat them as two views of the same pressure, not as a fixed multiple.
+
+Neither counter reports a stall. A `0` on both during a `WriteBufferManager` stall is the expected
+reading, not a broken counter: the flush trigger looks only at _mutable_ memory, so a budget held by
+retained history or by memtables already awaiting flush never requests a flush at all.
+`writeBufferManager.stallActive` is what distinguishes that state from an idle database.
+
 ## Transaction Log Stats
 
 `log.getStats()` returns a detailed snapshot for a single transaction log store. All sizes are in
@@ -477,7 +520,9 @@ const stats: TransactionLogStats = log.getStats();
   - `offset: number` The byte offset within that file.
 - `purge: object`
   - `oldestFileAgeMs: number` Age in milliseconds of the oldest file on disk.
-  - `purgeableFiles: number` Number of files eligible for purge under the retention policy.
+  - `purgeableFiles: number` Number of files below the retention floor (the `txn.state` sequence,
+    or the highest sequence when there is no persisted flush position) that are eligible for purge
+    under the retention policy.
   - `retainedUnflushedFiles: number` Number of files past the retention threshold but retained
     because they are unflushed.
   - `lastPurgeMs: number` Timestamp of the last purge scan.

package/docs/transaction-log.md

@@ -198,7 +198,9 @@ await db.transaction((txn) => {
 - Log files are automatically rotated when either the index or data file reaches their configured
   maximum sizes
 - Rotation happens on the next write after the size limit is exceeded
-- Old log files can be automatically purged based on retention policy
+- Old log files can be automatically purged based on retention policy. The sequence file named by
+  `txn.state` and every newer file form the live store's retention floor, so an idle store can keep
+  one bounded file past the cutoff until a later write rotates and flushes it.
 
 ### Error Handling
 
@@ -218,10 +220,21 @@ The CLI exposes the same check as `verify-logs [name]`.
 
 ## Reading The Transaction Log
 
-Log entries are not guaranteed to be in order, but are guaranteed to have a monotonic timestamp.
-When reading the transaction log file, each transaction entry header must be read, then sorted and
-indexed. Using this index, queries can find all entries within a time range using a binary search
-and seek to get the associated entry data.
+Log entries are not in timestamp order. A transaction claims its timestamp from the process-wide
+monotonic clock when it is constructed but is appended when it commits, so under concurrency a later
+entry can carry a smaller timestamp; an entry that adopted an origin timestamp with
+`txn.setTimestamp()` carries that origin's clock instead. Timestamps are not unique: the monotonic
+clock never issues the same value twice within a process, but a restart after the wall clock moved
+backwards can reissue one, and `txn.setTimestamp()` can assign any value — including one already
+in the log — to as many transactions as the caller likes. Deduplicating on the timestamp alone is
+therefore never safe; a consumer that needs identity has to supply it (Harper pairs the timestamp
+with the originating node).
+
+When reading the transaction log file, each transaction entry header must be read and indexed. The
+index records only the entries whose timestamp is greater than every earlier one in the file — a
+running maxima — and a query seeks to the lower bound of that index, which is guaranteed to sit at
+or before every entry in the requested range. Reading forward from there and filtering is what makes
+range queries correct on an unordered file.
 
 ### Sequential Read
 

package/docs/transaction-timestamp-integrity-design.md

@@ -0,0 +1,123 @@
+# Transaction timestamp integrity
+
+## Intent
+
+Harper's source-record version and a RocksDB transaction's write identity are separate values.
+Harper's `RecordEncoder` and `PrimaryRocksDatabase` already own the record value layout and decode
+its metadata; rocksdb-js owns the transaction timestamp and uses it as the transaction-log batch
+key. This change only makes overriding that timestamp safe for Harper's two legitimate adopters,
+replication apply and crash replay.
+
+## Invariant
+
+Within one transaction attempt, the timestamp cannot change after the transaction leaves `Pending`
+or stages a database write or transaction-log entry. A retry may adopt again only when the failed
+attempt did not durably write its log batch; once a batch is durable, its timestamp remains frozen
+across retry resets. Encoding a transaction timestamp into record bytes remains the producer's
+obligation because native code cannot observe when a producer reads and copies the timestamp.
+
+## Requirements traced to Harper
+
+- HarperFast/harper#2065 intentionally allowed a source record's version to differ from the
+  transaction timestamp.
+- HarperFast/harper#2409 fixed the immediate subscription regression by carrying both values in
+  Harper's audit representation.
+- HarperFast/harper#2411 and HarperFast/harper-pro#790 require replay and replication respectively
+  to adopt the origin transaction timestamp through `setTimestamp()` before applying writes.
+- HarperFast/harper#2412 owns the optional second version word in `RecordEncoder`, including its
+  encoding, decoding, compatibility gates, and call-site audit.
+
+Checked against Harper's source rather than the issue text: `resources/RecordEncoder.ts` decodes the
+value header itself (producing `localTime`, `version` and the metadata flags) and owns every flag bit
+in that word, and `resources/PrimaryRocksDatabase.ts#getEntry()` overrides the base read entirely —
+it calls `super.getSync()`/`super.get()` and reads the encoder's output. So nothing in Harper would
+call a rocksdb-js decoder for the second word. Harper's three `setTimestamp()` call sites
+(`resources/DatabaseTransaction.ts` at the read-transaction, save and retry-replay paths) each set it
+on a freshly constructed transaction before any write, which is what this change makes enforceable.
+
+One consequence for #2412 that is not in its call-site audit: `PrimaryRocksDatabase#getEntry()` passes
+the cached entry's `version` as `expectedVersion` to the verification table, and the table keys on the
+value's **first** word. Once `version` becomes the record version, that argument has to become
+`localTime`, or cached reads stop hitting the fast path — and a record version that happened to equal
+the key's current first word would report a stale copy as fresh.
+
+## Approaches considered
+
+### Different layer
+
+Teach rocksdb-js to define and decode Harper's second version word. Rejected because Harper's
+`PrimaryRocksDatabase#getEntry()` already overrides the base read path and receives the structured
+entry produced by Harper's `RecordEncoder`; a second decoder in rocksdb-js would duplicate the
+layout and cannot supply Harper's other metadata fields.
+
+Reserving Harper's proposed `0x20000` bit in rocksdb-js without decoding it was also considered.
+Rejected because rocksdb-js does not allocate the producer's metadata namespace: Harper already
+defines the other optional-field flags in that word, while rocksdb-js knows `0x10000` only because
+the native verification table gives that specific bit behavior. A bit with no native behavior or
+public rocksdb-js contract belongs beside Harper's encoder and downgrade gate.
+
+### Deeper cause
+
+Remove `setTimestamp()` and redesign replay and replication so rocksdb-js always issues a new local
+timestamp. Rejected because Harper's record-to-log lookup and per-origin resume cursor require the
+origin transaction timestamp to remain the transaction-log key on every receiver.
+
+A construction-time timestamp option was considered as a safer replacement for the setter. It
+would add another public API and migrate the replay and replication callers, but could not remove
+the existing setter compatibly; therefore it would not eliminate the state it is meant to prevent.
+
+### Do less
+
+Make no rocksdb-js change and rely on Harper always calling `setTimestamp()` before staging writes.
+Rejected because the current public API permits a later call, after the producer has encoded the old
+timestamp or the log batch has captured it, which can split one transaction across two identities.
+
+For the separate clock-rollback concern, seeding the process clock from the largest durable key
+across the database's transaction logs was considered and rejected on Harper's own log layout.
+`resources/RocksTransactionLogStore.ts` opens one log per origin node — `useLog('local')` for locally
+originated writes and `logById(nodeId)` for each peer — and a receiver deliberately keeps the origin's
+clock as the key in the peer's log (HarperFast/harper-pro#790). A "largest key in the database" seed
+therefore reads adopted peer keys as if they were locally issued: one peer running ahead ratchets this
+node's clock on its next restart, and the node then originates those keys to its own peers. Reading
+the headers only to warn has the same ambiguity — normal peer skew would be reported as a local clock
+rollback. A correct floor has to be told which log is locally originated (a per-log opt-in, or an
+explicit "raise the floor to this value" call Harper makes from its own `local` log), which is a
+different API than the one the issue sketched and is not a prerequisite for the dual-version record
+work.
+
+### Chosen
+
+Keep `setTimestamp()` as the adoption API, reject invalid timestamp values, and reject changes once
+the transaction is no longer pending or has staged a write or log entry. Preserve the existing
+retry behavior: a reset with no durable batch clears staged writes and may adopt again, while
+`committedPosition` freezes a timestamp whose batch already landed. Remove the second-word read API,
+flag, and clock-floor-at-open implementation from this PR. The first two belong to Harper's record
+codec; the clock floor addresses a pre-existing restart/clock-rollback concern and, as specified,
+would seed from adopted origin keys as readily as from locally issued ones. The existing
+`latestTimestamp` behavior used to populate new transaction-log segment headers remains unchanged.
+
+Document what is true about log keys today, which the issue also asked for: entries are appended in
+commit order rather than timestamp order, an adopted key carries another node's clock, keys are
+unique only within a process, and `findPositionByTimestamp` is a running-maxima seek rather than a
+sorted binary search.
+
+## Verification route
+
+The existing transaction tests cover all transaction modes, invalid inputs, post-write and
+post-dispatch rejection, log-entry staging, and a coordinated retry whose log batch is already
+durable. README and AGENTS documentation will state the enforceable transaction-layer rule and the
+producer-owned record boundary. The repository build, native tests, full Vitest suite, and checks
+will run before push.
+
+## Alternatives rejected
+
+Freezing the timestamp on the first `getTimestamp()` call would break existing documented and tested
+read-then-override behavior. A getter is observational for callers that do not encode its result,
+and making every read a state transition still could not prove that a producer's record bytes match
+the transaction timestamp. The enforced boundary therefore remains the first native staging event;
+producer ordering is documented explicitly.
+
+Retaining the removed clock-floor branch's far-future header bound would change the append path using
+a ten-year policy introduced only for that floor. This change instead restores `latestTimestamp` and
+segment-header behavior exactly to `origin/main`; any provenance-aware clock rollback design belongs
+in separate work.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@harperfast/rocksdb-js",
-  "version": "2.8.0",
+  "version": "2.9.1",
   "description": "RocksDB binding for Node.js",
   "homepage": "https://github.com/HarperFast/rocksdb-js",
   "bugs": [
@@ -32,38 +32,38 @@
     }
   },
   "dependencies": {
-    "@harperfast/extended-iterable": "1.0.3",
-    "msgpackr": "2.0.6",
-    "ordered-binary": "1.6.1"
+    "@harperfast/extended-iterable": "^1.0.3",
+    "msgpackr": "^2.0.6",
+    "ordered-binary": "^1.6.1"
   },
   "devDependencies": {
     "@rollup/plugin-replace": "6.0.3",
-    "@types/node": "26.2.0",
+    "@types/node": "26.5.0",
     "@vitest/coverage-v8": "4.1.11",
     "cross-env": "10.1.0",
     "dotenv": "17.4.2",
-    "lefthook": "2.1.10",
+    "lefthook": "2.1.12",
     "lmdb": "3.5.6",
-    "node-gyp": "13.0.1",
-    "oxfmt": "0.64.0",
-    "oxlint": "1.79.0",
+    "node-gyp": "13.0.2",
+    "oxfmt": "0.67.0",
+    "oxlint": "1.82.0",
     "prebuildify": "6.0.1",
     "semver": "7.8.5",
     "tar": "7.5.22",
-    "tsdown": "0.22.14",
+    "tsdown": "0.23.0",
     "tslib": "2.8.1",
     "typescript": "7.0.2",
     "vitest": "4.1.11"
   },
   "optionalDependencies": {
-    "@harperfast/rocksdb-js-darwin-arm64": "2.8.0",
-    "@harperfast/rocksdb-js-darwin-x64": "2.8.0",
-    "@harperfast/rocksdb-js-linux-arm64-glibc": "2.8.0",
-    "@harperfast/rocksdb-js-linux-arm64-musl": "2.8.0",
-    "@harperfast/rocksdb-js-linux-x64-glibc": "2.8.0",
-    "@harperfast/rocksdb-js-linux-x64-musl": "2.8.0",
-    "@harperfast/rocksdb-js-win32-arm64": "2.8.0",
-    "@harperfast/rocksdb-js-win32-x64": "2.8.0"
+    "@harperfast/rocksdb-js-darwin-arm64": "2.9.1",
+    "@harperfast/rocksdb-js-darwin-x64": "2.9.1",
+    "@harperfast/rocksdb-js-linux-arm64-glibc": "2.9.1",
+    "@harperfast/rocksdb-js-linux-arm64-musl": "2.9.1",
+    "@harperfast/rocksdb-js-linux-x64-glibc": "2.9.1",
+    "@harperfast/rocksdb-js-linux-x64-musl": "2.9.1",
+    "@harperfast/rocksdb-js-win32-arm64": "2.9.1",
+    "@harperfast/rocksdb-js-win32-x64": "2.9.1"
   },
   "engines": {
     "node": "^22.18.0 || >=24.0.0"
@@ -76,7 +76,7 @@
     "bench": "cross-env CI=1 node --expose-gc ./node_modules/vitest/vitest.mjs bench --passWithNoTests --outputJson benchmark-results.json",
     "bench:bun": "cross-env CI=1 bun --bun bench",
     "bench:deno": "cross-env CI=1 deno run --allow-all --sloppy-imports ./node_modules/vitest/vitest.mjs bench",
-    "build": "pnpm build:bundle && pnpm rebuild",
+    "build": "pnpm build:bundle && pnpm run rebuild",
     "build:binding": "node-gyp build",
     "build:binding:debug": "node-gyp build --coverage --debug --verbose",
     "build:bundle": "node scripts/clean-dist.mjs && tsdown -c tsdown.config.ts",