@harperfast/rocksdb-js 2.11.0 → 2.13.0

package/docs/backups.md

@@ -80,6 +80,33 @@ verification). Pass `verifyTransactionLogs: false` to check only the RocksDB fil
 stores (e.g. the `transaction_logs` directory of a closed database) can be validated directly with
 `validateTransactionLogStore()`.
 
+Pass `verifyTables: true` to check that **this build** can read the backup, not only that its files
+are intact. Every table file is opened in turn (one at a time), so verification rejects, naming the
+file, when a table uses a `format_version` or a compression algorithm this build does not support.
+That is the case after a downgrade, or when restoring a backup taken by a build that linked a
+compressor this one lacks. Blob files are not checked.
+
+Check before a destructive restore. The default `purgeAllFiles` mode deletes the destination before
+copying, so an unreadable backup would otherwise destroy the database it was meant to replace:
+
+```typescript
+import { backups, fileLockRelease, tryFileLock } from '@harperfast/rocksdb-js';
+import { join } from 'node:path';
+
+// A shared lock keeps a concurrent delete/purge from changing the backup between the two calls.
+// Shared locks coexist, so restore can still take its own.
+const token = tryFileLock(join(backupDir, '.backup.lock'), true);
+if (!token) {
+	throw new Error('A backup, delete, or purge is running on this directory; retry later');
+}
+try {
+	await backups.verify(backupDir, backupId, { verifyWithChecksum: true, verifyTables: true });
+	await backups.restore(backupDir, dbPath, { backupId });
+} finally {
+	fileLockRelease(token);
+}
+```
+
 ### Restoring
 
 ```typescript

package/docs/performance/mutex-contention.md

@@ -171,11 +171,12 @@ A separate native reproduction confirmed [#860](https://github.com/HarperFast/ro
 `tryClose()` releases its written-position lock before latching admission closed.
 A transaction can bind, finish `writeBatch()` and reduce the pending count back to zero
 between those checks. A temporary scheduling hook before phase 3 reproduced
-`pending=0`, one real uncommitted position, and `tryClose() == true`. This is a
+`pending=0`, one real uncommitted position, and `tryClose() == true`. This was a
 pre-existing close-protocol defect, not fixed by the completion-lock change. Its
 production trigger is a concurrent destructive log purge; no customer incident or
-JavaScript-level data loss was reproduced. Keep the binding lock until a separately
-reviewed admission/drain fix resolves that handoff.
+JavaScript-level data loss was reproduced. It is now fixed without changing the binding
+lock: every bind advances a generation that `tryClose()` rechecks before latching (see
+"Conditional close" in `src/binding/transaction_log/DESIGN.md`).
 
 The investigation also found [#859](https://github.com/HarperFast/rocksdb-js/issues/859):
 the DB-instance stress test awaits an empty promise array in its worker-close phase.

package/docs/stats.md

@@ -189,6 +189,10 @@ which includes all of the stats above plus the following:
 | `rocksdb.blobdb.gc.num.new.files`                                  | Number of new blob files generated by garbage collection (legacy stacked BlobDB).                                                             | ticker    |
 | `rocksdb.blobdb.get.micros`                                        | Distribution of `Get()` latencies (microseconds) (legacy stacked BlobDB).                                                                     | histogram |
 | `rocksdb.blobdb.key.size`                                          | Distribution of key sizes for BlobDB operations.                                                                                              | histogram |
+| `rocksdb.blobdb.lazy.partial.bytes.saved`                          | Bytes not read from blob files thanks to partial (byte-range) lazy reads: the value size minus the bytes actually read.                       | ticker    |
+| `rocksdb.blobdb.lazy.partial.read.count`                           | Number of lazy blob reads that read only a byte range of an uncompressed blob value.                                                          | ticker    |
+| `rocksdb.blobdb.lazy.read.bytes`                                   | Bytes read from storage while resolving lazy wide-column results (`GetEntityLazy`/`MultiGetEntityLazy`).                                      | ticker    |
+| `rocksdb.blobdb.lazy.read.count`                                   | Number of storage reads issued while resolving lazy wide-column results (`GetEntityLazy`/`MultiGetEntityLazy`).                               | ticker    |
 | `rocksdb.blobdb.multiget.micros`                                   | Distribution of `MultiGet()` latencies (microseconds) (legacy stacked BlobDB).                                                                | histogram |
 | `rocksdb.blobdb.next.micros`                                       | Distribution of iterator `Next()` latencies (microseconds) (legacy stacked BlobDB).                                                           | histogram |
 | `rocksdb.blobdb.num.get`                                           | Number of `Get()` calls (legacy stacked BlobDB).                                                                                              | ticker    |
@@ -493,7 +497,9 @@ const stats: TransactionLogStats = log.getStats();
 
 - `name: string` The name of the transaction log store.
 - `path: string` Filesystem path to the log store's directory.
-- `fileCount: number` Number of sequence log files on disk for this log.
+- `fileCount: number` Number of sequence log files on disk for this log. `0` does not mean the log
+  was never written: retention can purge every file of an idle log. A log that has never been written
+  reports `currentSequenceNumber` 1; one emptied by retention reports a higher one.
 - `currentSequenceNumber: number` Sequence number of the active write file.
 - `oldestSequenceNumber: number` Sequence number of the oldest file still on disk.
 - `totalSizeBytes: number` Total on-disk size of all of this log's files.
@@ -522,9 +528,10 @@ const stats: TransactionLogStats = log.getStats();
   - `offset: number` The byte offset within that file.
 - `purge: object`
   - `oldestFileAgeMs: number` Age in milliseconds of the oldest file on disk.
-  - `purgeableFiles: number` Number of files below the retention floor (the `txn.state` sequence,
-    or the highest sequence when there is no persisted flush position) that are eligible for purge
-    under the retention policy.
+  - `purgeableFiles: number` Number of files past the retention threshold that lie entirely at or
+    before the flushed position, including the current file, so the next purge can delete them.
+  - `retentionPins: number` Backups currently copying this store's files. Ordinary purges of the
+    store are skipped while it is nonzero, so a value that stays nonzero points at a stalled backup.
   - `retainedUnflushedFiles: number` Number of files past the retention threshold but retained
     because they are unflushed.
   - `lastPurgeMs: number` Timestamp of the last purge scan.

package/docs/steady-clock-design.md

@@ -87,7 +87,7 @@ separate properties. No promise is made that runtime timers use the same suspend
 ## Verification
 
 The end-to-end route is the real N-API export through `src/index.ts`, parent/worker bracketing on
-Node, Bun and Deno, staggered/restarted and concurrent workers (including `workerFirst >= parentBefore + stagger - tolerance`), elapsed
+Node and Bun (Deno: local `pnpm test:deno` only, #947), staggered/restarted and concurrent workers (including `workerFirst >= parentBefore + stagger - tolerance`), elapsed
 progression, and built ESM/CJS
 exports. Temporarily substituting a worker-local clock must make those brackets fail, proving
 they reject the motivating defect.

package/docs/transaction-log.md

@@ -58,7 +58,7 @@ The token is used to validate that the file is indeed a transaction log.
 
 #### `version`
 
-The transaction log file format version. Currently, version `1` is the latest.
+The transaction log file format version. Currently, version `1` is the latest. Opening a database whose newest transaction-log segments include one with a higher version (written by a newer release) fails with an error naming the segment, rather than skipping it, and this release never appends after such a segment. A malformed or unreadable segment is still skipped.
 
 #### `timestamp`
 
@@ -84,9 +84,10 @@ The size of the data entry directly following the transaction header.
 
 Transaction entry related flags.
 
-| Flag                              | Value  | Description                                |
-| --------------------------------- | ------ | ------------------------------------------ |
-| `TRANSACTION_LOG_ENTRY_LAST_FLAG` | `0x01` | Indicates the last entry for a transaction |
+| Flag                                 | Value  | Description                                                                                         |
+| ------------------------------------ | ------ | --------------------------------------------------------------------------------------------------- |
+| `TRANSACTION_LOG_ENTRY_LAST_FLAG`    | `0x01` | Indicates the last entry for a transaction                                                          |
+| `TRANSACTION_LOG_ENTRY_ABORTED_FLAG` | `0x02` | On a transaction's first entry: the transaction did not commit, and readers skip all of its entries |
 
 ## Encoding Details
 
@@ -198,9 +199,23 @@ await db.transaction((txn) => {
 - Log files are automatically rotated when either the index or data file reaches their configured
   maximum sizes
 - Rotation happens on the next write after the size limit is exceeded
-- Old log files can be automatically purged based on retention policy. The sequence file named by
-  `txn.state` and every newer file form the live store's retention floor, so an idle store can keep
-  one bounded file past the cutoff until a later write rotates and flushes it.
+- Old log files can be automatically purged based on retention policy. A file is purgeable once it
+  is past the cutoff and lies entirely at or before the flushed position recorded in `txn.state`.
+  That includes the file `txn.state` names and the file currently being appended to, so an idle
+  store whose entries are all flushed ends up with no log files at all. The next write starts a new,
+  higher sequence; `txn.state` stays behind as the record of the highest sequence used, and a
+  restart appends past it. Before deleting a store's last file, retention syncs `txn.state` and its
+  directory; on a filesystem that rejects a directory fsync it keeps that file and warns once.
+- `txn.state` is rewritten in place (8 bytes at offset 0, not synced on each flush). Retention relies
+  on storage writing that range atomically: a power loss that tears it into a mix of old and new
+  bytes can name a later position than was flushed, and retention would then treat unflushed
+  entries below it as flushed.
+- A reader that starts from the last flushed position, or one already inside a purged file,
+  continues at the lowest retained sequence (or at the next write when nothing is retained).
+- Downgrading to a release without this rule after a store's last file was purged is unsafe: older
+  releases restart an empty store at sequence 1, below the flushed position, so a replay from
+  `txn.state` skips the new writes and retention treats them as flushed. Remove the store's
+  `txn.state` before downgrading.
 
 ### Error Handling
 
@@ -251,6 +266,11 @@ handle is returned, so no transaction can be constructed below it. Keys are not
 across segments, and a segment header records the store's latest timestamp only as of that
 segment's creation, so there is no shortcut: every segment is read.
 
+Retention can purge every segment of the named log. The walk then finds no keys and leaves the clock
+alone, so a backward step after restart can reissue a key that was only in purged segments. With
+age-based retention alone that needs a backward step at least as large as the retention period; a
+`purgeLogs({ before })` cutoff closer to now shrinks it to the time since that cutoff.
+
 Name only a log this process originates. A log a replication receiver writes under an adopted origin
 timestamp is keyed by another node's clock, and seeding from it would ratchet this process's clock
 to the fastest of those nodes at each restart. Native code cannot tell the two kinds of log apart,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@harperfast/rocksdb-js",
-  "version": "2.11.0",
+  "version": "2.13.0",
   "description": "RocksDB binding for Node.js",
   "homepage": "https://github.com/HarperFast/rocksdb-js",
   "bugs": [
@@ -38,39 +38,39 @@
   },
   "devDependencies": {
     "@rollup/plugin-replace": "6.0.3",
-    "@types/node": "26.6.4",
+    "@types/node": "26.6.5",
     "@vitest/coverage-v8": "4.1.11",
     "cross-env": "10.1.0",
-    "dotenv": "18.0.5",
-    "lefthook": "2.1.16",
-    "lmdb": "3.5.7",
+    "dotenv": "18.0.9",
+    "lefthook": "2.2.2",
+    "lmdb": "3.5.9",
     "node-gyp": "13.1.0",
-    "oxfmt": "0.71.0",
-    "oxlint": "1.86.0",
+    "oxfmt": "0.75.0",
+    "oxlint": "1.90.0",
     "prebuildify": "6.0.1",
     "semver": "7.8.5",
     "tar": "7.5.22",
-    "tsdown": "0.23.0",
+    "tsdown": "0.23.3",
     "tslib": "2.8.1",
     "typescript": "7.0.2",
     "vitest": "4.1.11"
   },
   "optionalDependencies": {
-    "@harperfast/rocksdb-js-darwin-arm64": "2.11.0",
-    "@harperfast/rocksdb-js-darwin-x64": "2.11.0",
-    "@harperfast/rocksdb-js-linux-arm64-glibc": "2.11.0",
-    "@harperfast/rocksdb-js-linux-arm64-musl": "2.11.0",
-    "@harperfast/rocksdb-js-linux-x64-glibc": "2.11.0",
-    "@harperfast/rocksdb-js-linux-x64-musl": "2.11.0",
-    "@harperfast/rocksdb-js-win32-arm64": "2.11.0",
-    "@harperfast/rocksdb-js-win32-x64": "2.11.0"
+    "@harperfast/rocksdb-js-darwin-arm64": "2.13.0",
+    "@harperfast/rocksdb-js-darwin-x64": "2.13.0",
+    "@harperfast/rocksdb-js-linux-arm64-glibc": "2.13.0",
+    "@harperfast/rocksdb-js-linux-arm64-musl": "2.13.0",
+    "@harperfast/rocksdb-js-linux-x64-glibc": "2.13.0",
+    "@harperfast/rocksdb-js-linux-x64-musl": "2.13.0",
+    "@harperfast/rocksdb-js-win32-arm64": "2.13.0",
+    "@harperfast/rocksdb-js-win32-x64": "2.13.0"
   },
   "engines": {
     "node": "^22.18.0 || >=24.0.0"
   },
   "gypfile": true,
   "rocksdb": {
-    "version": "11.8.1"
+    "version": "11.12.0"
   },
   "scripts": {
     "bench": "cross-env CI=1 node --expose-gc ./node_modules/vitest/vitest.mjs bench --passWithNoTests --outputJson benchmark-results.json",
@@ -81,7 +81,8 @@
     "build:binding:debug": "node-gyp build --coverage --debug --verbose",
     "build:bundle": "node scripts/clean-dist.mjs && tsdown -c tsdown.config.ts",
     "build:bundle:minify": "cross-env MINIFY=1 pnpm build:bundle",
-    "check": "pnpm type-check && pnpm lint && pnpm fmt:check",
+    "check": "pnpm check:code && pnpm fmt:check",
+    "check:code": "pnpm type-check && pnpm lint",
     "clean": "node-gyp clean",
     "coverage": "node scripts/coverage/main.ts",
     "coverage:native": "node scripts/native-test/coverage.mjs",