@harperfast/rocksdb-js 2.10.0 → 2.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +252 -62
- package/bin/rocksdb-js.mjs +49 -5
- package/dist/index.cjs +152 -27
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +134 -5
- package/dist/index.d.mts +134 -5
- package/dist/index.mjs +152 -28
- package/dist/index.mjs.map +1 -1
- package/docs/backups.md +27 -0
- package/docs/stats.md +55 -51
- package/docs/transaction-log.md +142 -4
- package/docs/transaction-timestamp-integrity-design.md +70 -8
- package/package.json +19 -18
package/docs/backups.md
CHANGED
|
@@ -80,6 +80,33 @@ verification). Pass `verifyTransactionLogs: false` to check only the RocksDB fil
|
|
|
80
80
|
stores (e.g. the `transaction_logs` directory of a closed database) can be validated directly with
|
|
81
81
|
`validateTransactionLogStore()`.
|
|
82
82
|
|
|
83
|
+
Pass `verifyTables: true` to check that **this build** can read the backup, not only that its files
|
|
84
|
+
are intact. Every table file is opened in turn (one at a time), so verification rejects, naming the
|
|
85
|
+
file, when a table uses a `format_version` or a compression algorithm this build does not support.
|
|
86
|
+
That is the case after a downgrade, or when restoring a backup taken by a build that linked a
|
|
87
|
+
compressor this one lacks. Blob files are not checked.
|
|
88
|
+
|
|
89
|
+
Check before a destructive restore. The default `purgeAllFiles` mode deletes the destination before
|
|
90
|
+
copying, so an unreadable backup would otherwise destroy the database it was meant to replace:
|
|
91
|
+
|
|
92
|
+
```typescript
|
|
93
|
+
import { backups, fileLockRelease, tryFileLock } from '@harperfast/rocksdb-js';
|
|
94
|
+
import { join } from 'node:path';
|
|
95
|
+
|
|
96
|
+
// A shared lock keeps a concurrent delete/purge from changing the backup between the two calls.
|
|
97
|
+
// Shared locks coexist, so restore can still take its own.
|
|
98
|
+
const token = tryFileLock(join(backupDir, '.backup.lock'), true);
|
|
99
|
+
if (!token) {
|
|
100
|
+
throw new Error('A backup, delete, or purge is running on this directory; retry later');
|
|
101
|
+
}
|
|
102
|
+
try {
|
|
103
|
+
await backups.verify(backupDir, backupId, { verifyWithChecksum: true, verifyTables: true });
|
|
104
|
+
await backups.restore(backupDir, dbPath, { backupId });
|
|
105
|
+
} finally {
|
|
106
|
+
fileLockRelease(token);
|
|
107
|
+
}
|
|
108
|
+
```
|
|
109
|
+
|
|
83
110
|
### Restoring
|
|
84
111
|
|
|
85
112
|
```typescript
|
package/docs/stats.md
CHANGED
|
@@ -32,53 +32,54 @@ const keysWrittenHistogram: StatsHistogramData = db.getStat('rocksdb.db.flush.mi
|
|
|
32
32
|
|
|
33
33
|
By default, `db.getStats()` returns properties only.
|
|
34
34
|
|
|
35
|
-
| Name | Description
|
|
36
|
-
| ------------------------------------------- |
|
|
37
|
-
| `columnFamily.pendingReclaims` | Number of dropped column families of this database whose physical `DropColumnFamily` has not completed: a commit admitted before the drop still holds it, or the drop failed and is retried on the next drop, open of that name, or close (see "Dropping column families" in the README).
|
|
38
|
-
| `commitPipeline.commitQueueDepth` | Number of async commits queued
|
|
39
|
-
| `commitPipeline.
|
|
40
|
-
| `
|
|
41
|
-
| `rocksdb.block-cache-
|
|
42
|
-
| `rocksdb.block-cache-usage`
|
|
43
|
-
| `rocksdb.
|
|
44
|
-
| `rocksdb.
|
|
45
|
-
| `rocksdb.cur-size-
|
|
46
|
-
| `rocksdb.
|
|
47
|
-
| `rocksdb.
|
|
48
|
-
| `rocksdb.estimate-
|
|
49
|
-
| `rocksdb.estimate-
|
|
50
|
-
| `rocksdb.
|
|
51
|
-
| `rocksdb.live-
|
|
52
|
-
| `rocksdb.
|
|
53
|
-
| `rocksdb.
|
|
54
|
-
| `rocksdb.num-
|
|
55
|
-
| `rocksdb.num-
|
|
56
|
-
| `rocksdb.num-
|
|
57
|
-
| `rocksdb.num-immutable-mem-table
|
|
58
|
-
| `rocksdb.num-
|
|
59
|
-
| `rocksdb.num-
|
|
60
|
-
| `rocksdb.num-running-
|
|
61
|
-
| `rocksdb.num-
|
|
62
|
-
| `rocksdb.
|
|
63
|
-
| `rocksdb.
|
|
64
|
-
| `rocksdb.
|
|
65
|
-
| `rocksdb.total-
|
|
66
|
-
| `
|
|
67
|
-
| `txnlog.
|
|
68
|
-
| `txnlog.
|
|
69
|
-
| `txnlog.
|
|
70
|
-
| `txnlog.
|
|
71
|
-
| `txnlog.
|
|
72
|
-
| `txnlog.
|
|
73
|
-
| `txnlog.
|
|
74
|
-
| `txnlog.
|
|
75
|
-
| `txnlog.
|
|
76
|
-
| `txnlog.
|
|
77
|
-
| `
|
|
78
|
-
| `writeBufferManager.
|
|
79
|
-
| `writeBufferManager.
|
|
80
|
-
| `writeBufferManager.
|
|
81
|
-
| `writeBufferManager.
|
|
35
|
+
| Name | Description | Type |
|
|
36
|
+
| ------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------ |
|
|
37
|
+
| `columnFamily.pendingReclaims` | Number of dropped column families of this database whose physical `DropColumnFamily` has not completed: a commit admitted before the drop still holds it, or the drop failed and is retried on the next drop, open of that name, or close (see "Dropping column families" in the README). | gauge |
|
|
38
|
+
| `commitPipeline.commitQueueDepth` | Number of async commits queued for the database's commit threads but not yet started. In the default single-lane mode each queued commit is the whole commit (log write + RocksDB commit) regardless of `commitThreads`; only under the two-lane pipeline (`ROCKSDB_JS_COMMIT_THREAD=2`) does this queue cover just the RocksDB commit, with the log write queued separately (see `commitPipeline.logQueueDepth`). | gauge |
|
|
39
|
+
| `commitPipeline.commitThreads` | Number of commit threads the database has started, at most the `commitThreads` setting it was opened with. Threads start only when commits overlap. | gauge |
|
|
40
|
+
| `commitPipeline.logQueueDepth` | Number of async commits queued on the database's transaction-log lane but not yet started (always `0` unless `ROCKSDB_JS_COMMIT_THREAD=2`). | gauge |
|
|
41
|
+
| `rocksdb.block-cache-capacity` | Capacity in bytes of the block cache. | gauge |
|
|
42
|
+
| `rocksdb.block-cache-pinned-usage` | Bytes occupied by pinned block cache entries. | gauge |
|
|
43
|
+
| `rocksdb.block-cache-usage` | Bytes currently used by block cache entries. | gauge |
|
|
44
|
+
| `rocksdb.compaction-pending` | `1` if at least one compaction is pending, else `0`. | gauge |
|
|
45
|
+
| `rocksdb.cur-size-active-mem-table` | Approximate size in bytes of the active memtable. | gauge |
|
|
46
|
+
| `rocksdb.cur-size-all-mem-tables` | Approximate size in bytes of the active and unflushed immutable memtables. | gauge |
|
|
47
|
+
| `rocksdb.current-super-version-number` | Current super-version number, incremented on each LSM change. | gauge |
|
|
48
|
+
| `rocksdb.estimate-live-data-size` | Estimated size in bytes of the live data. | gauge |
|
|
49
|
+
| `rocksdb.estimate-num-keys` | Estimated number of live keys. | gauge |
|
|
50
|
+
| `rocksdb.estimate-pending-compaction-bytes` | Estimated bytes compaction must rewrite to rebalance the LSM tree. | gauge |
|
|
51
|
+
| `rocksdb.live-blob-file-size` | Total size in bytes of blob files in the current version. | gauge |
|
|
52
|
+
| `rocksdb.live-sst-files-size` | Total size in bytes of SST files in the current version. | gauge |
|
|
53
|
+
| `rocksdb.mem-table-flush-pending` | `1` if a memtable flush is pending, else `0`. | gauge |
|
|
54
|
+
| `rocksdb.num-blob-files` | Number of blob files in the current version. | gauge |
|
|
55
|
+
| `rocksdb.num-deletes-active-mem-table` | Number of delete entries in the active memtable. | gauge |
|
|
56
|
+
| `rocksdb.num-entries-active-mem-table` | Number of entries in the active memtable. | gauge |
|
|
57
|
+
| `rocksdb.num-immutable-mem-table` | Number of immutable memtables not yet flushed. | gauge |
|
|
58
|
+
| `rocksdb.num-immutable-mem-table-flushed` | Number of immutable memtables already flushed. | gauge |
|
|
59
|
+
| `rocksdb.num-live-versions` | Number of live LSM versions (high values indicate versions held by iterators/snapshots). | gauge |
|
|
60
|
+
| `rocksdb.num-running-compactions` | Number of compactions currently running. | gauge |
|
|
61
|
+
| `rocksdb.num-running-flushes` | Number of flushes currently running. | gauge |
|
|
62
|
+
| `rocksdb.num-snapshots` | Number of unreleased snapshots. Any nonzero value blocks reclamation of obsolete versions behind the oldest one. | gauge |
|
|
63
|
+
| `rocksdb.oldest-snapshot-time` | Unix timestamp of the oldest unreleased snapshot, or `0`. | gauge |
|
|
64
|
+
| `rocksdb.size-all-mem-tables` | Approximate size in bytes of active, unflushed, and pinned memtables. | gauge |
|
|
65
|
+
| `rocksdb.total-blob-file-size` | Total size in bytes of all blob files across all versions. | gauge |
|
|
66
|
+
| `rocksdb.total-sst-files-size` | Total size in bytes of all SST files across all versions. | gauge |
|
|
67
|
+
| `txnlog.activeMaps` | Number of transaction log files currently memory-mapped, summed across all logs. | gauge |
|
|
68
|
+
| `txnlog.bytesWritten` | Cumulative bytes written to transaction logs (entry payload plus entry header, excluding file headers), summed across all logs. | ticker |
|
|
69
|
+
| `txnlog.fileCount` | Total number of transaction log files on disk, summed across all logs. | gauge |
|
|
70
|
+
| `txnlog.logCount` | Number of transaction log stores attached to the database. | gauge |
|
|
71
|
+
| `txnlog.mappedBytes` | Virtual address space in bytes reserved for memory-mapped log files, summed across all logs (on POSIX the active write file maps the full configured `maxFileSize`, so this is allocated address space, not resident memory). | gauge |
|
|
72
|
+
| `txnlog.overlayBytes` | File-backed overlay portion in bytes (POSIX only; `0` on Windows), summed across all logs — a closer proxy for real memory consumption than `mappedBytes`. | gauge |
|
|
73
|
+
| `txnlog.pendingTransactions` | Number of transactions bound to a log but not yet written via `writeBatch()`, summed across all logs. | gauge |
|
|
74
|
+
| `txnlog.replayGapBytes` | Bytes between the last flushed position and the write head (written but not yet flushed to the database), summed across all logs. | gauge |
|
|
75
|
+
| `txnlog.totalSizeBytes` | Total on-disk size in bytes of all transaction log files, summed across all logs. | gauge |
|
|
76
|
+
| `txnlog.transactionsWritten` | Cumulative number of transactions successfully written to the logs (lifetime total), summed across all logs. | ticker |
|
|
77
|
+
| `txnlog.uncommittedTransactions` | Number of transactions written to a log but not yet committed to RocksDB, summed across all logs. | gauge |
|
|
78
|
+
| `writeBufferManager.bufferSize` | The `WriteBufferManager`'s memtable-memory budget in bytes, read live. `0` when no manager is configured. **Process-wide**, not per-database — see below. | gauge |
|
|
79
|
+
| `writeBufferManager.memoryUsage` | Total memtable memory in bytes charged against the manager. **Process-wide.** | gauge |
|
|
80
|
+
| `writeBufferManager.mutableMemoryUsage` | The share of `writeBufferManager.memoryUsage` held by active (mutable) memtables; the rest is awaiting flush or retained write history. **Process-wide.** | gauge |
|
|
81
|
+
| `writeBufferManager.stallActive` | `1` while the manager is stalling writes, else `0`. **Process-wide.** | gauge |
|
|
82
|
+
| `writeBufferManager.stallActiveMs` | How long the current stall has been active, in milliseconds; `0` when not stalled. Sampled once a second by the stall watchdog. **Process-wide.** | gauge |
|
|
82
83
|
|
|
83
84
|
#### The `writeBufferManager.*` keys are process-wide
|
|
84
85
|
|
|
@@ -492,7 +493,9 @@ const stats: TransactionLogStats = log.getStats();
|
|
|
492
493
|
|
|
493
494
|
- `name: string` The name of the transaction log store.
|
|
494
495
|
- `path: string` Filesystem path to the log store's directory.
|
|
495
|
-
- `fileCount: number` Number of sequence log files on disk for this log.
|
|
496
|
+
- `fileCount: number` Number of sequence log files on disk for this log. `0` does not mean the log
|
|
497
|
+
was never written: retention can purge every file of an idle log. A log that has never been written
|
|
498
|
+
reports `currentSequenceNumber` 1; one emptied by retention reports a higher one.
|
|
496
499
|
- `currentSequenceNumber: number` Sequence number of the active write file.
|
|
497
500
|
- `oldestSequenceNumber: number` Sequence number of the oldest file still on disk.
|
|
498
501
|
- `totalSizeBytes: number` Total on-disk size of all of this log's files.
|
|
@@ -521,9 +524,10 @@ const stats: TransactionLogStats = log.getStats();
|
|
|
521
524
|
- `offset: number` The byte offset within that file.
|
|
522
525
|
- `purge: object`
|
|
523
526
|
- `oldestFileAgeMs: number` Age in milliseconds of the oldest file on disk.
|
|
524
|
-
- `purgeableFiles: number` Number of files
|
|
525
|
-
|
|
526
|
-
|
|
527
|
+
- `purgeableFiles: number` Number of files past the retention threshold that lie entirely at or
|
|
528
|
+
before the flushed position, including the current file, so the next purge can delete them.
|
|
529
|
+
- `retentionPins: number` Backups currently copying this store's files. Ordinary purges of the
|
|
530
|
+
store are skipped while it is nonzero, so a value that stays nonzero points at a stalled backup.
|
|
527
531
|
- `retainedUnflushedFiles: number` Number of files past the retention threshold but retained
|
|
528
532
|
because they are unflushed.
|
|
529
533
|
- `lastPurgeMs: number` Timestamp of the last purge scan.
|
package/docs/transaction-log.md
CHANGED
|
@@ -198,9 +198,23 @@ await db.transaction((txn) => {
|
|
|
198
198
|
- Log files are automatically rotated when either the index or data file reaches their configured
|
|
199
199
|
maximum sizes
|
|
200
200
|
- Rotation happens on the next write after the size limit is exceeded
|
|
201
|
-
- Old log files can be automatically purged based on retention policy.
|
|
202
|
-
|
|
203
|
-
|
|
201
|
+
- Old log files can be automatically purged based on retention policy. A file is purgeable once it
|
|
202
|
+
is past the cutoff and lies entirely at or before the flushed position recorded in `txn.state`.
|
|
203
|
+
That includes the file `txn.state` names and the file currently being appended to, so an idle
|
|
204
|
+
store whose entries are all flushed ends up with no log files at all. The next write starts a new,
|
|
205
|
+
higher sequence; `txn.state` stays behind as the record of the highest sequence used, and a
|
|
206
|
+
restart appends past it. Before deleting a store's last file, retention syncs `txn.state` and its
|
|
207
|
+
directory; on a filesystem that rejects a directory fsync it keeps that file and warns once.
|
|
208
|
+
- `txn.state` is rewritten in place (8 bytes at offset 0, not synced on each flush). Retention relies
|
|
209
|
+
on storage writing that range atomically: a power loss that tears it into a mix of old and new
|
|
210
|
+
bytes can name a later position than was flushed, and retention would then treat unflushed
|
|
211
|
+
entries below it as flushed.
|
|
212
|
+
- A reader that starts from the last flushed position, or one already inside a purged file,
|
|
213
|
+
continues at the lowest retained sequence (or at the next write when nothing is retained).
|
|
214
|
+
- Downgrading to a release without this rule after a store's last file was purged is unsafe: older
|
|
215
|
+
releases restart an empty store at sequence 1, below the flushed position, so a replay from
|
|
216
|
+
`txn.state` skips the new writes and retention treats them as flushed. Remove the store's
|
|
217
|
+
`txn.state` before downgrading.
|
|
204
218
|
|
|
205
219
|
### Error Handling
|
|
206
220
|
|
|
@@ -225,7 +239,8 @@ monotonic clock when it is constructed but is appended when it commits, so under
|
|
|
225
239
|
entry can carry a smaller timestamp; an entry that adopted an origin timestamp with
|
|
226
240
|
`txn.setTimestamp()` carries that origin's clock instead. Timestamps are not unique: the monotonic
|
|
227
241
|
clock never issues the same value twice within a process, but a restart after the wall clock moved
|
|
228
|
-
backwards can reissue one, and `txn.setTimestamp()` can assign any
|
|
242
|
+
backwards can reissue one unless the floor below is seeded, and `txn.setTimestamp()` can assign any
|
|
243
|
+
value — including one already
|
|
229
244
|
in the log — to as many transactions as the caller likes. Deduplicating on the timestamp alone is
|
|
230
245
|
therefore never safe; a consumer that needs identity has to supply it (Harper pairs the timestamp
|
|
231
246
|
with the originating node).
|
|
@@ -236,6 +251,129 @@ running maxima — and a query seeks to the lower bound of that index, which is
|
|
|
236
251
|
or before every entry in the requested range. Reading forward from there and filtering is what makes
|
|
237
252
|
range queries correct on an unordered file.
|
|
238
253
|
|
|
254
|
+
## The Timestamp Floor At Open
|
|
255
|
+
|
|
256
|
+
A transaction's timestamp is the key of the batch it is written under, so it has to stay unique
|
|
257
|
+
within the log it is written to. The process clock (`db.getMonotonicTimestamp()`) guarantees that
|
|
258
|
+
only within one process: a new process reads the wall clock again, so a backward step between runs
|
|
259
|
+
can reissue a key that is already durable in the log.
|
|
260
|
+
|
|
261
|
+
Opening with the `timestampFloorLog` option names the log whose keys this process originates. Every
|
|
262
|
+
segment of that store is then walked once, after open-time recovery has decided which bytes are
|
|
263
|
+
still durable, and the process clock is raised above the largest key found — before the database
|
|
264
|
+
handle is returned, so no transaction can be constructed below it. Keys are not ordered within or
|
|
265
|
+
across segments, and a segment header records the store's latest timestamp only as of that
|
|
266
|
+
segment's creation, so there is no shortcut: every segment is read.
|
|
267
|
+
|
|
268
|
+
Retention can purge every segment of the named log. The walk then finds no keys and leaves the clock
|
|
269
|
+
alone, so a backward step after restart can reissue a key that was only in purged segments. With
|
|
270
|
+
age-based retention alone that needs a backward step at least as large as the retention period; a
|
|
271
|
+
`purgeLogs({ before })` cutoff closer to now shrinks it to the time since that cutoff.
|
|
272
|
+
|
|
273
|
+
Name only a log this process originates. A log a replication receiver writes under an adopted origin
|
|
274
|
+
timestamp is keyed by another node's clock, and seeding from it would ratchet this process's clock
|
|
275
|
+
to the fastest of those nodes at each restart. Native code cannot tell the two kinds of log apart,
|
|
276
|
+
which is why the caller names it and why an unset option leaves the clock alone.
|
|
277
|
+
|
|
278
|
+
The walk is the cost of the guarantee, and it is paid on the calling thread before `open()` returns.
|
|
279
|
+
It reads entry headers through a shared 64 KiB window and seeks past payloads, so it scales with the
|
|
280
|
+
number of entries in the named log rather than its size. Measured on Linux with a warm page cache,
|
|
281
|
+
against the same database opened without the option:
|
|
282
|
+
|
|
283
|
+
| Named log | Added open time (median) |
|
|
284
|
+
| --------------------------------- | ------------------------ |
|
|
285
|
+
| 25,000 entries, 99 MB, 7 segments | ~31 ms |
|
|
286
|
+
| 250,000 entries, 19 MB | ~273 ms |
|
|
287
|
+
|
|
288
|
+
Roughly a millisecond per thousand entries. There is a second term on Windows only. A segment there
|
|
289
|
+
is pre-extended to `transactionLogMaxSize` when it is mapped and keeps that physical size for life —
|
|
290
|
+
rotation does not truncate it — so a segment's bytes past its last entry are zero padding, and the
|
|
291
|
+
walk has to read that padding to prove nothing durable hides in it (see fail closed, below). So on
|
|
292
|
+
Windows the walk also scales with `segments x transactionLogMaxSize`. Measured on Linux against
|
|
293
|
+
deliberately Windows-shaped segments, 80 MiB of padding across 5 segments added ~148 ms to the open
|
|
294
|
+
with a warm page cache; a cold cache or slower storage is proportionally worse, and this has not been
|
|
295
|
+
measured on Windows itself. Size the budget below with that in mind before enabling the option on a
|
|
296
|
+
Windows deployment with a long retention window.
|
|
297
|
+
|
|
298
|
+
Retention bounds a log's age, not its entry count, so
|
|
299
|
+
the walk is bounded directly instead: `ROCKSDB_JS_TIMESTAMP_FLOOR_SCAN_MS` (default `2000`) caps it,
|
|
300
|
+
newest segment first, and a walk that runs out of budget rejects an open that names the log. The
|
|
301
|
+
rejection reports the segments and bytes the walk covered, so the budget can be sized from what the
|
|
302
|
+
store actually costs.
|
|
303
|
+
The deadline is checked before each physical read; sequential headers share a 64 KiB window, so the
|
|
304
|
+
small amount of already-buffered header parsing can finish after the deadline but one large segment
|
|
305
|
+
cannot run unbounded.
|
|
306
|
+
The value is honored literally between `0` (scan nothing, and reject) and one day, and there is no
|
|
307
|
+
unbounded setting — the failure it bounds is an `open()` that does not return, so a deployment that
|
|
308
|
+
would rather wait raises the number. Above a day it is clamped, because the deadline is a monotonic
|
|
309
|
+
clock time point and a larger value overflows its resolution and wraps into the past.
|
|
310
|
+
A database opened without `timestampFloorLog` pays none of it. The walk runs while database opens
|
|
311
|
+
and closes are serialized process-wide, so increasing its budget can delay unrelated opens and
|
|
312
|
+
closes too — and the budget is **per database path**, not per process, so a process that opens twenty
|
|
313
|
+
opted-in databases can spend twenty times the number set here, serialized.
|
|
314
|
+
|
|
315
|
+
The seed is fail closed when `timestampFloorLog` is set, and it asks a stronger question of each
|
|
316
|
+
segment than recovery does. Recovery decides where it is safe to truncate; this has to establish that
|
|
317
|
+
it observed every durable key, so the walk may stop short of a segment's extent only on a proof that
|
|
318
|
+
no complete frame can remain: the framed entries reach the extent, or everything past them is zero,
|
|
319
|
+
or fewer than 13 bytes remain (too few to open an entry header). Anything else — a framing break, a
|
|
320
|
+
declared length that overruns, bytes that are neither framed nor padding — rejects the open, as does a
|
|
321
|
+
segment that cannot be opened or scanned, an exhausted budget, or a key more than ten years ahead of
|
|
322
|
+
the wall clock. Entries past a break can still be durable and `query()` resyncs to them, which is
|
|
323
|
+
exactly why this walk cannot treat them as absent.
|
|
324
|
+
|
|
325
|
+
The first open that names `timestampFloorLog` must be the writable primary. A read-only or secondary
|
|
326
|
+
handle cannot establish a cross-process snapshot: a primary could durably append a larger key as the
|
|
327
|
+
handle scans. Once this process has seeded that same log through a writable primary, read-only and
|
|
328
|
+
secondary descriptors reuse the established seed without rescanning. A handle that cannot meet this
|
|
329
|
+
rule must omit the option.
|
|
330
|
+
|
|
331
|
+
The writable recovery scan is separately bounded by `ROCKSDB_JS_TRANSACTION_LOG_RECOVERY_SCAN_MS`
|
|
332
|
+
(default `2000`). At a zero end-of-entries marker it proves every remaining byte is zero before a
|
|
333
|
+
POSIX opener durably truncates copied Windows padding; on timeout, non-zero data, or a failed sync it
|
|
334
|
+
refuses the writable open rather than allowing a later `O_APPEND` write to become invisible past the
|
|
335
|
+
marker.
|
|
336
|
+
|
|
337
|
+
Discovery only registers segments whose filename is one the writer could have produced —
|
|
338
|
+
`<sequence>.txnlog`, a positive decimal with no padding or suffix. A `.txnlog` that is not, or a
|
|
339
|
+
second file claiming a sequence already registered, is ignored and counted as a segment discovery
|
|
340
|
+
could not place, which refuses an opted-in open and names the file. This matters because the
|
|
341
|
+
filename _is_ the segment's identity: a `1 copy.txnlog` next to `1.txnlog` used to take its place in
|
|
342
|
+
the registry, so the real segment was never read and the floor was raised without its keys. **An
|
|
343
|
+
ordinary open is affected too** — such a file used to be exposed as a real segment and could be
|
|
344
|
+
recovered or appended relative to; it is now ignored, so a directory holding manually renamed or
|
|
345
|
+
copied segments starts from the canonical sequence state instead. Rename only after establishing
|
|
346
|
+
which file is canonical. Note that a canonical name is a syntax check, not provenance: a valid
|
|
347
|
+
`9.txnlog` copied in from another node passes it, so directory integrity and naming the log this
|
|
348
|
+
process originates stay the caller's responsibility.
|
|
349
|
+
|
|
350
|
+
A scan that succeeds after using most of its budget warns, so the growth that would otherwise turn
|
|
351
|
+
into a refusal on some later restart is visible before it does.
|
|
352
|
+
|
|
353
|
+
A `timestampFloorLog` naming a log the database does not have warns — including on a database with no
|
|
354
|
+
logs at all, which is where a misspelled name is most likely: a newly named locally originated log has
|
|
355
|
+
no existing keys to seed. Callers that need restart-safe uniqueness must provision and name their
|
|
356
|
+
local log consistently before its first write; a missing name can otherwise be a configuration mistake
|
|
357
|
+
that leaves no key to scan.
|
|
358
|
+
|
|
359
|
+
After that warning, the first use can create a genuinely fresh or empty named directory. It cannot
|
|
360
|
+
adopt a nonempty directory that reappears before that first use: its keys were not covered by the
|
|
361
|
+
missing-log seed, and a transaction's timestamp already exists before `txn.useLog()` could inspect it.
|
|
362
|
+
That use — and a read-only or secondary re-open that discovers the directory — refuses. Close every
|
|
363
|
+
handle for the path and reopen with `timestampFloorLog` so the restored log is scanned before use.
|
|
364
|
+
|
|
365
|
+
A floor raised more than a second above the wall clock also warns. Only a backward clock step puts
|
|
366
|
+
durable keys ahead of now, and from that point the process issues timestamps off the floor rather than
|
|
367
|
+
tracking wall time — and each restart re-seeds from the keys it just wrote — so it stays ahead until
|
|
368
|
+
the clock catches up.
|
|
369
|
+
|
|
370
|
+
The seed belongs to the physical database path, not to one handle. The first open of a path resolves
|
|
371
|
+
it; a later open of the same path — including a read-only or secondary open, which gets its own
|
|
372
|
+
descriptor — finds it already resolved and does not walk the log again, which also means it never
|
|
373
|
+
scans a store the first handle may be appending to. An open that names a _different_ log than the one
|
|
374
|
+
already resolved for the path, or names one when the path was opened without it, is rejected rather
|
|
375
|
+
than appearing to work.
|
|
376
|
+
|
|
239
377
|
### Sequential Read
|
|
240
378
|
|
|
241
379
|
1. Parse file header to get version
|
|
@@ -90,10 +90,10 @@ work.
|
|
|
90
90
|
Keep `setTimestamp()` as the adoption API, reject invalid timestamp values, and reject changes once
|
|
91
91
|
the transaction is no longer pending or has staged a write or log entry. Preserve the existing
|
|
92
92
|
retry behavior: a reset with no durable batch clears staged writes and may adopt again, while
|
|
93
|
-
`committedPosition` freezes a timestamp whose batch already landed. Remove the second-word read API
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
93
|
+
`committedPosition` freezes a timestamp whose batch already landed. Remove the second-word read API
|
|
94
|
+
and flag from this PR; they belong to Harper's record codec. The clock-floor-at-open implementation
|
|
95
|
+
is retained as an explicit, caller-named option: it scans only the locally originated log, and
|
|
96
|
+
fails the opted-in open when it cannot prove a complete, plausible floor. The existing
|
|
97
97
|
`latestTimestamp` behavior used to populate new transaction-log segment headers remains unchanged.
|
|
98
98
|
|
|
99
99
|
Document what is true about log keys today, which the issue also asked for: entries are appended in
|
|
@@ -117,7 +117,69 @@ and making every read a state transition still could not prove that a producer's
|
|
|
117
117
|
the transaction timestamp. The enforced boundary therefore remains the first native staging event;
|
|
118
118
|
producer ordering is documented explicitly.
|
|
119
119
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
120
|
+
The clock-floor scan's far-future check stays in the open path rather than changing appends: an
|
|
121
|
+
implausible key refuses the explicitly opted-in open, while normal append and segment-header
|
|
122
|
+
behavior remains unchanged. A floor based on unnamed logs remains out of scope because it cannot
|
|
123
|
+
distinguish locally issued keys from adopted peer keys.
|
|
124
|
+
|
|
125
|
+
## Clock-floor fail-closed amendment
|
|
126
|
+
|
|
127
|
+
The task owner explicitly selected restart-safe key uniqueness over availability when
|
|
128
|
+
`timestampFloorLog` is configured. This changes that option's failure behavior only; omitting the
|
|
129
|
+
option preserves the existing process-local clock behavior.
|
|
130
|
+
|
|
131
|
+
### Approaches considered
|
|
132
|
+
|
|
133
|
+
#### Do less
|
|
134
|
+
|
|
135
|
+
Warn and raise the floor from the readable prefix. Rejected: an unreadable segment, framing break,
|
|
136
|
+
or exhausted budget can hide a durable key, leaving the next process able to issue the same key.
|
|
137
|
+
|
|
138
|
+
#### Different layer
|
|
139
|
+
|
|
140
|
+
Make `getMonotonicTimestamp()` persist its own global floor. Rejected: the timestamp source cannot
|
|
141
|
+
know which transaction-log keys this process originated, and a global persisted value would add a
|
|
142
|
+
separate durability protocol without proving it covers the caller's log.
|
|
143
|
+
|
|
144
|
+
#### Deeper cause
|
|
145
|
+
|
|
146
|
+
Require every transaction-log append to use an externally durable counter. Rejected: that changes
|
|
147
|
+
the transaction timestamp contract and replication/replay adoption path, far beyond the opt-in
|
|
148
|
+
startup safety check requested here.
|
|
149
|
+
|
|
150
|
+
#### Chosen
|
|
151
|
+
|
|
152
|
+
After recovery, scan every segment in the caller-named local log under the bounded deadline. Reject
|
|
153
|
+
the open before raising the process-wide floor if the scan is incomplete or sees an implausibly
|
|
154
|
+
future key. A read-only torn tail shorter than an entry header remains valid: no frame can begin in it, so its
|
|
155
|
+
contiguous prefix provably covers every durable entry. A longer one is refused, because nothing
|
|
156
|
+
distinguishes a partial append from corruption hiding a complete higher-keyed frame. The named-log-missing warning remains non-fatal so a fresh local log can be named
|
|
157
|
+
before its first append.
|
|
158
|
+
|
|
159
|
+
### First-use guard for an absent log
|
|
160
|
+
|
|
161
|
+
The warning remains usable for a fresh log, but it cannot authorize adopting old data later in the
|
|
162
|
+
same process. `SeedTimestampFloor` records that the named log was absent. Before a writable first
|
|
163
|
+
use creates a store, the registry rejects a nonempty directory with that name; it also rejects a
|
|
164
|
+
later read-only or secondary descriptor that discovers one. A transaction obtains its timestamp
|
|
165
|
+
before `txn.useLog()`, so rescanning at lazy resolution could not repair an already-issued key.
|
|
166
|
+
Closing every handle removes that per-path state; the next opted-in open discovers and scans the
|
|
167
|
+
restored log before a transaction can be constructed. A process-created fresh log clears the guard,
|
|
168
|
+
so normal destroy-and-recreate log maintenance remains available.
|
|
169
|
+
|
|
170
|
+
### Verification route
|
|
171
|
+
|
|
172
|
+
The forked integration fixture writes a named log in one process and opens it in another. It verifies
|
|
173
|
+
successful seeding, then independently corrupts a segment, exhausts the deadline, skips discovery,
|
|
174
|
+
creates a mid-file framing break, writes an implausible key, and restores a named directory after an
|
|
175
|
+
absent seed; each unsafe case must reject the opted-in open. Native tests cover scan classification
|
|
176
|
+
and the repository build/check gates validate the binding and TypeScript surface.
|
|
177
|
+
|
|
178
|
+
### Late opt-in and the scan budget
|
|
179
|
+
|
|
180
|
+
A handle that names `timestampFloorLog` after the path was opened without it is rejected rather than
|
|
181
|
+
warned: the descriptor cannot rerun its one-time seed, and accepting the request would suggest
|
|
182
|
+
restart-safe uniqueness that is not in force. The scan budget stays bounded by default. An unbounded
|
|
183
|
+
scan can hold serialized opens indefinitely; the bounded scan instead rejects an opted-in open when
|
|
184
|
+
it cannot prove the floor, and an operator who needs more time raises
|
|
185
|
+
`ROCKSDB_JS_TIMESTAMP_FLOOR_SCAN_MS`.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@harperfast/rocksdb-js",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.12.0",
|
|
4
4
|
"description": "RocksDB binding for Node.js",
|
|
5
5
|
"homepage": "https://github.com/HarperFast/rocksdb-js",
|
|
6
6
|
"bugs": [
|
|
@@ -38,32 +38,32 @@
|
|
|
38
38
|
},
|
|
39
39
|
"devDependencies": {
|
|
40
40
|
"@rollup/plugin-replace": "6.0.3",
|
|
41
|
-
"@types/node": "26.6.
|
|
41
|
+
"@types/node": "26.6.5",
|
|
42
42
|
"@vitest/coverage-v8": "4.1.11",
|
|
43
43
|
"cross-env": "10.1.0",
|
|
44
|
-
"dotenv": "
|
|
45
|
-
"lefthook": "2.1
|
|
46
|
-
"lmdb": "3.5.
|
|
47
|
-
"node-gyp": "13.0
|
|
48
|
-
"oxfmt": "0.
|
|
49
|
-
"oxlint": "1.
|
|
44
|
+
"dotenv": "18.0.7",
|
|
45
|
+
"lefthook": "2.2.1",
|
|
46
|
+
"lmdb": "3.5.7",
|
|
47
|
+
"node-gyp": "13.1.0",
|
|
48
|
+
"oxfmt": "0.72.0",
|
|
49
|
+
"oxlint": "1.87.0",
|
|
50
50
|
"prebuildify": "6.0.1",
|
|
51
51
|
"semver": "7.8.5",
|
|
52
52
|
"tar": "7.5.22",
|
|
53
|
-
"tsdown": "0.23.
|
|
53
|
+
"tsdown": "0.23.1",
|
|
54
54
|
"tslib": "2.8.1",
|
|
55
55
|
"typescript": "7.0.2",
|
|
56
56
|
"vitest": "4.1.11"
|
|
57
57
|
},
|
|
58
58
|
"optionalDependencies": {
|
|
59
|
-
"@harperfast/rocksdb-js-darwin-arm64": "2.
|
|
60
|
-
"@harperfast/rocksdb-js-darwin-x64": "2.
|
|
61
|
-
"@harperfast/rocksdb-js-linux-arm64-glibc": "2.
|
|
62
|
-
"@harperfast/rocksdb-js-linux-arm64-musl": "2.
|
|
63
|
-
"@harperfast/rocksdb-js-linux-x64-glibc": "2.
|
|
64
|
-
"@harperfast/rocksdb-js-linux-x64-musl": "2.
|
|
65
|
-
"@harperfast/rocksdb-js-win32-arm64": "2.
|
|
66
|
-
"@harperfast/rocksdb-js-win32-x64": "2.
|
|
59
|
+
"@harperfast/rocksdb-js-darwin-arm64": "2.12.0",
|
|
60
|
+
"@harperfast/rocksdb-js-darwin-x64": "2.12.0",
|
|
61
|
+
"@harperfast/rocksdb-js-linux-arm64-glibc": "2.12.0",
|
|
62
|
+
"@harperfast/rocksdb-js-linux-arm64-musl": "2.12.0",
|
|
63
|
+
"@harperfast/rocksdb-js-linux-x64-glibc": "2.12.0",
|
|
64
|
+
"@harperfast/rocksdb-js-linux-x64-musl": "2.12.0",
|
|
65
|
+
"@harperfast/rocksdb-js-win32-arm64": "2.12.0",
|
|
66
|
+
"@harperfast/rocksdb-js-win32-x64": "2.12.0"
|
|
67
67
|
},
|
|
68
68
|
"engines": {
|
|
69
69
|
"node": "^22.18.0 || >=24.0.0"
|
|
@@ -81,7 +81,8 @@
|
|
|
81
81
|
"build:binding:debug": "node-gyp build --coverage --debug --verbose",
|
|
82
82
|
"build:bundle": "node scripts/clean-dist.mjs && tsdown -c tsdown.config.ts",
|
|
83
83
|
"build:bundle:minify": "cross-env MINIFY=1 pnpm build:bundle",
|
|
84
|
-
"check": "pnpm
|
|
84
|
+
"check": "pnpm check:code && pnpm fmt:check",
|
|
85
|
+
"check:code": "pnpm type-check && pnpm lint",
|
|
85
86
|
"clean": "node-gyp clean",
|
|
86
87
|
"coverage": "node scripts/coverage/main.ts",
|
|
87
88
|
"coverage:native": "node scripts/native-test/coverage.mjs",
|