@harperfast/rocksdb-js 2.7.1 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +287 -26
- package/dist/index.cjs +460 -12
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +440 -12
- package/dist/index.d.mts +440 -12
- package/dist/index.mjs +458 -13
- package/dist/index.mjs.map +1 -1
- package/docs/stats.md +309 -268
- package/package.json +19 -20
package/dist/index.mjs
CHANGED
|
@@ -55,6 +55,16 @@ function locateBinding() {
|
|
|
55
55
|
throw new Error("Unable to locate rocksdb-js native binding");
|
|
56
56
|
}
|
|
57
57
|
const binding = req(locateBinding());
|
|
58
|
+
/**
|
|
59
|
+
* The native `BackgroundError` constructor (a real `Error` subclass). Exported
|
|
60
|
+
* as both a value — for `err instanceof BackgroundError` — and, via declaration
|
|
61
|
+
* merging with the interface above, a type. Instances are produced by the
|
|
62
|
+
* `'error'` event and `db.getLastError()`; consumers rarely construct their own.
|
|
63
|
+
* The `details` param is required and typed to the instance fields so a bare
|
|
64
|
+
* `new BackgroundError()` (which would omit `severity` / `writesDisabled` / …)
|
|
65
|
+
* is a type error.
|
|
66
|
+
*/
|
|
67
|
+
const BackgroundError = binding.BackgroundError;
|
|
58
68
|
const config = binding.config;
|
|
59
69
|
const FRESH_VERSION_FLAG$1 = binding.constants.FRESH_VERSION_FLAG;
|
|
60
70
|
const addGlobalListener = binding.addListener;
|
|
@@ -446,6 +456,150 @@ const backups = {
|
|
|
446
456
|
}
|
|
447
457
|
};
|
|
448
458
|
|
|
459
|
+
//#endregion
|
|
460
|
+
//#region src/count-estimator.ts
|
|
461
|
+
/**
|
|
462
|
+
* Below this many traversed entries the observed density is too noisy to
|
|
463
|
+
* calibrate with, so `estimate()` returns the raw statistical estimate.
|
|
464
|
+
*/
|
|
465
|
+
const CALIBRATION_MIN_TRAVERSED = 16;
|
|
466
|
+
const CALIBRATION_MIN_CONFIDENCE = .8;
|
|
467
|
+
/**
|
|
468
|
+
* Bounds on how far the observed-vs-estimated ratio may scale the remainder;
|
|
469
|
+
* the statistical estimate can be arbitrarily wrong at data-block granularity
|
|
470
|
+
* and an unclamped ratio would let one bad sample dominate.
|
|
471
|
+
*/
|
|
472
|
+
const CALIBRATION_MAX = 8;
|
|
473
|
+
/**
|
|
474
|
+
* Progressively refines a range key-count estimate as an iterator traverses
|
|
475
|
+
* the range. The estimate starts as the pure statistical estimate
|
|
476
|
+
* (`estimateCount`) and converges toward the exact count: the traversed
|
|
477
|
+
* portion is exact, and the statistical estimate of the untraversed remainder
|
|
478
|
+
* is calibrated by the observed ratio of actual-to-estimated entries over the
|
|
479
|
+
* portion already traversed.
|
|
480
|
+
*
|
|
481
|
+
* The estimator never touches the iterator itself — the caller reports
|
|
482
|
+
* progress with `advance(lastKey, count)` whenever it wants a checkpoint
|
|
483
|
+
* (e.g. once per page), then reads `estimate()`. The caller owns the
|
|
484
|
+
* progress contract: cursors must move monotonically through the range and
|
|
485
|
+
* each entry must be reported once (a re-reported page inflates the count
|
|
486
|
+
* undetectably). When traversal completes, call `finish()` so `estimate()`
|
|
487
|
+
* returns the exact total instead of adding a block-granular remainder.
|
|
488
|
+
*
|
|
489
|
+
* Each `estimate()` checkpoint queries RocksDB statistics for the two range
|
|
490
|
+
* segments (cost scales with the SSTs overlapping them, not with key count);
|
|
491
|
+
* results are memoized per checkpoint, so repeated reads between `advance()`
|
|
492
|
+
* calls are free.
|
|
493
|
+
*/
|
|
494
|
+
var CountEstimator = class {
|
|
495
|
+
#store;
|
|
496
|
+
#start;
|
|
497
|
+
#end;
|
|
498
|
+
#exclusiveStart;
|
|
499
|
+
#inclusiveEnd;
|
|
500
|
+
#reverse;
|
|
501
|
+
#cursor;
|
|
502
|
+
#traversed = 0;
|
|
503
|
+
#finished = false;
|
|
504
|
+
#memoized;
|
|
505
|
+
constructor(store, options) {
|
|
506
|
+
this.#store = store;
|
|
507
|
+
this.#reverse = options?.reverse ?? false;
|
|
508
|
+
this.#start = this.#reverse ? options?.end : options?.start;
|
|
509
|
+
this.#end = this.#reverse ? options?.start : options?.end;
|
|
510
|
+
this.#exclusiveStart = options?.exclusiveStart ?? this.#reverse;
|
|
511
|
+
this.#inclusiveEnd = options?.inclusiveEnd ?? this.#reverse;
|
|
512
|
+
}
|
|
513
|
+
/**
|
|
514
|
+
* The number of entries reported traversed so far.
|
|
515
|
+
*/
|
|
516
|
+
get traversed() {
|
|
517
|
+
return this.#traversed;
|
|
518
|
+
}
|
|
519
|
+
/**
|
|
520
|
+
* Records that iteration has advanced through `count` more entries, ending
|
|
521
|
+
* at `lastKey`. Pass `count: 0` when an empty page has no last key.
|
|
522
|
+
*/
|
|
523
|
+
advance(lastKey, count = 1) {
|
|
524
|
+
if (lastKey === void 0) {
|
|
525
|
+
if (count !== 0) throw new Error("CountEstimator.advance requires lastKey when count is nonzero");
|
|
526
|
+
return;
|
|
527
|
+
}
|
|
528
|
+
this.#cursor = lastKey;
|
|
529
|
+
this.#traversed += count;
|
|
530
|
+
this.#memoized = void 0;
|
|
531
|
+
}
|
|
532
|
+
/**
|
|
533
|
+
* Marks traversal of the range as complete: `estimate()` becomes the exact
|
|
534
|
+
* traversed count.
|
|
535
|
+
*/
|
|
536
|
+
finish() {
|
|
537
|
+
this.#finished = true;
|
|
538
|
+
}
|
|
539
|
+
/**
|
|
540
|
+
* Estimates the total number of entries in the full range: the exact
|
|
541
|
+
* traversed count plus a calibrated statistical estimate of the remainder.
|
|
542
|
+
* `confidence` is the exactness-weighted blend of the traversed portion
|
|
543
|
+
* (exact) and the remainder's statistical confidence, so it approaches 1
|
|
544
|
+
* as the exact portion grows (though a checkpoint can decrease when
|
|
545
|
+
* calibration makes a large correction), and is exactly 1 after `finish()`.
|
|
546
|
+
*/
|
|
547
|
+
estimate() {
|
|
548
|
+
if (this.#finished) return {
|
|
549
|
+
count: this.#traversed,
|
|
550
|
+
confidence: 1
|
|
551
|
+
};
|
|
552
|
+
if (this.#memoized !== void 0) return { ...this.#memoized };
|
|
553
|
+
if (this.#cursor === void 0) {
|
|
554
|
+
this.#memoized = this.#store.estimateCount({
|
|
555
|
+
start: this.#start,
|
|
556
|
+
end: this.#end,
|
|
557
|
+
exclusiveStart: this.#exclusiveStart,
|
|
558
|
+
inclusiveEnd: this.#inclusiveEnd
|
|
559
|
+
});
|
|
560
|
+
return { ...this.#memoized };
|
|
561
|
+
}
|
|
562
|
+
const traversedRange = this.#reverse ? {
|
|
563
|
+
start: this.#cursor,
|
|
564
|
+
end: this.#end,
|
|
565
|
+
inclusiveEnd: this.#inclusiveEnd
|
|
566
|
+
} : {
|
|
567
|
+
start: this.#start,
|
|
568
|
+
exclusiveStart: this.#exclusiveStart,
|
|
569
|
+
end: this.#cursor,
|
|
570
|
+
inclusiveEnd: true
|
|
571
|
+
};
|
|
572
|
+
const remainingRange = this.#reverse ? {
|
|
573
|
+
start: this.#start,
|
|
574
|
+
exclusiveStart: this.#exclusiveStart,
|
|
575
|
+
end: this.#cursor
|
|
576
|
+
} : {
|
|
577
|
+
start: this.#cursor,
|
|
578
|
+
exclusiveStart: true,
|
|
579
|
+
end: this.#end,
|
|
580
|
+
inclusiveEnd: this.#inclusiveEnd
|
|
581
|
+
};
|
|
582
|
+
const remainingEstimate = this.#store.estimateCount(remainingRange);
|
|
583
|
+
let remaining = remainingEstimate.count;
|
|
584
|
+
let calibrationConfidence = 1;
|
|
585
|
+
if (this.#traversed >= CALIBRATION_MIN_TRAVERSED) {
|
|
586
|
+
const traversedEstimate = this.#store.estimateCount(traversedRange);
|
|
587
|
+
if (traversedEstimate.count >= CALIBRATION_MIN_TRAVERSED && traversedEstimate.confidence >= CALIBRATION_MIN_CONFIDENCE) {
|
|
588
|
+
const calibration = Math.min(CALIBRATION_MAX, Math.max(1 / CALIBRATION_MAX, this.#traversed / traversedEstimate.count));
|
|
589
|
+
remaining *= calibration;
|
|
590
|
+
calibrationConfidence = Math.min(calibration, 1 / calibration);
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
const count = Math.round(this.#traversed + remaining);
|
|
594
|
+
const confidence = count > 0 ? Math.min(.999, (this.#traversed + calibrationConfidence * remainingEstimate.confidence * remaining) / count) : Math.min(.999, calibrationConfidence * remainingEstimate.confidence);
|
|
595
|
+
this.#memoized = {
|
|
596
|
+
count,
|
|
597
|
+
confidence
|
|
598
|
+
};
|
|
599
|
+
return { ...this.#memoized };
|
|
600
|
+
}
|
|
601
|
+
};
|
|
602
|
+
|
|
449
603
|
//#endregion
|
|
450
604
|
//#region src/util.ts
|
|
451
605
|
/**
|
|
@@ -1404,7 +1558,9 @@ var Store = class {
|
|
|
1404
1558
|
pessimistic;
|
|
1405
1559
|
/**
|
|
1406
1560
|
* Whether the database is open in readonly mode. When `true`, write
|
|
1407
|
-
* operations will throw an error with code `ERR_DATABASE_READONLY
|
|
1561
|
+
* operations will throw an error with code `ERR_DATABASE_READONLY`, except
|
|
1562
|
+
* for `flush()`/`flushSync()`/`compact()`/`compactSync()`, which succeed as
|
|
1563
|
+
* no-ops.
|
|
1408
1564
|
*/
|
|
1409
1565
|
readOnly;
|
|
1410
1566
|
/**
|
|
@@ -1431,8 +1587,8 @@ var Store = class {
|
|
|
1431
1587
|
*/
|
|
1432
1588
|
transactionLogMaxAgeThreshold;
|
|
1433
1589
|
/**
|
|
1434
|
-
* The maximum size of a transaction log before
|
|
1435
|
-
*
|
|
1590
|
+
* The target maximum size of a transaction log before rotation. Transactions
|
|
1591
|
+
* are never split across files, so one transaction may exceed this target.
|
|
1436
1592
|
*/
|
|
1437
1593
|
transactionLogMaxSize;
|
|
1438
1594
|
/**
|
|
@@ -1641,6 +1797,24 @@ var Store = class {
|
|
|
1641
1797
|
}
|
|
1642
1798
|
return result;
|
|
1643
1799
|
}
|
|
1800
|
+
/**
|
|
1801
|
+
* Estimates the number of keys in a range from RocksDB statistics
|
|
1802
|
+
* (memtable stats + approximate SST sizes with range-local entry density)
|
|
1803
|
+
* without iterating. Estimates always reflect committed state, so there is
|
|
1804
|
+
* no transactional variant.
|
|
1805
|
+
*/
|
|
1806
|
+
estimateCount(options) {
|
|
1807
|
+
let startBuffer;
|
|
1808
|
+
let endBuffer;
|
|
1809
|
+
const reverse = options?.reverse ?? false;
|
|
1810
|
+
const start = reverse ? options?.end : options?.start;
|
|
1811
|
+
const end = reverse ? options?.start : options?.end;
|
|
1812
|
+
const exclusiveStart = options?.exclusiveStart ?? reverse;
|
|
1813
|
+
const inclusiveEnd = options?.inclusiveEnd ?? reverse;
|
|
1814
|
+
if (start !== void 0) startBuffer = copyEncodedKey(this.encodeKey(start), exclusiveStart);
|
|
1815
|
+
if (end !== void 0) endBuffer = copyEncodedKey(this.encodeKey(end), inclusiveEnd);
|
|
1816
|
+
return this.db.estimateCount(startBuffer, endBuffer);
|
|
1817
|
+
}
|
|
1644
1818
|
getCount(context, options) {
|
|
1645
1819
|
options = { ...options };
|
|
1646
1820
|
if (options?.start !== void 0) {
|
|
@@ -1854,7 +2028,7 @@ var Store = class {
|
|
|
1854
2028
|
useLog(context, name) {
|
|
1855
2029
|
if (typeof name !== "string" && typeof name !== "number") throw new TypeError("Log name must be a string or number");
|
|
1856
2030
|
if (typeof name === "string" && /[\t\n\r\\/]/.test(name)) throw new Error(`Invalid transaction log name "${name}"`);
|
|
1857
|
-
return context.useLog(String(name));
|
|
2031
|
+
return context.useLog(String(name), this.db);
|
|
1858
2032
|
}
|
|
1859
2033
|
/**
|
|
1860
2034
|
* Checks the process-global verification table for a fresh version match
|
|
@@ -1888,6 +2062,18 @@ var Store = class {
|
|
|
1888
2062
|
}
|
|
1889
2063
|
};
|
|
1890
2064
|
/**
|
|
2065
|
+
* Copies an encoded key out of the shared key buffer (which the next
|
|
2066
|
+
* `encodeKey` call would clobber), optionally appending a zero byte to
|
|
2067
|
+
* produce the key's bytewise successor.
|
|
2068
|
+
*/
|
|
2069
|
+
function copyEncodedKey(encoded, appendSuccessorByte) {
|
|
2070
|
+
const length = encoded.end - encoded.start;
|
|
2071
|
+
const copy = Buffer.allocUnsafe(length + (appendSuccessorByte ? 1 : 0));
|
|
2072
|
+
copy.set(encoded.subarray(encoded.start, encoded.end));
|
|
2073
|
+
if (appendSuccessorByte) copy[length] = 0;
|
|
2074
|
+
return copy;
|
|
2075
|
+
}
|
|
2076
|
+
/**
|
|
1891
2077
|
* Ensure that they key has been copied into our shared buffer, and return the ending position
|
|
1892
2078
|
* @param keyBuffer
|
|
1893
2079
|
*/
|
|
@@ -2084,6 +2270,16 @@ var Transaction = class extends DBI {
|
|
|
2084
2270
|
|
|
2085
2271
|
//#endregion
|
|
2086
2272
|
//#region src/database.ts
|
|
2273
|
+
const BACKGROUND_ERROR_SEVERITY_NAMES = [
|
|
2274
|
+
"none",
|
|
2275
|
+
"soft",
|
|
2276
|
+
"hard",
|
|
2277
|
+
"fatal",
|
|
2278
|
+
"unrecoverable"
|
|
2279
|
+
];
|
|
2280
|
+
function backgroundErrorSeverityName(severity) {
|
|
2281
|
+
return BACKGROUND_ERROR_SEVERITY_NAMES[severity] ?? "unknown";
|
|
2282
|
+
}
|
|
2087
2283
|
/**
|
|
2088
2284
|
* The main class for interacting with a RocksDB database.
|
|
2089
2285
|
*
|
|
@@ -2155,6 +2351,9 @@ var RocksDatabase = class RocksDatabase extends DBI {
|
|
|
2155
2351
|
* Compacts the entire key range of the database asynchronously.
|
|
2156
2352
|
* This triggers manual compaction which removes tombstones and reclaims space.
|
|
2157
2353
|
*
|
|
2354
|
+
* On a read-only database this is a no-op: arguments are still validated, but
|
|
2355
|
+
* the returned promise resolves without compacting rather than rejecting.
|
|
2356
|
+
*
|
|
2158
2357
|
* @example
|
|
2159
2358
|
* ```typescript
|
|
2160
2359
|
* const db = RocksDatabase.open('/path/to/database');
|
|
@@ -2206,6 +2405,9 @@ var RocksDatabase = class RocksDatabase extends DBI {
|
|
|
2206
2405
|
* Compacts the entire key range of the database synchronously.
|
|
2207
2406
|
* This triggers manual compaction which removes tombstones and reclaims space.
|
|
2208
2407
|
*
|
|
2408
|
+
* On a read-only database this validates its arguments and then returns
|
|
2409
|
+
* without compacting, rather than throwing.
|
|
2410
|
+
*
|
|
2209
2411
|
* @example
|
|
2210
2412
|
* ```typescript
|
|
2211
2413
|
* const db = RocksDatabase.open('/path/to/database');
|
|
@@ -2339,18 +2541,107 @@ var RocksDatabase = class RocksDatabase extends DBI {
|
|
|
2339
2541
|
/**
|
|
2340
2542
|
* Flushes the underlying database by performing a commit or clearing any buffered operations.
|
|
2341
2543
|
*
|
|
2544
|
+
* On a read-only database this is a no-op: the options bag is still validated, but the
|
|
2545
|
+
* returned promise resolves without flushing rather than rejecting.
|
|
2546
|
+
*
|
|
2547
|
+
* @param options - Flush options; see {@link FlushOptions.allowWriteStall} before relying on
|
|
2548
|
+
* this resolving promptly under write pressure.
|
|
2342
2549
|
* @return {void} Does not return a value.
|
|
2343
2550
|
*/
|
|
2344
|
-
flush() {
|
|
2345
|
-
return new Promise((resolve, reject) => this.store.db.flush(resolve, reject));
|
|
2551
|
+
flush(options) {
|
|
2552
|
+
return new Promise((resolve, reject) => this.store.db.flush(resolve, reject, options));
|
|
2346
2553
|
}
|
|
2347
2554
|
/**
|
|
2348
2555
|
* Synchronously flushes the underlying database by performing a commit or clearing any buffered operations.
|
|
2349
2556
|
*
|
|
2557
|
+
* The default `allowWriteStall: false` wait is taken on the JS thread here, so it freezes the
|
|
2558
|
+
* event loop rather than parking a libuv worker — a larger blast radius than {@link flush}, not
|
|
2559
|
+
* a smaller one.
|
|
2560
|
+
*
|
|
2561
|
+
* On a read-only database this validates its options and then returns without flushing, rather
|
|
2562
|
+
* than throwing.
|
|
2563
|
+
*
|
|
2564
|
+
* @param options - Flush options; see {@link FlushOptions.allowWriteStall}.
|
|
2350
2565
|
* @return {void} Does not return a value.
|
|
2351
2566
|
*/
|
|
2352
|
-
flushSync() {
|
|
2353
|
-
return this.store.db.flushSync();
|
|
2567
|
+
flushSync(options) {
|
|
2568
|
+
return this.store.db.flushSync(options);
|
|
2569
|
+
}
|
|
2570
|
+
/**
|
|
2571
|
+
* Returns the most recent {@link BackgroundError} observed on this database, or
|
|
2572
|
+
* `null` when none has occurred. The `'error'` event is the push counterpart;
|
|
2573
|
+
* this is the pull equivalent for on-demand checks (e.g. a health probe) and
|
|
2574
|
+
* for catching an error that fired after open but before a listener was
|
|
2575
|
+
* attached. (A background error that occurs *during* the initial open is a
|
|
2576
|
+
* narrow startup window not yet captured, addressed separately.)
|
|
2577
|
+
*
|
|
2578
|
+
* Purely historical — it is not cleared by a successful {@link RocksDatabase.resume}.
|
|
2579
|
+
* When the returned error's `writesDisabled` is `true`, writes are stopped
|
|
2580
|
+
* until recovery.
|
|
2581
|
+
*
|
|
2582
|
+
* @example
|
|
2583
|
+
* ```typescript
|
|
2584
|
+
* const err = db.getLastError();
|
|
2585
|
+
* if (err?.writesDisabled) {
|
|
2586
|
+
* // ...free disk space...
|
|
2587
|
+
* db.resume();
|
|
2588
|
+
* }
|
|
2589
|
+
* ```
|
|
2590
|
+
*/
|
|
2591
|
+
getLastError() {
|
|
2592
|
+
return this.store.db.getLastError();
|
|
2593
|
+
}
|
|
2594
|
+
/**
|
|
2595
|
+
* Sets or clears the last background error, mirroring the Win32
|
|
2596
|
+
* `SetLastError`/`GetLastError` pair. Pass `null` (or no argument) to **reset**
|
|
2597
|
+
* it — {@link RocksDatabase.getLastError} then returns `null` and no event
|
|
2598
|
+
* fires; this is the intended way to clear the error after handling or
|
|
2599
|
+
* recovering it (e.g. after {@link RocksDatabase.resume}). Pass an object to
|
|
2600
|
+
* set it: it is stored and the `'error'` event fires with the reconstructed
|
|
2601
|
+
* {@link BackgroundError}. `type` defaults to `'background'`.
|
|
2602
|
+
*
|
|
2603
|
+
* @example
|
|
2604
|
+
* ```typescript
|
|
2605
|
+
* db.resume();
|
|
2606
|
+
* db.setLastError(null); // recovered — clear the historical error
|
|
2607
|
+
* ```
|
|
2608
|
+
*/
|
|
2609
|
+
setLastError(error) {
|
|
2610
|
+
if (error == null) return this.store.db.setLastError(null);
|
|
2611
|
+
const severity = error.severity ?? 0;
|
|
2612
|
+
return this.store.db.setLastError({
|
|
2613
|
+
...error,
|
|
2614
|
+
type: error.type ?? "background",
|
|
2615
|
+
severity,
|
|
2616
|
+
severityName: error.severityName ?? backgroundErrorSeverityName(severity),
|
|
2617
|
+
writesDisabled: error.writesDisabled ?? severity >= 2
|
|
2618
|
+
});
|
|
2619
|
+
}
|
|
2620
|
+
/**
|
|
2621
|
+
* Attempts to recover the database from a background error by calling
|
|
2622
|
+
* RocksDB's `DB::Resume()`. When a write fails at the filesystem level (e.g. a
|
|
2623
|
+
* full disk), RocksDB stops accepting writes and this database emits an
|
|
2624
|
+
* {@link BackgroundError} via the `'error'` event. Call `resume()` after the
|
|
2625
|
+
* underlying condition has cleared — e.g. once disk space has been freed. On
|
|
2626
|
+
* success writes are accepted again; on failure (the condition has not
|
|
2627
|
+
* actually cleared) it throws and the database stays read-only. A no-op on a
|
|
2628
|
+
* healthy database.
|
|
2629
|
+
*
|
|
2630
|
+
* Runs synchronously and may briefly block, since recovery can re-flush
|
|
2631
|
+
* memtables.
|
|
2632
|
+
*
|
|
2633
|
+
* @example
|
|
2634
|
+
* ```typescript
|
|
2635
|
+
* db.on('error', (err) => {
|
|
2636
|
+
* if (err.writesDisabled) {
|
|
2637
|
+
* // ...free disk space...
|
|
2638
|
+
* db.resume();
|
|
2639
|
+
* }
|
|
2640
|
+
* });
|
|
2641
|
+
* ```
|
|
2642
|
+
*/
|
|
2643
|
+
resume() {
|
|
2644
|
+
return this.store.db.resume();
|
|
2354
2645
|
}
|
|
2355
2646
|
/**
|
|
2356
2647
|
* Gets a RocksDB database property as an integer.
|
|
@@ -2385,7 +2676,9 @@ var RocksDatabase = class RocksDatabase extends DBI {
|
|
|
2385
2676
|
return this.store.db.getDBProperty(propertyName);
|
|
2386
2677
|
}
|
|
2387
2678
|
/**
|
|
2388
|
-
* Retrieves the estimated number of keys in the database.
|
|
2679
|
+
* Retrieves the estimated number of keys in the database. This is an alias
|
|
2680
|
+
* for `db.estimateCount().count`; use `estimateCount()` for range support
|
|
2681
|
+
* and a confidence indicator.
|
|
2389
2682
|
*
|
|
2390
2683
|
* @example
|
|
2391
2684
|
* ```typescript
|
|
@@ -2398,6 +2691,58 @@ var RocksDatabase = class RocksDatabase extends DBI {
|
|
|
2398
2691
|
return this.getDBIntProperty("rocksdb.estimate-num-keys") ?? 0;
|
|
2399
2692
|
}
|
|
2400
2693
|
/**
|
|
2694
|
+
* Estimates the number of keys in the database, or within a key range,
|
|
2695
|
+
* returning `{ count, confidence }`. Unlike `getKeysCount()`, this never
|
|
2696
|
+
* iterates: the estimate is derived from RocksDB statistics (memtable
|
|
2697
|
+
* stats plus approximate SST sizes converted through the entry density of
|
|
2698
|
+
* the SSTs overlapping the range), so its cost scales with the number of
|
|
2699
|
+
* SSTs overlapping the range rather than the number of keys — though
|
|
2700
|
+
* reading table properties for cold files can do I/O through the table
|
|
2701
|
+
* cache, and a start-only range does the work of its complement below
|
|
2702
|
+
* `start`. Accuracy improves with range size. Resolution is bounded by SST
|
|
2703
|
+
* data-block granularity, so tiny ranges may over-report or report zero for
|
|
2704
|
+
* present keys; low `confidence` is the signal. Recently deleted or
|
|
2705
|
+
* overwritten entries may be counted until compaction. An inverted range
|
|
2706
|
+
* (`start` >= `end`) returns `{ count: 0, confidence: 1 }`.
|
|
2707
|
+
*
|
|
2708
|
+
* `confidence` is a heuristic 0–1 trust indicator (1 only when exact) —
|
|
2709
|
+
* see `CountEstimate`. Estimates always reflect committed state; writes
|
|
2710
|
+
* pending in a transaction are not included.
|
|
2711
|
+
*
|
|
2712
|
+
* @example
|
|
2713
|
+
* ```typescript
|
|
2714
|
+
* const db = RocksDatabase.open('/path/to/database');
|
|
2715
|
+
* const { count, confidence } = db.estimateCount({ start: 'a', end: 'z' });
|
|
2716
|
+
* ```
|
|
2717
|
+
*/
|
|
2718
|
+
estimateCount(options) {
|
|
2719
|
+
return this.store.estimateCount(options);
|
|
2720
|
+
}
|
|
2721
|
+
/**
|
|
2722
|
+
* Creates a `CountEstimator` for progressively refining a range count
|
|
2723
|
+
* estimate while iterating the range: report progress with
|
|
2724
|
+
* `advance(lastKey, count)` (e.g. once per page of results) and
|
|
2725
|
+
* `estimate()` returns the exact traversed count plus a calibrated
|
|
2726
|
+
* estimate of the remainder, converging toward the exact total. Call
|
|
2727
|
+
* `finish()` when traversal completes to make `estimate()` exact.
|
|
2728
|
+
*
|
|
2729
|
+
* @example
|
|
2730
|
+
* ```typescript
|
|
2731
|
+
* const estimator = db.createCountEstimator({ start: 'a', end: 'z' });
|
|
2732
|
+
* let lastKey;
|
|
2733
|
+
* let pageSize = 0;
|
|
2734
|
+
* for (const { key } of db.getRange({ start: 'a', end: 'z', limit: 25 })) {
|
|
2735
|
+
* lastKey = key;
|
|
2736
|
+
* pageSize++;
|
|
2737
|
+
* }
|
|
2738
|
+
* estimator.advance(lastKey, pageSize);
|
|
2739
|
+
* const { count, confidence } = estimator.estimate();
|
|
2740
|
+
* ```
|
|
2741
|
+
*/
|
|
2742
|
+
createCountEstimator(options) {
|
|
2743
|
+
return new CountEstimator(this.store, options);
|
|
2744
|
+
}
|
|
2745
|
+
/**
|
|
2401
2746
|
* Returns the current timestamp as a monotonically increasing timestamp in
|
|
2402
2747
|
* milliseconds represented as a decimal number.
|
|
2403
2748
|
*
|
|
@@ -2903,6 +3248,98 @@ const FLOAT_TO_UINT32 = /* @__PURE__ */ new Float64Array(1);
|
|
|
2903
3248
|
const UINT32_FROM_FLOAT = new Uint32Array(FLOAT_TO_UINT32.buffer);
|
|
2904
3249
|
const { TRANSACTION_LOG_FILE_HEADER_SIZE, TRANSACTION_LOG_ENTRY_HEADER_SIZE } = constants;
|
|
2905
3250
|
/**
|
|
3251
|
+
* Number of consecutive well-formed frames that, on their own, signal that real log data has
|
|
3252
|
+
* resumed after a framing break. Kept in step with `RESYNC_MIN_FRAMES` in
|
|
3253
|
+
* `src/binding/transaction_log/transaction_log_recovery.cpp`, which uses the same heuristic to
|
|
3254
|
+
* classify a break as mid-file corruption (valid entries follow) rather than a torn tail.
|
|
3255
|
+
*/
|
|
3256
|
+
const RESYNC_MIN_FRAMES = 8;
|
|
3257
|
+
/**
|
|
3258
|
+
* Thrown when an entry's framing is broken. `resyncPosition` is the offset where valid framing
|
|
3259
|
+
* resumes, when one was found: the frames after a mid-log break are intact and reachable, and
|
|
3260
|
+
* only the broken span between `position` and `resyncPosition` is unreadable. It is `undefined`
|
|
3261
|
+
* for a torn tail, where nothing follows the break.
|
|
3262
|
+
*
|
|
3263
|
+
* The iterator is left positioned at `resyncPosition` (or at end-of-log when there is none), so a
|
|
3264
|
+
* caller that chooses to recover the entries past the break calls `next()` again.
|
|
3265
|
+
*/
|
|
3266
|
+
var CorruptFrameError = class extends RangeError {
|
|
3267
|
+
/** Sequence number of the log file the break is in. */
|
|
3268
|
+
logId;
|
|
3269
|
+
/** Offset of the broken frame. */
|
|
3270
|
+
position;
|
|
3271
|
+
/** Offset where valid framing resumes, or `undefined` when nothing valid follows. */
|
|
3272
|
+
resyncPosition;
|
|
3273
|
+
/** Bytes between the break and the resume point that cannot be read. */
|
|
3274
|
+
unreadableBytes;
|
|
3275
|
+
constructor(message, logId, position, resyncPosition) {
|
|
3276
|
+
super(resyncPosition === void 0 ? message : `${message}; valid framing resumes at ${resyncPosition.toString(16)}, ${resyncPosition - position} byte(s) unreadable`);
|
|
3277
|
+
this.name = "CorruptFrameError";
|
|
3278
|
+
this.logId = logId;
|
|
3279
|
+
this.position = position;
|
|
3280
|
+
this.resyncPosition = resyncPosition;
|
|
3281
|
+
this.unreadableBytes = resyncPosition === void 0 ? 0 : resyncPosition - position;
|
|
3282
|
+
}
|
|
3283
|
+
};
|
|
3284
|
+
/**
|
|
3285
|
+
* Whether a complete, in-bounds frame begins at `pos`. The only sane bound on an entry's length is
|
|
3286
|
+
* the written extent: a single entry can legitimately exceed the rotation threshold (the first
|
|
3287
|
+
* entry written to a fresh file is always written in full). A zero timestamp is the
|
|
3288
|
+
* end-of-entries marker, not a frame.
|
|
3289
|
+
*/
|
|
3290
|
+
function frameFits(dataView, pos, dataEnd) {
|
|
3291
|
+
if (pos + TRANSACTION_LOG_ENTRY_HEADER_SIZE > dataEnd) return false;
|
|
3292
|
+
if (dataView.getFloat64(pos) === 0) return false;
|
|
3293
|
+
const length = dataView.getUint32(pos + 8);
|
|
3294
|
+
if (length === 0) return false;
|
|
3295
|
+
return pos + TRANSACTION_LOG_ENTRY_HEADER_SIZE + length <= dataEnd;
|
|
3296
|
+
}
|
|
3297
|
+
/**
|
|
3298
|
+
* Finds the offset in `[from, dataEnd)` where valid log framing resumes: a run of at least
|
|
3299
|
+
* `RESYNC_MIN_FRAMES` well-formed frames, or a run that lands exactly on `dataEnd`. Returns
|
|
3300
|
+
* `undefined` when nothing resumes (a torn tail).
|
|
3301
|
+
*
|
|
3302
|
+
* `dataEnd` must be the **written extent**, not the mapped capacity. The log's memory map is
|
|
3303
|
+
* pre-extended well past the data, and every offset inside that zero padding reads as an
|
|
3304
|
+
* end-of-entries marker — so scanning against the map would both lose the exact-end signal (the
|
|
3305
|
+
* chain can never land on a capacity that data does not reach) and, if a zero were accepted as a
|
|
3306
|
+
* terminator, let a chain "end" anywhere in megabytes of padding. Against the written extent the
|
|
3307
|
+
* end is a single offset, so a chain landing on it is ~1/2^32 to be coincidence.
|
|
3308
|
+
*
|
|
3309
|
+
* The JS counterpart of `validFramingResumes()` in `transaction_log_recovery.cpp`, which answers
|
|
3310
|
+
* the same question at open time but keeps only the yes/no, discarding the offset.
|
|
3311
|
+
*/
|
|
3312
|
+
function findResyncPosition(dataView, from, dataEnd) {
|
|
3313
|
+
const lastStart = dataEnd - TRANSACTION_LOG_ENTRY_HEADER_SIZE;
|
|
3314
|
+
for (let start = from; start <= lastStart; start++) {
|
|
3315
|
+
let pos = start;
|
|
3316
|
+
let frames = 0;
|
|
3317
|
+
while (frames < RESYNC_MIN_FRAMES && frameFits(dataView, pos, dataEnd)) {
|
|
3318
|
+
pos += TRANSACTION_LOG_ENTRY_HEADER_SIZE + dataView.getUint32(pos + 8);
|
|
3319
|
+
if (++frames >= RESYNC_MIN_FRAMES || pos === dataEnd) return start;
|
|
3320
|
+
}
|
|
3321
|
+
}
|
|
3322
|
+
}
|
|
3323
|
+
/**
|
|
3324
|
+
* Builds the report for a broken frame, with the offset iteration should continue from. Resolves
|
|
3325
|
+
* the written extent here rather than per frame: `getLogFileSize` crosses into native and takes
|
|
3326
|
+
* the store mutex, which a healthy read must never pay.
|
|
3327
|
+
*/
|
|
3328
|
+
function corruptFrame(message, transactionLog, dataView, logBuffer, position, limit, size, readUncommitted) {
|
|
3329
|
+
let dataEnd = limit;
|
|
3330
|
+
if (readUncommitted) try {
|
|
3331
|
+
const writtenExtent = transactionLog.getLogFileSize(logBuffer.logId);
|
|
3332
|
+
dataEnd = writtenExtent > 0 ? Math.min(logBuffer.length, writtenExtent) : 0;
|
|
3333
|
+
} catch {
|
|
3334
|
+
dataEnd = 0;
|
|
3335
|
+
}
|
|
3336
|
+
const resyncPosition = findResyncPosition(dataView, position + 1, dataEnd);
|
|
3337
|
+
return {
|
|
3338
|
+
error: new CorruptFrameError(message, logBuffer.logId, position, resyncPosition),
|
|
3339
|
+
nextPosition: resyncPosition ?? Math.max(dataEnd || limit, size, position + 1)
|
|
3340
|
+
};
|
|
3341
|
+
}
|
|
3342
|
+
/**
|
|
2906
3343
|
* Returns an iterable for transaction entries within the specified range of timestamps
|
|
2907
3344
|
* This iterable can be iterated over multiple times, and subsequent iterations will continue
|
|
2908
3345
|
* from where the last iteration left off, allowing for iteration through the log file
|
|
@@ -2992,9 +3429,17 @@ Object.defineProperty(TransactionLog.prototype, "query", { value({ start, end, e
|
|
|
2992
3429
|
value: void 0
|
|
2993
3430
|
};
|
|
2994
3431
|
const limit = readUncommitted ? logBuffer.length : Math.min(size, logBuffer.length);
|
|
2995
|
-
if (position + TRANSACTION_LOG_ENTRY_HEADER_SIZE > limit)
|
|
3432
|
+
if (position + TRANSACTION_LOG_ENTRY_HEADER_SIZE > limit) {
|
|
3433
|
+
const broken = corruptFrame(`Corrupt transaction log: truncated entry header at position ${position.toString(16)} of log ${logBuffer.logId} (available=${limit - position})`, transactionLog, dataView, logBuffer, position, limit, size, !!readUncommitted);
|
|
3434
|
+
position = broken.nextPosition;
|
|
3435
|
+
throw broken.error;
|
|
3436
|
+
}
|
|
2996
3437
|
const length = dataView.getUint32(position + 8);
|
|
2997
|
-
if (position + TRANSACTION_LOG_ENTRY_HEADER_SIZE + length > limit)
|
|
3438
|
+
if (length === 0 || position + TRANSACTION_LOG_ENTRY_HEADER_SIZE + length > limit) {
|
|
3439
|
+
const broken = corruptFrame(length === 0 ? `Corrupt transaction log entry at position ${position.toString(16)} of log ${logBuffer.logId}: zero-length entry` : `Corrupt transaction log entry at position ${position.toString(16)} of log ${logBuffer.logId}: declared length ${length} overruns the log (limit=${limit})`, transactionLog, dataView, logBuffer, position, limit, size, !!readUncommitted);
|
|
3440
|
+
position = broken.nextPosition;
|
|
3441
|
+
throw broken.error;
|
|
3442
|
+
}
|
|
2998
3443
|
position += TRANSACTION_LOG_ENTRY_HEADER_SIZE;
|
|
2999
3444
|
let matchesRange;
|
|
3000
3445
|
if (foundExactStart) matchesRange = (!exclusiveStart || timestamp !== start) && timestamp < end;
|
|
@@ -3085,9 +3530,9 @@ function loadLastPosition(transactionLog, readUncommitted) {
|
|
|
3085
3530
|
//#region src/index.ts
|
|
3086
3531
|
const versions = {
|
|
3087
3532
|
rocksdb: version,
|
|
3088
|
-
"rocksdb-js": "2.
|
|
3533
|
+
"rocksdb-js": "2.8.0"
|
|
3089
3534
|
};
|
|
3090
3535
|
|
|
3091
3536
|
//#endregion
|
|
3092
|
-
export { DBI, DBIterator, RocksDatabase, Store, Transaction, TransactionLog, backups, constants, coolTransactionLogs, currentThreadId, fileLockRelease, parseTransactionLog, registryStatus, shutdown, stats, supportedCompression, tryFileLock, validateTransactionLogStore, versions };
|
|
3537
|
+
export { BackgroundError, CorruptFrameError, CountEstimator, DBI, DBIterator, RocksDatabase, Store, Transaction, TransactionLog, backups, constants, coolTransactionLogs, currentThreadId, fileLockRelease, parseTransactionLog, registryStatus, shutdown, stats, supportedCompression, tryFileLock, validateTransactionLogStore, versions };
|
|
3093
3538
|
//# sourceMappingURL=index.mjs.map
|