@harperfast/rocksdb-js 2.7.1 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +624 -45
- package/dist/index.cjs +730 -47
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +752 -41
- package/dist/index.d.mts +752 -41
- package/dist/index.mjs +726 -47
- package/dist/index.mjs.map +1 -1
- package/docs/stats.md +355 -269
- package/docs/transaction-log.md +18 -5
- package/docs/transaction-timestamp-integrity-design.md +123 -0
- package/package.json +25 -26
package/dist/index.mjs
CHANGED
|
@@ -55,7 +55,37 @@ function locateBinding() {
|
|
|
55
55
|
throw new Error("Unable to locate rocksdb-js native binding");
|
|
56
56
|
}
|
|
57
57
|
const binding = req(locateBinding());
|
|
58
|
+
/**
|
|
59
|
+
* The native `BackgroundError` constructor (a real `Error` subclass). Exported
|
|
60
|
+
* as both a value — for `err instanceof BackgroundError` — and, via declaration
|
|
61
|
+
* merging with the interface above, a type. Instances are produced by the
|
|
62
|
+
* `'error'` event and `db.getLastError()`; consumers rarely construct their own.
|
|
63
|
+
* The `details` param is required and typed to the instance fields so a bare
|
|
64
|
+
* `new BackgroundError()` (which would omit `severity` / `writesDisabled` / …)
|
|
65
|
+
* is a type error.
|
|
66
|
+
*/
|
|
67
|
+
const BackgroundError = binding.BackgroundError;
|
|
58
68
|
const config = binding.config;
|
|
69
|
+
/**
|
|
70
|
+
* Reads the live state of the process-wide `WriteBufferManager`: its budget,
|
|
71
|
+
* charged memory, mutable share, stall state, and attached column-family
|
|
72
|
+
* inventory. The manager is shared by every database and worker thread in the
|
|
73
|
+
* process.
|
|
74
|
+
*
|
|
75
|
+
* A WriteBufferManager stall does not pass through RocksDB's `WriteController`,
|
|
76
|
+
* so it is not reflected by `rocksdb.stall.micros`, `db.isWriteStalled()`, or
|
|
77
|
+
* the `'writeStall'` event. Use `stallActive` to distinguish this condition from
|
|
78
|
+
* an idle database.
|
|
79
|
+
*
|
|
80
|
+
* @example
|
|
81
|
+
* ```typescript
|
|
82
|
+
* const wbm = getWriteBufferManagerStats();
|
|
83
|
+
* if (wbm.stallActive) {
|
|
84
|
+
* log.warn(`writes stalled for ${wbm.stallActiveMs}ms`);
|
|
85
|
+
* }
|
|
86
|
+
* ```
|
|
87
|
+
*/
|
|
88
|
+
const getWriteBufferManagerStats = binding.getWriteBufferManagerStats;
|
|
59
89
|
const FRESH_VERSION_FLAG$1 = binding.constants.FRESH_VERSION_FLAG;
|
|
60
90
|
const addGlobalListener = binding.addListener;
|
|
61
91
|
const removeGlobalListener = binding.removeListener;
|
|
@@ -104,6 +134,8 @@ const transactionLogMapCount = binding.transactionLogMapCount;
|
|
|
104
134
|
* disarm. Used by the ERR_TRY_AGAIN retry regression test.
|
|
105
135
|
*/
|
|
106
136
|
const forceTryAgainForTesting = binding.forceTryAgainForTesting;
|
|
137
|
+
/** Delays one selected watchdog join in the concurrent-shutdown regression test. */
|
|
138
|
+
const setWriteBufferManagerJoinDelayForTesting = binding.setWriteBufferManagerJoinDelayForTesting;
|
|
107
139
|
/**
|
|
108
140
|
* Creates a native file lock using the specified file path (`flock` on POSIX,
|
|
109
141
|
* `LockFileEx` on Windows), creating the file and any missing parent
|
|
@@ -446,6 +478,150 @@ const backups = {
|
|
|
446
478
|
}
|
|
447
479
|
};
|
|
448
480
|
|
|
481
|
+
//#endregion
|
|
482
|
+
//#region src/count-estimator.ts
|
|
483
|
+
/**
|
|
484
|
+
* Below this many traversed entries the observed density is too noisy to
|
|
485
|
+
* calibrate with, so `estimate()` returns the raw statistical estimate.
|
|
486
|
+
*/
|
|
487
|
+
const CALIBRATION_MIN_TRAVERSED = 16;
|
|
488
|
+
const CALIBRATION_MIN_CONFIDENCE = .8;
|
|
489
|
+
/**
|
|
490
|
+
* Bounds on how far the observed-vs-estimated ratio may scale the remainder;
|
|
491
|
+
* the statistical estimate can be arbitrarily wrong at data-block granularity
|
|
492
|
+
* and an unclamped ratio would let one bad sample dominate.
|
|
493
|
+
*/
|
|
494
|
+
const CALIBRATION_MAX = 8;
|
|
495
|
+
/**
|
|
496
|
+
* Progressively refines a range key-count estimate as an iterator traverses
|
|
497
|
+
* the range. The estimate starts as the pure statistical estimate
|
|
498
|
+
* (`estimateCount`) and converges toward the exact count: the traversed
|
|
499
|
+
* portion is exact, and the statistical estimate of the untraversed remainder
|
|
500
|
+
* is calibrated by the observed ratio of actual-to-estimated entries over the
|
|
501
|
+
* portion already traversed.
|
|
502
|
+
*
|
|
503
|
+
* The estimator never touches the iterator itself — the caller reports
|
|
504
|
+
* progress with `advance(lastKey, count)` whenever it wants a checkpoint
|
|
505
|
+
* (e.g. once per page), then reads `estimate()`. The caller owns the
|
|
506
|
+
* progress contract: cursors must move monotonically through the range and
|
|
507
|
+
* each entry must be reported once (a re-reported page inflates the count
|
|
508
|
+
* undetectably). When traversal completes, call `finish()` so `estimate()`
|
|
509
|
+
* returns the exact total instead of adding a block-granular remainder.
|
|
510
|
+
*
|
|
511
|
+
* Each `estimate()` checkpoint queries RocksDB statistics for the two range
|
|
512
|
+
* segments (cost scales with the SSTs overlapping them, not with key count);
|
|
513
|
+
* results are memoized per checkpoint, so repeated reads between `advance()`
|
|
514
|
+
* calls are free.
|
|
515
|
+
*/
|
|
516
|
+
var CountEstimator = class {
|
|
517
|
+
#store;
|
|
518
|
+
#start;
|
|
519
|
+
#end;
|
|
520
|
+
#exclusiveStart;
|
|
521
|
+
#inclusiveEnd;
|
|
522
|
+
#reverse;
|
|
523
|
+
#cursor;
|
|
524
|
+
#traversed = 0;
|
|
525
|
+
#finished = false;
|
|
526
|
+
#memoized;
|
|
527
|
+
constructor(store, options) {
|
|
528
|
+
this.#store = store;
|
|
529
|
+
this.#reverse = options?.reverse ?? false;
|
|
530
|
+
this.#start = this.#reverse ? options?.end : options?.start;
|
|
531
|
+
this.#end = this.#reverse ? options?.start : options?.end;
|
|
532
|
+
this.#exclusiveStart = options?.exclusiveStart ?? this.#reverse;
|
|
533
|
+
this.#inclusiveEnd = options?.inclusiveEnd ?? this.#reverse;
|
|
534
|
+
}
|
|
535
|
+
/**
|
|
536
|
+
* The number of entries reported traversed so far.
|
|
537
|
+
*/
|
|
538
|
+
get traversed() {
|
|
539
|
+
return this.#traversed;
|
|
540
|
+
}
|
|
541
|
+
/**
|
|
542
|
+
* Records that iteration has advanced through `count` more entries, ending
|
|
543
|
+
* at `lastKey`. Pass `count: 0` when an empty page has no last key.
|
|
544
|
+
*/
|
|
545
|
+
advance(lastKey, count = 1) {
|
|
546
|
+
if (lastKey === void 0) {
|
|
547
|
+
if (count !== 0) throw new Error("CountEstimator.advance requires lastKey when count is nonzero");
|
|
548
|
+
return;
|
|
549
|
+
}
|
|
550
|
+
this.#cursor = lastKey;
|
|
551
|
+
this.#traversed += count;
|
|
552
|
+
this.#memoized = void 0;
|
|
553
|
+
}
|
|
554
|
+
/**
|
|
555
|
+
* Marks traversal of the range as complete: `estimate()` becomes the exact
|
|
556
|
+
* traversed count.
|
|
557
|
+
*/
|
|
558
|
+
finish() {
|
|
559
|
+
this.#finished = true;
|
|
560
|
+
}
|
|
561
|
+
/**
|
|
562
|
+
* Estimates the total number of entries in the full range: the exact
|
|
563
|
+
* traversed count plus a calibrated statistical estimate of the remainder.
|
|
564
|
+
* `confidence` is the exactness-weighted blend of the traversed portion
|
|
565
|
+
* (exact) and the remainder's statistical confidence, so it approaches 1
|
|
566
|
+
* as the exact portion grows (though a checkpoint can decrease when
|
|
567
|
+
* calibration makes a large correction), and is exactly 1 after `finish()`.
|
|
568
|
+
*/
|
|
569
|
+
estimate() {
|
|
570
|
+
if (this.#finished) return {
|
|
571
|
+
count: this.#traversed,
|
|
572
|
+
confidence: 1
|
|
573
|
+
};
|
|
574
|
+
if (this.#memoized !== void 0) return { ...this.#memoized };
|
|
575
|
+
if (this.#cursor === void 0) {
|
|
576
|
+
this.#memoized = this.#store.estimateCount({
|
|
577
|
+
start: this.#start,
|
|
578
|
+
end: this.#end,
|
|
579
|
+
exclusiveStart: this.#exclusiveStart,
|
|
580
|
+
inclusiveEnd: this.#inclusiveEnd
|
|
581
|
+
});
|
|
582
|
+
return { ...this.#memoized };
|
|
583
|
+
}
|
|
584
|
+
const traversedRange = this.#reverse ? {
|
|
585
|
+
start: this.#cursor,
|
|
586
|
+
end: this.#end,
|
|
587
|
+
inclusiveEnd: this.#inclusiveEnd
|
|
588
|
+
} : {
|
|
589
|
+
start: this.#start,
|
|
590
|
+
exclusiveStart: this.#exclusiveStart,
|
|
591
|
+
end: this.#cursor,
|
|
592
|
+
inclusiveEnd: true
|
|
593
|
+
};
|
|
594
|
+
const remainingRange = this.#reverse ? {
|
|
595
|
+
start: this.#start,
|
|
596
|
+
exclusiveStart: this.#exclusiveStart,
|
|
597
|
+
end: this.#cursor
|
|
598
|
+
} : {
|
|
599
|
+
start: this.#cursor,
|
|
600
|
+
exclusiveStart: true,
|
|
601
|
+
end: this.#end,
|
|
602
|
+
inclusiveEnd: this.#inclusiveEnd
|
|
603
|
+
};
|
|
604
|
+
const remainingEstimate = this.#store.estimateCount(remainingRange);
|
|
605
|
+
let remaining = remainingEstimate.count;
|
|
606
|
+
let calibrationConfidence = 1;
|
|
607
|
+
if (this.#traversed >= CALIBRATION_MIN_TRAVERSED) {
|
|
608
|
+
const traversedEstimate = this.#store.estimateCount(traversedRange);
|
|
609
|
+
if (traversedEstimate.count >= CALIBRATION_MIN_TRAVERSED && traversedEstimate.confidence >= CALIBRATION_MIN_CONFIDENCE) {
|
|
610
|
+
const calibration = Math.min(CALIBRATION_MAX, Math.max(1 / CALIBRATION_MAX, this.#traversed / traversedEstimate.count));
|
|
611
|
+
remaining *= calibration;
|
|
612
|
+
calibrationConfidence = Math.min(calibration, 1 / calibration);
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
const count = Math.round(this.#traversed + remaining);
|
|
616
|
+
const confidence = count > 0 ? Math.min(.999, (this.#traversed + calibrationConfidence * remainingEstimate.confidence * remaining) / count) : Math.min(.999, calibrationConfidence * remainingEstimate.confidence);
|
|
617
|
+
this.#memoized = {
|
|
618
|
+
count,
|
|
619
|
+
confidence
|
|
620
|
+
};
|
|
621
|
+
return { ...this.#memoized };
|
|
622
|
+
}
|
|
623
|
+
};
|
|
624
|
+
|
|
449
625
|
//#endregion
|
|
450
626
|
//#region src/util.ts
|
|
451
627
|
/**
|
|
@@ -488,14 +664,12 @@ function parseDuration(duration) {
|
|
|
488
664
|
num *= 1e3;
|
|
489
665
|
break;
|
|
490
666
|
case "m":
|
|
491
|
-
num *=
|
|
667
|
+
num *= 6e4;
|
|
492
668
|
break;
|
|
493
669
|
case "h":
|
|
494
|
-
num *=
|
|
495
|
-
break;
|
|
496
|
-
case "d":
|
|
497
|
-
num *= 1e3 * 60 * 60 * 24;
|
|
670
|
+
num *= 36e5;
|
|
498
671
|
break;
|
|
672
|
+
case "d": num *= 864e5;
|
|
499
673
|
}
|
|
500
674
|
result += num;
|
|
501
675
|
}
|
|
@@ -1198,10 +1372,10 @@ function createFixedBuffer(size) {
|
|
|
1198
1372
|
|
|
1199
1373
|
//#endregion
|
|
1200
1374
|
//#region src/store.ts
|
|
1201
|
-
const { ONLY_IF_IN_MEMORY_CACHE_FLAG, NOT_IN_MEMORY_CACHE_FLAG, ALWAYS_CREATE_NEW_BUFFER_FLAG, FRESH_VERSION_FLAG, POPULATE_VERSION_FLAG, ITERATOR_REVERSE_FLAG, ITERATOR_INCLUSIVE_END_FLAG, ITERATOR_EXCLUSIVE_START_FLAG, ITERATOR_INCLUDE_VALUES_FLAG, ITERATOR_NEEDS_STABLE_VALUE_BUFFER_FLAG, ITERATOR_CONTEXT_IS_TRANSACTION_FLAG } = constants;
|
|
1375
|
+
const { ONLY_IF_IN_MEMORY_CACHE_FLAG, NOT_IN_MEMORY_CACHE_FLAG, ALWAYS_CREATE_NEW_BUFFER_FLAG, FRESH_VERSION_FLAG, POPULATE_VERSION_FLAG, ITERATOR_REVERSE_FLAG, ITERATOR_INCLUSIVE_END_FLAG, ITERATOR_EXCLUSIVE_START_FLAG, ITERATOR_INCLUDE_VALUES_FLAG, ITERATOR_NEEDS_STABLE_VALUE_BUFFER_FLAG, ITERATOR_CONTEXT_IS_TRANSACTION_FLAG, ITERATOR_HAS_TRANSACTION_ID_FLAG } = constants;
|
|
1202
1376
|
const KEY_BUFFER_SIZE = 4096;
|
|
1203
1377
|
const KEY_BUFFER = createFixedBuffer(KEY_BUFFER_SIZE);
|
|
1204
|
-
const VALUE_BUFFER = createFixedBuffer(
|
|
1378
|
+
const VALUE_BUFFER = createFixedBuffer(65536);
|
|
1205
1379
|
/**
|
|
1206
1380
|
* Backing buffer for the shared iterator state. Layout (Uint32Array view):
|
|
1207
1381
|
* [0] = key length written into KEY_BUFFER by the most recent iterator step
|
|
@@ -1209,7 +1383,7 @@ const VALUE_BUFFER = createFixedBuffer(64 * 1024);
|
|
|
1209
1383
|
*/
|
|
1210
1384
|
const ITERATOR_STATE_BUFFER = Buffer.allocUnsafeSlow(8);
|
|
1211
1385
|
const ITERATOR_STATE = new Uint32Array(ITERATOR_STATE_BUFFER.buffer, ITERATOR_STATE_BUFFER.byteOffset, 2);
|
|
1212
|
-
const MAX_KEY_SIZE =
|
|
1386
|
+
const MAX_KEY_SIZE = 1048576;
|
|
1213
1387
|
const SAVE_BUFFER_SIZE = 8192;
|
|
1214
1388
|
/**
|
|
1215
1389
|
* Normalizes the public `compression` option into the primitive fields the
|
|
@@ -1244,6 +1418,15 @@ function normalizeCompression(option) {
|
|
|
1244
1418
|
};
|
|
1245
1419
|
}
|
|
1246
1420
|
/**
|
|
1421
|
+
* Whether `codec` is a pre-constructed encoder/decoder instance that `open()`
|
|
1422
|
+
* would reuse as-is (rather than instantiate fresh from an `Encoder` factory).
|
|
1423
|
+
* Such an instance carries per-column-family mutable state (its `name`, msgpack
|
|
1424
|
+
* `structures`), so it cannot be shared across column families.
|
|
1425
|
+
*/
|
|
1426
|
+
function sharesMutableCodec(codec) {
|
|
1427
|
+
return codec != null && typeof codec === "object" && typeof codec.Encoder !== "function" && (typeof codec.encode === "function" || typeof codec.decode === "function");
|
|
1428
|
+
}
|
|
1429
|
+
/**
|
|
1247
1430
|
* A store wraps the `NativeDatabase` binding and database settings so that a
|
|
1248
1431
|
* single database instance can be shared between the main `RocksDatabase`
|
|
1249
1432
|
* instance and the `Transaction` instance.
|
|
@@ -1262,6 +1445,13 @@ var Store = class {
|
|
|
1262
1445
|
*/
|
|
1263
1446
|
#claimed = false;
|
|
1264
1447
|
/**
|
|
1448
|
+
* Shallow snapshot of the options this store was constructed with, used by
|
|
1449
|
+
* `createColumnFamilyStore()` to derive sibling column-family stores. Snapshot
|
|
1450
|
+
* (not the caller's object) so later mutations of the caller's options don't
|
|
1451
|
+
* leak into derived views.
|
|
1452
|
+
*/
|
|
1453
|
+
#columnFamilyOptions;
|
|
1454
|
+
/**
|
|
1265
1455
|
* The database instance.
|
|
1266
1456
|
*/
|
|
1267
1457
|
db;
|
|
@@ -1397,6 +1587,11 @@ var Store = class {
|
|
|
1397
1587
|
*/
|
|
1398
1588
|
path;
|
|
1399
1589
|
/**
|
|
1590
|
+
* The open database's resolved filesystem identity, read once from
|
|
1591
|
+
* `NativeDatabase.identityPath`. `undefined` until opened.
|
|
1592
|
+
*/
|
|
1593
|
+
identityPath;
|
|
1594
|
+
/**
|
|
1400
1595
|
* Whether to use pessimistic locking for transactions. When `true`,
|
|
1401
1596
|
* transactions will fail as soon as a conflict is detected. When `false`,
|
|
1402
1597
|
* transactions will only fail when `commit()` is called.
|
|
@@ -1404,10 +1599,19 @@ var Store = class {
|
|
|
1404
1599
|
pessimistic;
|
|
1405
1600
|
/**
|
|
1406
1601
|
* Whether the database is open in readonly mode. When `true`, write
|
|
1407
|
-
* operations will throw an error with code `ERR_DATABASE_READONLY
|
|
1602
|
+
* operations will throw an error with code `ERR_DATABASE_READONLY`, except
|
|
1603
|
+
* for `flush()`/`flushSync()`/`compact()`/`compactSync()`, which succeed as
|
|
1604
|
+
* no-ops.
|
|
1408
1605
|
*/
|
|
1409
1606
|
readOnly;
|
|
1410
1607
|
/**
|
|
1608
|
+
* The secondary instance's workspace directory when the database is opened
|
|
1609
|
+
* as a secondary (see `RocksDatabaseOptions.secondaryPath`); `undefined`
|
|
1610
|
+
* otherwise. A secondary is read-only, so `readOnly` is `true` whenever
|
|
1611
|
+
* this is set.
|
|
1612
|
+
*/
|
|
1613
|
+
secondaryPath;
|
|
1614
|
+
/**
|
|
1411
1615
|
* Encoder specific flag used to signal that the encoder should use a random
|
|
1412
1616
|
* access structure.
|
|
1413
1617
|
*/
|
|
@@ -1431,8 +1635,8 @@ var Store = class {
|
|
|
1431
1635
|
*/
|
|
1432
1636
|
transactionLogMaxAgeThreshold;
|
|
1433
1637
|
/**
|
|
1434
|
-
* The maximum size of a transaction log before
|
|
1435
|
-
*
|
|
1638
|
+
* The target maximum size of a transaction log before rotation. Transactions
|
|
1639
|
+
* are never split across files, so one transaction may exceed this target.
|
|
1436
1640
|
*/
|
|
1437
1641
|
transactionLogMaxSize;
|
|
1438
1642
|
/**
|
|
@@ -1493,7 +1697,10 @@ var Store = class {
|
|
|
1493
1697
|
this.parallelismThreads = options?.parallelismThreads;
|
|
1494
1698
|
this.path = path;
|
|
1495
1699
|
this.pessimistic = options?.pessimistic ?? false;
|
|
1496
|
-
|
|
1700
|
+
const secondaryPath = options?.secondaryPath ?? void 0;
|
|
1701
|
+
if (secondaryPath !== void 0 && options?.readOnly === false) throw new Error("A secondary open is read-only; secondaryPath cannot be combined with readOnly: false");
|
|
1702
|
+
this.readOnly = options?.readOnly ?? secondaryPath !== void 0;
|
|
1703
|
+
this.secondaryPath = secondaryPath;
|
|
1497
1704
|
this.randomAccessStructure = options?.randomAccessStructure ?? false;
|
|
1498
1705
|
this.readKey = readKey;
|
|
1499
1706
|
this.sharedStructuresKey = options?.sharedStructuresKey;
|
|
@@ -1505,6 +1712,36 @@ var Store = class {
|
|
|
1505
1712
|
this.verificationTable = options?.verificationTable;
|
|
1506
1713
|
this.writeBufferSize = options?.writeBufferSize;
|
|
1507
1714
|
this.writeKey = writeKey;
|
|
1715
|
+
this.#columnFamilyOptions = options ? { ...options } : void 0;
|
|
1716
|
+
}
|
|
1717
|
+
/**
|
|
1718
|
+
* Creates an independent `Store` for another column family of the same
|
|
1719
|
+
* database, used by {@link RocksDatabase#use}. The returned store shares the
|
|
1720
|
+
* underlying database but must have its own codec state: a mutable
|
|
1721
|
+
* encoder/decoder instance cannot be shared across column families (opening a
|
|
1722
|
+
* view mutates the encoder's `name`, and clearing a view resets its
|
|
1723
|
+
* `structures`), so a pre-constructed instance is rejected here — supply an
|
|
1724
|
+
* encoder factory (`{ Encoder }`) or a named `encoding` instead.
|
|
1725
|
+
*
|
|
1726
|
+
* Override this in a `Store` subclass whose constructor cannot be recreated
|
|
1727
|
+
* from `(path, options)` (e.g. it takes injected dependencies) or that needs
|
|
1728
|
+
* to control how a view's codec state is built.
|
|
1729
|
+
*
|
|
1730
|
+
* @param name - The column family name for the derived store.
|
|
1731
|
+
* @param options - Options overriding this store's, for the derived store.
|
|
1732
|
+
* @returns A new, independent `Store` bound to `name`.
|
|
1733
|
+
*/
|
|
1734
|
+
createColumnFamilyStore(name, options) {
|
|
1735
|
+
const derived = {
|
|
1736
|
+
...this.#columnFamilyOptions,
|
|
1737
|
+
...options,
|
|
1738
|
+
name
|
|
1739
|
+
};
|
|
1740
|
+
if (sharesMutableCodec(derived.encoder) || sharesMutableCodec(derived.decoder)) throw new Error("Cannot derive a column family view from a Store configured with a pre-constructed encoder/decoder instance — its mutable state would be shared across column families. Provide an encoder factory (`{ Encoder }`) or a named `encoding`, or override `Store#createColumnFamilyStore`.");
|
|
1741
|
+
const StoreClass = this.constructor;
|
|
1742
|
+
const store = new StoreClass(this.path, derived);
|
|
1743
|
+
if (store.name !== name) throw new Error(`Store#createColumnFamilyStore produced a store bound to "${store.name}" instead of "${name}"; a custom Store must forward options to super() or override this method.`);
|
|
1744
|
+
return store;
|
|
1508
1745
|
}
|
|
1509
1746
|
/**
|
|
1510
1747
|
* Closes the database.
|
|
@@ -1641,6 +1878,24 @@ var Store = class {
|
|
|
1641
1878
|
}
|
|
1642
1879
|
return result;
|
|
1643
1880
|
}
|
|
1881
|
+
/**
|
|
1882
|
+
* Estimates the number of keys in a range from RocksDB statistics
|
|
1883
|
+
* (memtable stats + approximate SST sizes with range-local entry density)
|
|
1884
|
+
* without iterating. Estimates always reflect committed state, so there is
|
|
1885
|
+
* no transactional variant.
|
|
1886
|
+
*/
|
|
1887
|
+
estimateCount(options) {
|
|
1888
|
+
let startBuffer;
|
|
1889
|
+
let endBuffer;
|
|
1890
|
+
const reverse = options?.reverse ?? false;
|
|
1891
|
+
const start = reverse ? options?.end : options?.start;
|
|
1892
|
+
const end = reverse ? options?.start : options?.end;
|
|
1893
|
+
const exclusiveStart = options?.exclusiveStart ?? reverse;
|
|
1894
|
+
const inclusiveEnd = options?.inclusiveEnd ?? reverse;
|
|
1895
|
+
if (start !== void 0) startBuffer = copyEncodedKey(this.encodeKey(start), exclusiveStart);
|
|
1896
|
+
if (end !== void 0) endBuffer = copyEncodedKey(this.encodeKey(end), inclusiveEnd);
|
|
1897
|
+
return this.db.estimateCount(startBuffer, endBuffer);
|
|
1898
|
+
}
|
|
1644
1899
|
getCount(context, options) {
|
|
1645
1900
|
options = { ...options };
|
|
1646
1901
|
if (options?.start !== void 0) {
|
|
@@ -1668,6 +1923,7 @@ var Store = class {
|
|
|
1668
1923
|
let endUnencoded = options?.key ?? options?.end;
|
|
1669
1924
|
const includeValues = options?.values ?? true;
|
|
1670
1925
|
const reverse = options?.reverse ?? false;
|
|
1926
|
+
const txnId = this.getTxnId(options);
|
|
1671
1927
|
let exclusiveStart = options?.exclusiveStart ?? false;
|
|
1672
1928
|
let inclusiveEnd = options?.inclusiveEnd ?? false;
|
|
1673
1929
|
if (options?.key !== void 0) inclusiveEnd = true;
|
|
@@ -1686,13 +1942,15 @@ var Store = class {
|
|
|
1686
1942
|
startKeyEnd = this.writeKey(startUnencoded, keyBuffer, 0);
|
|
1687
1943
|
if (startKeyEnd === 0) throw new Error("Zero length key is not allowed");
|
|
1688
1944
|
}
|
|
1689
|
-
if (endUnencoded !== void 0)
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1945
|
+
if (endUnencoded !== void 0) {
|
|
1946
|
+
if (endUnencoded === startUnencoded) {
|
|
1947
|
+
endKeyStart = 0;
|
|
1948
|
+
endKeyEnd = startKeyEnd;
|
|
1949
|
+
} else {
|
|
1950
|
+
endKeyStart = startKeyEnd;
|
|
1951
|
+
endKeyEnd = this.writeKey(endUnencoded, keyBuffer, endKeyStart);
|
|
1952
|
+
if (endKeyEnd === endKeyStart) throw new Error("Zero length key is not allowed");
|
|
1953
|
+
}
|
|
1696
1954
|
}
|
|
1697
1955
|
let flags = 0;
|
|
1698
1956
|
if (reverse) flags |= ITERATOR_REVERSE_FLAG;
|
|
@@ -1701,8 +1959,9 @@ var Store = class {
|
|
|
1701
1959
|
if (includeValues) flags |= ITERATOR_INCLUDE_VALUES_FLAG;
|
|
1702
1960
|
if (includeValues && !this.decoderCopies) flags |= ITERATOR_NEEDS_STABLE_VALUE_BUFFER_FLAG;
|
|
1703
1961
|
if (context !== this.db) flags |= ITERATOR_CONTEXT_IS_TRANSACTION_FLAG;
|
|
1962
|
+
if (txnId !== void 0) flags |= ITERATOR_HAS_TRANSACTION_ID_FLAG;
|
|
1704
1963
|
const advancedOptions = options !== void 0 && (options.adaptiveReadahead !== void 0 || options.asyncIO !== void 0 || options.autoReadaheadSize !== void 0 || options.backgroundPurgeOnIteratorCleanup !== void 0 || options.fillCache !== void 0 || options.readaheadSize !== void 0 || options.tailing !== void 0) ? options : void 0;
|
|
1705
|
-
return new ExtendedIterable(new DBIterator(new NativeIterator(context, flags, startKeyEnd, endKeyStart, endKeyEnd, advancedOptions), this, includeValues, options?.limit));
|
|
1964
|
+
return new ExtendedIterable(new DBIterator(new NativeIterator(context, flags, startKeyEnd, endKeyStart, endKeyEnd, advancedOptions, txnId), this, includeValues, options?.limit));
|
|
1706
1965
|
}
|
|
1707
1966
|
getSync(context, key, alwaysCreateNewBuffer = false, options) {
|
|
1708
1967
|
const keyParam = getKeyParam(this.encodeKey(key));
|
|
@@ -1723,9 +1982,12 @@ var Store = class {
|
|
|
1723
1982
|
*/
|
|
1724
1983
|
getTxnId(options) {
|
|
1725
1984
|
let txnId;
|
|
1726
|
-
|
|
1727
|
-
|
|
1985
|
+
const transaction = options?.transaction;
|
|
1986
|
+
if (!this.readOnly && transaction) {
|
|
1987
|
+
txnId = transaction.id;
|
|
1728
1988
|
if (txnId === void 0) throw new TypeError("Invalid transaction");
|
|
1989
|
+
if (transaction.store === void 0) throw new TypeError("Invalid transaction");
|
|
1990
|
+
if (this.identityPath !== void 0 && transaction.store.identityPath !== this.identityPath) throw new TypeError("Transaction belongs to a different database");
|
|
1729
1991
|
}
|
|
1730
1992
|
return txnId;
|
|
1731
1993
|
}
|
|
@@ -1783,7 +2045,10 @@ var Store = class {
|
|
|
1783
2045
|
* are performed.
|
|
1784
2046
|
*/
|
|
1785
2047
|
open() {
|
|
1786
|
-
if (this.db.opened)
|
|
2048
|
+
if (this.db.opened) {
|
|
2049
|
+
this.identityPath = this.db.identityPath;
|
|
2050
|
+
return true;
|
|
2051
|
+
}
|
|
1787
2052
|
const { compression, compressionLevel } = normalizeCompression(this.compression);
|
|
1788
2053
|
this.db.open(this.path, {
|
|
1789
2054
|
compression,
|
|
@@ -1802,6 +2067,7 @@ var Store = class {
|
|
|
1802
2067
|
noBlockCache: this.noBlockCache,
|
|
1803
2068
|
parallelismThreads: this.parallelismThreads,
|
|
1804
2069
|
readOnly: this.readOnly,
|
|
2070
|
+
secondaryPath: this.secondaryPath,
|
|
1805
2071
|
statsLevel: this.statsLevel,
|
|
1806
2072
|
transactionLogMaxAgeThreshold: this.transactionLogMaxAgeThreshold,
|
|
1807
2073
|
transactionLogMaxSize: this.transactionLogMaxSize,
|
|
@@ -1810,6 +2076,7 @@ var Store = class {
|
|
|
1810
2076
|
verificationTable: this.verificationTable,
|
|
1811
2077
|
writeBufferSize: this.writeBufferSize
|
|
1812
2078
|
});
|
|
2079
|
+
this.identityPath = this.db.identityPath;
|
|
1813
2080
|
return false;
|
|
1814
2081
|
}
|
|
1815
2082
|
putSync(context, key, value, options) {
|
|
@@ -1854,7 +2121,7 @@ var Store = class {
|
|
|
1854
2121
|
useLog(context, name) {
|
|
1855
2122
|
if (typeof name !== "string" && typeof name !== "number") throw new TypeError("Log name must be a string or number");
|
|
1856
2123
|
if (typeof name === "string" && /[\t\n\r\\/]/.test(name)) throw new Error(`Invalid transaction log name "${name}"`);
|
|
1857
|
-
return context.useLog(String(name));
|
|
2124
|
+
return context.useLog(String(name), this.db);
|
|
1858
2125
|
}
|
|
1859
2126
|
/**
|
|
1860
2127
|
* Checks the process-global verification table for a fresh version match
|
|
@@ -1888,6 +2155,18 @@ var Store = class {
|
|
|
1888
2155
|
}
|
|
1889
2156
|
};
|
|
1890
2157
|
/**
|
|
2158
|
+
* Copies an encoded key out of the shared key buffer (which the next
|
|
2159
|
+
* `encodeKey` call would clobber), optionally appending a zero byte to
|
|
2160
|
+
* produce the key's bytewise successor.
|
|
2161
|
+
*/
|
|
2162
|
+
function copyEncodedKey(encoded, appendSuccessorByte) {
|
|
2163
|
+
const length = encoded.end - encoded.start;
|
|
2164
|
+
const copy = Buffer.allocUnsafe(length + (appendSuccessorByte ? 1 : 0));
|
|
2165
|
+
copy.set(encoded.subarray(encoded.start, encoded.end));
|
|
2166
|
+
if (appendSuccessorByte) copy[length] = 0;
|
|
2167
|
+
return copy;
|
|
2168
|
+
}
|
|
2169
|
+
/**
|
|
1891
2170
|
* Ensure that they key has been copied into our shared buffer, and return the ending position
|
|
1892
2171
|
* @param keyBuffer
|
|
1893
2172
|
*/
|
|
@@ -2058,10 +2337,12 @@ var Transaction = class extends DBI {
|
|
|
2058
2337
|
return err;
|
|
2059
2338
|
}
|
|
2060
2339
|
/**
|
|
2061
|
-
* Returns the transaction
|
|
2062
|
-
* the transaction
|
|
2340
|
+
* Returns the transaction timestamp in milliseconds since the Unix epoch. It
|
|
2341
|
+
* defaults to a process-wide monotonic value assigned when the transaction
|
|
2342
|
+
* was created. A producer may use this value as its write identity; the
|
|
2343
|
+
* transaction log uses it as the batch key.
|
|
2063
2344
|
*
|
|
2064
|
-
* @returns The transaction
|
|
2345
|
+
* @returns The transaction timestamp in milliseconds.
|
|
2065
2346
|
*/
|
|
2066
2347
|
getTimestamp() {
|
|
2067
2348
|
return this.#txn.getTimestamp();
|
|
@@ -2073,9 +2354,14 @@ var Transaction = class extends DBI {
|
|
|
2073
2354
|
return this.#txn.id;
|
|
2074
2355
|
}
|
|
2075
2356
|
/**
|
|
2076
|
-
*
|
|
2357
|
+
* Overrides the transaction timestamp in milliseconds since the Unix epoch.
|
|
2358
|
+
* Call this before reading the timestamp or staging writes when adopting an
|
|
2359
|
+
* origin timestamp for replication or replay. Once a write or log entry is
|
|
2360
|
+
* staged, the timestamp is frozen, but reapplying the same timestamp remains
|
|
2361
|
+
* idempotent while the transaction is pending. The value must be finite,
|
|
2362
|
+
* positive, and below 8640000000000000.
|
|
2077
2363
|
*
|
|
2078
|
-
* @param timestamp - The timestamp to set in
|
|
2364
|
+
* @param timestamp - The timestamp to set in milliseconds.
|
|
2079
2365
|
*/
|
|
2080
2366
|
setTimestamp(timestamp) {
|
|
2081
2367
|
this.#txn.setTimestamp(timestamp);
|
|
@@ -2084,6 +2370,16 @@ var Transaction = class extends DBI {
|
|
|
2084
2370
|
|
|
2085
2371
|
//#endregion
|
|
2086
2372
|
//#region src/database.ts
|
|
2373
|
+
const BACKGROUND_ERROR_SEVERITY_NAMES = [
|
|
2374
|
+
"none",
|
|
2375
|
+
"soft",
|
|
2376
|
+
"hard",
|
|
2377
|
+
"fatal",
|
|
2378
|
+
"unrecoverable"
|
|
2379
|
+
];
|
|
2380
|
+
function backgroundErrorSeverityName(severity) {
|
|
2381
|
+
return BACKGROUND_ERROR_SEVERITY_NAMES[severity] ?? "unknown";
|
|
2382
|
+
}
|
|
2087
2383
|
/**
|
|
2088
2384
|
* The main class for interacting with a RocksDB database.
|
|
2089
2385
|
*
|
|
@@ -2099,9 +2395,23 @@ var Transaction = class extends DBI {
|
|
|
2099
2395
|
*/
|
|
2100
2396
|
var RocksDatabase = class RocksDatabase extends DBI {
|
|
2101
2397
|
/**
|
|
2102
|
-
* The name of the database.
|
|
2398
|
+
* The name of the database's column family.
|
|
2103
2399
|
*/
|
|
2104
2400
|
#name;
|
|
2401
|
+
/**
|
|
2402
|
+
* Weak cache of column-family views opened via `use()`, keyed by column
|
|
2403
|
+
* family name — mirroring `useLog`'s get-or-create semantics. The reference
|
|
2404
|
+
* is weak so a view is never pinned for this database's lifetime: an
|
|
2405
|
+
* unreferenced view can be collected (closing its own `DBHandle`), and the
|
|
2406
|
+
* next `use()` transparently recreates it. `#columnFamilyRegistry` deletes an
|
|
2407
|
+
* entry once its view is collected so a churn of distinct names (e.g.
|
|
2408
|
+
* `use(uuid())`) does not grow the map without bound.
|
|
2409
|
+
*/
|
|
2410
|
+
#columnFamilies = /* @__PURE__ */ new Map();
|
|
2411
|
+
#columnFamilyRegistry = new FinalizationRegistry((name) => {
|
|
2412
|
+
const ref = this.#columnFamilies.get(name);
|
|
2413
|
+
if (ref !== void 0 && ref.deref() === void 0) this.#columnFamilies.delete(name);
|
|
2414
|
+
});
|
|
2105
2415
|
constructor(pathOrStore, options) {
|
|
2106
2416
|
let store;
|
|
2107
2417
|
if (typeof pathOrStore === "string") store = new Store(pathOrStore, options);
|
|
@@ -2109,7 +2419,7 @@ var RocksDatabase = class RocksDatabase extends DBI {
|
|
|
2109
2419
|
else throw new TypeError("Invalid database path or store");
|
|
2110
2420
|
store.claim();
|
|
2111
2421
|
super(store);
|
|
2112
|
-
this.#name =
|
|
2422
|
+
this.#name = this.store.name;
|
|
2113
2423
|
}
|
|
2114
2424
|
/**
|
|
2115
2425
|
* Removes all data from the database asynchronously.
|
|
@@ -2155,6 +2465,9 @@ var RocksDatabase = class RocksDatabase extends DBI {
|
|
|
2155
2465
|
* Compacts the entire key range of the database asynchronously.
|
|
2156
2466
|
* This triggers manual compaction which removes tombstones and reclaims space.
|
|
2157
2467
|
*
|
|
2468
|
+
* On a read-only database this is a no-op: arguments are still validated, but
|
|
2469
|
+
* the returned promise resolves without compacting rather than rejecting.
|
|
2470
|
+
*
|
|
2158
2471
|
* @example
|
|
2159
2472
|
* ```typescript
|
|
2160
2473
|
* const db = RocksDatabase.open('/path/to/database');
|
|
@@ -2206,6 +2519,9 @@ var RocksDatabase = class RocksDatabase extends DBI {
|
|
|
2206
2519
|
* Compacts the entire key range of the database synchronously.
|
|
2207
2520
|
* This triggers manual compaction which removes tombstones and reclaims space.
|
|
2208
2521
|
*
|
|
2522
|
+
* On a read-only database this validates its arguments and then returns
|
|
2523
|
+
* without compacting, rather than throwing.
|
|
2524
|
+
*
|
|
2209
2525
|
* @example
|
|
2210
2526
|
* ```typescript
|
|
2211
2527
|
* const db = RocksDatabase.open('/path/to/database');
|
|
@@ -2322,6 +2638,40 @@ var RocksDatabase = class RocksDatabase extends DBI {
|
|
|
2322
2638
|
static notify(event, ...args) {
|
|
2323
2639
|
return globalNotify(event, args);
|
|
2324
2640
|
}
|
|
2641
|
+
/**
|
|
2642
|
+
* Advances a secondary instance to the primary's current state by tailing
|
|
2643
|
+
* and replaying the primary's MANIFEST and WAL. A secondary does not see
|
|
2644
|
+
* the primary's writes until this (or {@link catchUpWithPrimarySync}) is
|
|
2645
|
+
* called. Concurrent reads through this handle remain safe while the
|
|
2646
|
+
* catch-up runs.
|
|
2647
|
+
*
|
|
2648
|
+
* Only valid on a database opened with the `secondaryPath` option; throws
|
|
2649
|
+
* with code `ERR_NOT_SECONDARY` otherwise.
|
|
2650
|
+
*
|
|
2651
|
+
* Advances the **database** view only. Against a cross-process primary,
|
|
2652
|
+
* transaction-log reads keep serving the stores discovered at open (a store
|
|
2653
|
+
* the primary creates later, and its appends, need a reopen); a primary in
|
|
2654
|
+
* the same process shares the store registry, so its stores and appends are
|
|
2655
|
+
* visible immediately. Either way a log entry can describe data this
|
|
2656
|
+
* handle's database view does not have yet — the log write completes before
|
|
2657
|
+
* the RocksDB commit for every writer, so consumers must tolerate the log
|
|
2658
|
+
* leading the database.
|
|
2659
|
+
*
|
|
2660
|
+
* Await one call before starting the next: the replay holds a libuv worker
|
|
2661
|
+
* for its whole duration and concurrent calls queue on a per-database mutex
|
|
2662
|
+
* while holding theirs, so overlapping catch-ups on a backlogged follower
|
|
2663
|
+
* can exhaust the default four-thread pool and stall unrelated I/O.
|
|
2664
|
+
*/
|
|
2665
|
+
catchUpWithPrimary() {
|
|
2666
|
+
return new Promise((resolve, reject) => this.store.db.catchUpWithPrimary(resolve, reject));
|
|
2667
|
+
}
|
|
2668
|
+
/**
|
|
2669
|
+
* Synchronous version of {@link catchUpWithPrimary}. The replay runs on the
|
|
2670
|
+
* JS thread, so prefer the async form unless the caller is already blocking.
|
|
2671
|
+
*/
|
|
2672
|
+
catchUpWithPrimarySync() {
|
|
2673
|
+
this.store.db.catchUpWithPrimarySync();
|
|
2674
|
+
}
|
|
2325
2675
|
destroy() {
|
|
2326
2676
|
this.store.db.destroy();
|
|
2327
2677
|
}
|
|
@@ -2339,18 +2689,107 @@ var RocksDatabase = class RocksDatabase extends DBI {
|
|
|
2339
2689
|
/**
|
|
2340
2690
|
* Flushes the underlying database by performing a commit or clearing any buffered operations.
|
|
2341
2691
|
*
|
|
2692
|
+
* On a read-only database this is a no-op: the options bag is still validated, but the
|
|
2693
|
+
* returned promise resolves without flushing rather than rejecting.
|
|
2694
|
+
*
|
|
2695
|
+
* @param options - Flush options; see {@link FlushOptions.allowWriteStall} before relying on
|
|
2696
|
+
* this resolving promptly under write pressure.
|
|
2342
2697
|
* @return {void} Does not return a value.
|
|
2343
2698
|
*/
|
|
2344
|
-
flush() {
|
|
2345
|
-
return new Promise((resolve, reject) => this.store.db.flush(resolve, reject));
|
|
2699
|
+
flush(options) {
|
|
2700
|
+
return new Promise((resolve, reject) => this.store.db.flush(resolve, reject, options));
|
|
2346
2701
|
}
|
|
2347
2702
|
/**
|
|
2348
2703
|
* Synchronously flushes the underlying database by performing a commit or clearing any buffered operations.
|
|
2349
2704
|
*
|
|
2705
|
+
* The default `allowWriteStall: false` wait is taken on the JS thread here, so it freezes the
|
|
2706
|
+
* event loop rather than parking a libuv worker — a larger blast radius than {@link flush}, not
|
|
2707
|
+
* a smaller one.
|
|
2708
|
+
*
|
|
2709
|
+
* On a read-only database this validates its options and then returns without flushing, rather
|
|
2710
|
+
* than throwing.
|
|
2711
|
+
*
|
|
2712
|
+
* @param options - Flush options; see {@link FlushOptions.allowWriteStall}.
|
|
2350
2713
|
* @return {void} Does not return a value.
|
|
2351
2714
|
*/
|
|
2352
|
-
flushSync() {
|
|
2353
|
-
return this.store.db.flushSync();
|
|
2715
|
+
flushSync(options) {
|
|
2716
|
+
return this.store.db.flushSync(options);
|
|
2717
|
+
}
|
|
2718
|
+
/**
|
|
2719
|
+
* Returns the most recent {@link BackgroundError} observed on this database, or
|
|
2720
|
+
* `null` when none has occurred. The `'error'` event is the push counterpart;
|
|
2721
|
+
* this is the pull equivalent for on-demand checks (e.g. a health probe) and
|
|
2722
|
+
* for catching an error that fired after open but before a listener was
|
|
2723
|
+
* attached. (A background error that occurs *during* the initial open is a
|
|
2724
|
+
* narrow startup window not yet captured, addressed separately.)
|
|
2725
|
+
*
|
|
2726
|
+
* Purely historical — it is not cleared by a successful {@link RocksDatabase.resume}.
|
|
2727
|
+
* When the returned error's `writesDisabled` is `true`, writes are stopped
|
|
2728
|
+
* until recovery.
|
|
2729
|
+
*
|
|
2730
|
+
* @example
|
|
2731
|
+
* ```typescript
|
|
2732
|
+
* const err = db.getLastError();
|
|
2733
|
+
* if (err?.writesDisabled) {
|
|
2734
|
+
* // ...free disk space...
|
|
2735
|
+
* db.resume();
|
|
2736
|
+
* }
|
|
2737
|
+
* ```
|
|
2738
|
+
*/
|
|
2739
|
+
getLastError() {
|
|
2740
|
+
return this.store.db.getLastError();
|
|
2741
|
+
}
|
|
2742
|
+
/**
|
|
2743
|
+
* Sets or clears the last background error, mirroring the Win32
|
|
2744
|
+
* `SetLastError`/`GetLastError` pair. Pass `null` (or no argument) to **reset**
|
|
2745
|
+
* it — {@link RocksDatabase.getLastError} then returns `null` and no event
|
|
2746
|
+
* fires; this is the intended way to clear the error after handling or
|
|
2747
|
+
* recovering it (e.g. after {@link RocksDatabase.resume}). Pass an object to
|
|
2748
|
+
* set it: it is stored and the `'error'` event fires with the reconstructed
|
|
2749
|
+
* {@link BackgroundError}. `type` defaults to `'background'`.
|
|
2750
|
+
*
|
|
2751
|
+
* @example
|
|
2752
|
+
* ```typescript
|
|
2753
|
+
* db.resume();
|
|
2754
|
+
* db.setLastError(null); // recovered — clear the historical error
|
|
2755
|
+
* ```
|
|
2756
|
+
*/
|
|
2757
|
+
setLastError(error) {
|
|
2758
|
+
if (error == null) return this.store.db.setLastError(null);
|
|
2759
|
+
const severity = error.severity ?? 0;
|
|
2760
|
+
return this.store.db.setLastError({
|
|
2761
|
+
...error,
|
|
2762
|
+
type: error.type ?? "background",
|
|
2763
|
+
severity,
|
|
2764
|
+
severityName: error.severityName ?? backgroundErrorSeverityName(severity),
|
|
2765
|
+
writesDisabled: error.writesDisabled ?? severity >= 2
|
|
2766
|
+
});
|
|
2767
|
+
}
|
|
2768
|
+
/**
|
|
2769
|
+
* Attempts to recover the database from a background error by calling
|
|
2770
|
+
* RocksDB's `DB::Resume()`. When a write fails at the filesystem level (e.g. a
|
|
2771
|
+
* full disk), RocksDB stops accepting writes and this database emits an
|
|
2772
|
+
* {@link BackgroundError} via the `'error'` event. Call `resume()` after the
|
|
2773
|
+
* underlying condition has cleared — e.g. once disk space has been freed. On
|
|
2774
|
+
* success writes are accepted again; on failure (the condition has not
|
|
2775
|
+
* actually cleared) it throws and the database stays read-only. A no-op on a
|
|
2776
|
+
* healthy database.
|
|
2777
|
+
*
|
|
2778
|
+
* Runs synchronously and may briefly block, since recovery can re-flush
|
|
2779
|
+
* memtables.
|
|
2780
|
+
*
|
|
2781
|
+
* @example
|
|
2782
|
+
* ```typescript
|
|
2783
|
+
* db.on('error', (err) => {
|
|
2784
|
+
* if (err.writesDisabled) {
|
|
2785
|
+
* // ...free disk space...
|
|
2786
|
+
* db.resume();
|
|
2787
|
+
* }
|
|
2788
|
+
* });
|
|
2789
|
+
* ```
|
|
2790
|
+
*/
|
|
2791
|
+
resume() {
|
|
2792
|
+
return this.store.db.resume();
|
|
2354
2793
|
}
|
|
2355
2794
|
/**
|
|
2356
2795
|
* Gets a RocksDB database property as an integer.
|
|
@@ -2385,7 +2824,32 @@ var RocksDatabase = class RocksDatabase extends DBI {
|
|
|
2385
2824
|
return this.store.db.getDBProperty(propertyName);
|
|
2386
2825
|
}
|
|
2387
2826
|
/**
|
|
2388
|
-
*
|
|
2827
|
+
* Whether RocksDB is currently applying write backpressure to this database —
|
|
2828
|
+
* either delaying (rate-limiting) or fully stopping writes. This is the
|
|
2829
|
+
* authoritative, live pull counterpart to the `'writeStall'`
|
|
2830
|
+
* [event](../README.md#event-writestall): the event pushes per-column-family
|
|
2831
|
+
* transitions (and is rate-limited), while this reads the current state on
|
|
2832
|
+
* demand and can never be stale.
|
|
2833
|
+
*
|
|
2834
|
+
* The write controller is database-wide (shared across every column family),
|
|
2835
|
+
* so this reflects the database as a whole, not a single column family — a
|
|
2836
|
+
* stall triggered by any column family stops writes for all of them. Reads
|
|
2837
|
+
* `rocksdb.is-write-stopped` and `rocksdb.actual-delayed-write-rate`.
|
|
2838
|
+
*
|
|
2839
|
+
* @example
|
|
2840
|
+
* ```typescript
|
|
2841
|
+
* db.on('writeStall', (cf) => {
|
|
2842
|
+
* if (db.isWriteStalled()) log.warn(`writes throttled/blocked (triggered by ${cf})`);
|
|
2843
|
+
* });
|
|
2844
|
+
* ```
|
|
2845
|
+
*/
|
|
2846
|
+
isWriteStalled() {
|
|
2847
|
+
return this.getDBIntProperty("rocksdb.is-write-stopped") === 1 || (this.getDBIntProperty("rocksdb.actual-delayed-write-rate") ?? 0) > 0;
|
|
2848
|
+
}
|
|
2849
|
+
/**
|
|
2850
|
+
* Retrieves the estimated number of keys in the database. This is an alias
|
|
2851
|
+
* for `db.estimateCount().count`; use `estimateCount()` for range support
|
|
2852
|
+
* and a confidence indicator.
|
|
2389
2853
|
*
|
|
2390
2854
|
* @example
|
|
2391
2855
|
* ```typescript
|
|
@@ -2398,6 +2862,58 @@ var RocksDatabase = class RocksDatabase extends DBI {
|
|
|
2398
2862
|
return this.getDBIntProperty("rocksdb.estimate-num-keys") ?? 0;
|
|
2399
2863
|
}
|
|
2400
2864
|
/**
|
|
2865
|
+
* Estimates the number of keys in the database, or within a key range,
|
|
2866
|
+
* returning `{ count, confidence }`. Unlike `getKeysCount()`, this never
|
|
2867
|
+
* iterates: the estimate is derived from RocksDB statistics (memtable
|
|
2868
|
+
* stats plus approximate SST sizes converted through the entry density of
|
|
2869
|
+
* the SSTs overlapping the range), so its cost scales with the number of
|
|
2870
|
+
* SSTs overlapping the range rather than the number of keys — though
|
|
2871
|
+
* reading table properties for cold files can do I/O through the table
|
|
2872
|
+
* cache, and a start-only range does the work of its complement below
|
|
2873
|
+
* `start`. Accuracy improves with range size. Resolution is bounded by SST
|
|
2874
|
+
* data-block granularity, so tiny ranges may over-report or report zero for
|
|
2875
|
+
* present keys; low `confidence` is the signal. Recently deleted or
|
|
2876
|
+
* overwritten entries may be counted until compaction. An inverted range
|
|
2877
|
+
* (`start` >= `end`) returns `{ count: 0, confidence: 1 }`.
|
|
2878
|
+
*
|
|
2879
|
+
* `confidence` is a heuristic 0–1 trust indicator (1 only when exact) —
|
|
2880
|
+
* see `CountEstimate`. Estimates always reflect committed state; writes
|
|
2881
|
+
* pending in a transaction are not included.
|
|
2882
|
+
*
|
|
2883
|
+
* @example
|
|
2884
|
+
* ```typescript
|
|
2885
|
+
* const db = RocksDatabase.open('/path/to/database');
|
|
2886
|
+
* const { count, confidence } = db.estimateCount({ start: 'a', end: 'z' });
|
|
2887
|
+
* ```
|
|
2888
|
+
*/
|
|
2889
|
+
estimateCount(options) {
|
|
2890
|
+
return this.store.estimateCount(options);
|
|
2891
|
+
}
|
|
2892
|
+
/**
|
|
2893
|
+
* Creates a `CountEstimator` for progressively refining a range count
|
|
2894
|
+
* estimate while iterating the range: report progress with
|
|
2895
|
+
* `advance(lastKey, count)` (e.g. once per page of results) and
|
|
2896
|
+
* `estimate()` returns the exact traversed count plus a calibrated
|
|
2897
|
+
* estimate of the remainder, converging toward the exact total. Call
|
|
2898
|
+
* `finish()` when traversal completes to make `estimate()` exact.
|
|
2899
|
+
*
|
|
2900
|
+
* @example
|
|
2901
|
+
* ```typescript
|
|
2902
|
+
* const estimator = db.createCountEstimator({ start: 'a', end: 'z' });
|
|
2903
|
+
* let lastKey;
|
|
2904
|
+
* let pageSize = 0;
|
|
2905
|
+
* for (const { key } of db.getRange({ start: 'a', end: 'z', limit: 25 })) {
|
|
2906
|
+
* lastKey = key;
|
|
2907
|
+
* pageSize++;
|
|
2908
|
+
* }
|
|
2909
|
+
* estimator.advance(lastKey, pageSize);
|
|
2910
|
+
* const { count, confidence } = estimator.estimate();
|
|
2911
|
+
* ```
|
|
2912
|
+
*/
|
|
2913
|
+
createCountEstimator(options) {
|
|
2914
|
+
return new CountEstimator(this.store, options);
|
|
2915
|
+
}
|
|
2916
|
+
/**
|
|
2401
2917
|
* Returns the current timestamp as a monotonically increasing timestamp in
|
|
2402
2918
|
* milliseconds represented as a decimal number.
|
|
2403
2919
|
*
|
|
@@ -2508,18 +3024,29 @@ var RocksDatabase = class RocksDatabase extends DBI {
|
|
|
2508
3024
|
return this.store.listLogs();
|
|
2509
3025
|
}
|
|
2510
3026
|
/**
|
|
2511
|
-
* The name of the database.
|
|
3027
|
+
* The name of the database's column family.
|
|
2512
3028
|
*/
|
|
2513
3029
|
get name() {
|
|
2514
3030
|
return this.#name;
|
|
2515
3031
|
}
|
|
2516
3032
|
/**
|
|
2517
|
-
* Whether the database is open in readonly mode.
|
|
3033
|
+
* Whether the database is open in readonly mode. Also `true` for a
|
|
3034
|
+
* secondary instance (see {@link secondaryPath}), which is read-only by
|
|
3035
|
+
* construction.
|
|
2518
3036
|
*/
|
|
2519
3037
|
get readOnly() {
|
|
2520
3038
|
return this.store.readOnly;
|
|
2521
3039
|
}
|
|
2522
3040
|
/**
|
|
3041
|
+
* The secondary instance's workspace directory when the database was opened
|
|
3042
|
+
* as a secondary (a read-only follower of a live primary — see the
|
|
3043
|
+
* `secondaryPath` open option), or `undefined` for a regular or plain
|
|
3044
|
+
* read-only open.
|
|
3045
|
+
*/
|
|
3046
|
+
get secondaryPath() {
|
|
3047
|
+
return this.store.secondaryPath;
|
|
3048
|
+
}
|
|
3049
|
+
/**
|
|
2523
3050
|
* Sugar method for opening a database.
|
|
2524
3051
|
*
|
|
2525
3052
|
* @param pathOrStore - The filesystem path to the database or a custom store.
|
|
@@ -2792,6 +3319,57 @@ var RocksDatabase = class RocksDatabase extends DBI {
|
|
|
2792
3319
|
return this.store.unlock(key);
|
|
2793
3320
|
}
|
|
2794
3321
|
/**
|
|
3322
|
+
* Returns a `RocksDatabase` bound to the `name` column family of this same
|
|
3323
|
+
* database, opening (and creating, if missing) the column family on first
|
|
3324
|
+
* use. This is a factory — create-and-open sugar — not a stateful switch: the
|
|
3325
|
+
* returned view is an independent instance whose own reads/writes target its
|
|
3326
|
+
* column family, while sharing the underlying database (so a transaction,
|
|
3327
|
+
* backup, or checkpoint still spans every column family). Because the view
|
|
3328
|
+
* opens its own handle, this database need not itself be open — a discarded
|
|
3329
|
+
* parent (`new RocksDatabase(path).use('events')`) still yields a valid,
|
|
3330
|
+
* opened view.
|
|
3331
|
+
*
|
|
3332
|
+
* Get-or-create, backed by a weak cache: `db.use('events') === db.use('events')`
|
|
3333
|
+
* while the view is still referenced **and open**, and calling with this
|
|
3334
|
+
* database's own column-family name returns `this`. A view that has been
|
|
3335
|
+
* closed (or garbage-collected — the cache does not pin it) is transparently
|
|
3336
|
+
* recreated on the next `use()`. Views are independent handles: closing this
|
|
3337
|
+
* database does not close them, and vice versa — the underlying database
|
|
3338
|
+
* stays open until every handle is closed or collected.
|
|
3339
|
+
*
|
|
3340
|
+
* The view's store is derived by {@link Store#createColumnFamilyStore}, which
|
|
3341
|
+
* builds an independent store of the same class with its own codec state (so
|
|
3342
|
+
* views never share a mutable encoder/decoder) and inherits the options this
|
|
3343
|
+
* store was constructed with; pass `options` to override them for this column
|
|
3344
|
+
* family. A database configured with a pre-constructed encoder/decoder
|
|
3345
|
+
* instance cannot derive views (its state would be shared) — use an encoder
|
|
3346
|
+
* factory (`{ Encoder }`) or a named `encoding`, or override
|
|
3347
|
+
* `createColumnFamilyStore` on a custom `Store`.
|
|
3348
|
+
*
|
|
3349
|
+
* @param name - The column family name.
|
|
3350
|
+
* @param options - Options for the column family, overriding inherited ones.
|
|
3351
|
+
* @returns A `RocksDatabase` bound to the column family.
|
|
3352
|
+
*
|
|
3353
|
+
* @example
|
|
3354
|
+
* ```typescript
|
|
3355
|
+
* const db = RocksDatabase.open('/path/to/database');
|
|
3356
|
+
* const events = db.use('events');
|
|
3357
|
+
* await events.put('e1', payload);
|
|
3358
|
+
* await db.put('k', 'v'); // default column family, unaffected
|
|
3359
|
+
* ```
|
|
3360
|
+
*/
|
|
3361
|
+
use(name, options) {
|
|
3362
|
+
if (typeof name !== "string" || name === "") throw new TypeError("Column family name must be a non-empty string");
|
|
3363
|
+
if (name === this.#name) return this.open();
|
|
3364
|
+
const cached = this.#columnFamilies.get(name)?.deref();
|
|
3365
|
+
if (cached !== void 0 && cached.isOpen()) return cached;
|
|
3366
|
+
const columnFamily = new RocksDatabase(this.store.createColumnFamilyStore(name, options));
|
|
3367
|
+
columnFamily.open();
|
|
3368
|
+
this.#columnFamilies.set(name, new WeakRef(columnFamily));
|
|
3369
|
+
this.#columnFamilyRegistry.register(columnFamily, name);
|
|
3370
|
+
return columnFamily;
|
|
3371
|
+
}
|
|
3372
|
+
/**
|
|
2795
3373
|
* Excecutes a function using a thread-safe lock to ensure mutual
|
|
2796
3374
|
* exclusion.
|
|
2797
3375
|
*
|
|
@@ -2903,6 +3481,98 @@ const FLOAT_TO_UINT32 = /* @__PURE__ */ new Float64Array(1);
|
|
|
2903
3481
|
const UINT32_FROM_FLOAT = new Uint32Array(FLOAT_TO_UINT32.buffer);
|
|
2904
3482
|
const { TRANSACTION_LOG_FILE_HEADER_SIZE, TRANSACTION_LOG_ENTRY_HEADER_SIZE } = constants;
|
|
2905
3483
|
/**
|
|
3484
|
+
* Number of consecutive well-formed frames that, on their own, signal that real log data has
|
|
3485
|
+
* resumed after a framing break. Kept in step with `RESYNC_MIN_FRAMES` in
|
|
3486
|
+
* `src/binding/transaction_log/transaction_log_recovery.cpp`, which uses the same heuristic to
|
|
3487
|
+
* classify a break as mid-file corruption (valid entries follow) rather than a torn tail.
|
|
3488
|
+
*/
|
|
3489
|
+
const RESYNC_MIN_FRAMES = 8;
|
|
3490
|
+
/**
|
|
3491
|
+
* Thrown when an entry's framing is broken. `resyncPosition` is the offset where valid framing
|
|
3492
|
+
* resumes, when one was found: the frames after a mid-log break are intact and reachable, and
|
|
3493
|
+
* only the broken span between `position` and `resyncPosition` is unreadable. It is `undefined`
|
|
3494
|
+
* for a torn tail, where nothing follows the break.
|
|
3495
|
+
*
|
|
3496
|
+
* The iterator is left positioned at `resyncPosition` (or at end-of-log when there is none), so a
|
|
3497
|
+
* caller that chooses to recover the entries past the break calls `next()` again.
|
|
3498
|
+
*/
|
|
3499
|
+
var CorruptFrameError = class extends RangeError {
|
|
3500
|
+
/** Sequence number of the log file the break is in. */
|
|
3501
|
+
logId;
|
|
3502
|
+
/** Offset of the broken frame. */
|
|
3503
|
+
position;
|
|
3504
|
+
/** Offset where valid framing resumes, or `undefined` when nothing valid follows. */
|
|
3505
|
+
resyncPosition;
|
|
3506
|
+
/** Bytes between the break and the resume point that cannot be read. */
|
|
3507
|
+
unreadableBytes;
|
|
3508
|
+
constructor(message, logId, position, resyncPosition) {
|
|
3509
|
+
super(resyncPosition === void 0 ? message : `${message}; valid framing resumes at ${resyncPosition.toString(16)}, ${resyncPosition - position} byte(s) unreadable`);
|
|
3510
|
+
this.name = "CorruptFrameError";
|
|
3511
|
+
this.logId = logId;
|
|
3512
|
+
this.position = position;
|
|
3513
|
+
this.resyncPosition = resyncPosition;
|
|
3514
|
+
this.unreadableBytes = resyncPosition === void 0 ? 0 : resyncPosition - position;
|
|
3515
|
+
}
|
|
3516
|
+
};
|
|
3517
|
+
/**
|
|
3518
|
+
* Whether a complete, in-bounds frame begins at `pos`. The only sane bound on an entry's length is
|
|
3519
|
+
* the written extent: a single entry can legitimately exceed the rotation threshold (the first
|
|
3520
|
+
* entry written to a fresh file is always written in full). A zero timestamp is the
|
|
3521
|
+
* end-of-entries marker, not a frame.
|
|
3522
|
+
*/
|
|
3523
|
+
function frameFits(dataView, pos, dataEnd) {
|
|
3524
|
+
if (pos + TRANSACTION_LOG_ENTRY_HEADER_SIZE > dataEnd) return false;
|
|
3525
|
+
if (dataView.getFloat64(pos) === 0) return false;
|
|
3526
|
+
const length = dataView.getUint32(pos + 8);
|
|
3527
|
+
if (length === 0) return false;
|
|
3528
|
+
return pos + TRANSACTION_LOG_ENTRY_HEADER_SIZE + length <= dataEnd;
|
|
3529
|
+
}
|
|
3530
|
+
/**
|
|
3531
|
+
* Finds the offset in `[from, dataEnd)` where valid log framing resumes: a run of at least
|
|
3532
|
+
* `RESYNC_MIN_FRAMES` well-formed frames, or a run that lands exactly on `dataEnd`. Returns
|
|
3533
|
+
* `undefined` when nothing resumes (a torn tail).
|
|
3534
|
+
*
|
|
3535
|
+
* `dataEnd` must be the **written extent**, not the mapped capacity. The log's memory map is
|
|
3536
|
+
* pre-extended well past the data, and every offset inside that zero padding reads as an
|
|
3537
|
+
* end-of-entries marker — so scanning against the map would both lose the exact-end signal (the
|
|
3538
|
+
* chain can never land on a capacity that data does not reach) and, if a zero were accepted as a
|
|
3539
|
+
* terminator, let a chain "end" anywhere in megabytes of padding. Against the written extent the
|
|
3540
|
+
* end is a single offset, so a chain landing on it is ~1/2^32 to be coincidence.
|
|
3541
|
+
*
|
|
3542
|
+
* The JS counterpart of `validFramingResumes()` in `transaction_log_recovery.cpp`, which answers
|
|
3543
|
+
* the same question at open time but keeps only the yes/no, discarding the offset.
|
|
3544
|
+
*/
|
|
3545
|
+
function findResyncPosition(dataView, from, dataEnd) {
|
|
3546
|
+
const lastStart = dataEnd - TRANSACTION_LOG_ENTRY_HEADER_SIZE;
|
|
3547
|
+
for (let start = from; start <= lastStart; start++) {
|
|
3548
|
+
let pos = start;
|
|
3549
|
+
let frames = 0;
|
|
3550
|
+
while (frames < RESYNC_MIN_FRAMES && frameFits(dataView, pos, dataEnd)) {
|
|
3551
|
+
pos += TRANSACTION_LOG_ENTRY_HEADER_SIZE + dataView.getUint32(pos + 8);
|
|
3552
|
+
if (++frames >= RESYNC_MIN_FRAMES || pos === dataEnd) return start;
|
|
3553
|
+
}
|
|
3554
|
+
}
|
|
3555
|
+
}
|
|
3556
|
+
/**
|
|
3557
|
+
* Builds the report for a broken frame, with the offset iteration should continue from. Resolves
|
|
3558
|
+
* the written extent here rather than per frame: `getLogFileSize` crosses into native and takes
|
|
3559
|
+
* the store mutex, which a healthy read must never pay.
|
|
3560
|
+
*/
|
|
3561
|
+
function corruptFrame(message, transactionLog, dataView, logBuffer, position, limit, size, readUncommitted) {
|
|
3562
|
+
let dataEnd = limit;
|
|
3563
|
+
if (readUncommitted) try {
|
|
3564
|
+
const writtenExtent = transactionLog.getLogFileSize(logBuffer.logId);
|
|
3565
|
+
dataEnd = writtenExtent > 0 ? Math.min(logBuffer.length, writtenExtent) : 0;
|
|
3566
|
+
} catch {
|
|
3567
|
+
dataEnd = 0;
|
|
3568
|
+
}
|
|
3569
|
+
const resyncPosition = findResyncPosition(dataView, position + 1, dataEnd);
|
|
3570
|
+
return {
|
|
3571
|
+
error: new CorruptFrameError(message, logBuffer.logId, position, resyncPosition),
|
|
3572
|
+
nextPosition: resyncPosition ?? Math.max(dataEnd || limit, size, position + 1)
|
|
3573
|
+
};
|
|
3574
|
+
}
|
|
3575
|
+
/**
|
|
2906
3576
|
* Returns an iterable for transaction entries within the specified range of timestamps
|
|
2907
3577
|
* This iterable can be iterated over multiple times, and subsequent iterations will continue
|
|
2908
3578
|
* from where the last iteration left off, allowing for iteration through the log file
|
|
@@ -2992,17 +3662,26 @@ Object.defineProperty(TransactionLog.prototype, "query", { value({ start, end, e
|
|
|
2992
3662
|
value: void 0
|
|
2993
3663
|
};
|
|
2994
3664
|
const limit = readUncommitted ? logBuffer.length : Math.min(size, logBuffer.length);
|
|
2995
|
-
if (position + TRANSACTION_LOG_ENTRY_HEADER_SIZE > limit)
|
|
3665
|
+
if (position + TRANSACTION_LOG_ENTRY_HEADER_SIZE > limit) {
|
|
3666
|
+
const broken = corruptFrame(`Corrupt transaction log: truncated entry header at position ${position.toString(16)} of log ${logBuffer.logId} (available=${limit - position})`, transactionLog, dataView, logBuffer, position, limit, size, !!readUncommitted);
|
|
3667
|
+
position = broken.nextPosition;
|
|
3668
|
+
throw broken.error;
|
|
3669
|
+
}
|
|
2996
3670
|
const length = dataView.getUint32(position + 8);
|
|
2997
|
-
if (position + TRANSACTION_LOG_ENTRY_HEADER_SIZE + length > limit)
|
|
3671
|
+
if (length === 0 || position + TRANSACTION_LOG_ENTRY_HEADER_SIZE + length > limit) {
|
|
3672
|
+
const broken = corruptFrame(length === 0 ? `Corrupt transaction log entry at position ${position.toString(16)} of log ${logBuffer.logId}: zero-length entry` : `Corrupt transaction log entry at position ${position.toString(16)} of log ${logBuffer.logId}: declared length ${length} overruns the log (limit=${limit})`, transactionLog, dataView, logBuffer, position, limit, size, !!readUncommitted);
|
|
3673
|
+
position = broken.nextPosition;
|
|
3674
|
+
throw broken.error;
|
|
3675
|
+
}
|
|
2998
3676
|
position += TRANSACTION_LOG_ENTRY_HEADER_SIZE;
|
|
2999
3677
|
let matchesRange;
|
|
3000
3678
|
if (foundExactStart) matchesRange = (!exclusiveStart || timestamp !== start) && timestamp < end;
|
|
3001
|
-
else if (exactStart)
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3679
|
+
else if (exactStart) {
|
|
3680
|
+
if (timestamp === start) {
|
|
3681
|
+
matchesRange = !exclusiveStart;
|
|
3682
|
+
foundExactStart = true;
|
|
3683
|
+
} else matchesRange = false;
|
|
3684
|
+
} else matchesRange = (exclusiveStart ? timestamp > start : timestamp >= start) && timestamp < end;
|
|
3006
3685
|
const entryStart = position;
|
|
3007
3686
|
position += length;
|
|
3008
3687
|
if (matchesRange) return {
|
|
@@ -3085,9 +3764,9 @@ function loadLastPosition(transactionLog, readUncommitted) {
|
|
|
3085
3764
|
//#region src/index.ts
|
|
3086
3765
|
const versions = {
|
|
3087
3766
|
rocksdb: version,
|
|
3088
|
-
"rocksdb-js": "2.
|
|
3767
|
+
"rocksdb-js": "2.9.0"
|
|
3089
3768
|
};
|
|
3090
3769
|
|
|
3091
3770
|
//#endregion
|
|
3092
|
-
export { DBI, DBIterator, RocksDatabase, Store, Transaction, TransactionLog, backups, constants, coolTransactionLogs, currentThreadId, fileLockRelease, parseTransactionLog, registryStatus, shutdown, stats, supportedCompression, tryFileLock, validateTransactionLogStore, versions };
|
|
3771
|
+
export { BackgroundError, CorruptFrameError, CountEstimator, DBI, DBIterator, RocksDatabase, Store, Transaction, TransactionLog, backups, constants, coolTransactionLogs, currentThreadId, fileLockRelease, getWriteBufferManagerStats, parseTransactionLog, registryStatus, shutdown, stats, supportedCompression, tryFileLock, validateTransactionLogStore, versions };
|
|
3093
3772
|
//# sourceMappingURL=index.mjs.map
|