@cloudbitmaps/core 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +342 -62
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +229 -9
- package/dist/index.d.ts +229 -9
- package/dist/index.js +339 -63
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -122,6 +122,29 @@ declare function resolvePerOpBudget(opt: BudgetOption | undefined, storeBudget:
|
|
|
122
122
|
* (disabled).
|
|
123
123
|
*/
|
|
124
124
|
declare function checkBudget(budget: Budget | null, projectedRequests: number, op: string): void;
|
|
125
|
+
/**
|
|
126
|
+
* Drain an async iterable into an array, refusing **as soon as** the budget is exceeded.
|
|
127
|
+
*
|
|
128
|
+
* This exists because the obvious spelling is wrong in a way that is easy to miss:
|
|
129
|
+
*
|
|
130
|
+
* ```ts
|
|
131
|
+
* for await (const rec of registry.list(ns)) recs.push(rec);
|
|
132
|
+
* checkBudget(budget, recs.length, op); // ← too late; the memory is already spent
|
|
133
|
+
* ```
|
|
134
|
+
*
|
|
135
|
+
* That pattern bounds the **fan-out** but not the **enumeration**, so a tight budget refuses the work only
|
|
136
|
+
* after paying for the list — which defeats the purpose. It was measured buffering 20,000 registry records
|
|
137
|
+
* under `budget: { maxRequests: 2 }` before throwing. Draining incrementally bounds resident memory to
|
|
138
|
+
* `O(budget)` instead of `O(fleet)`.
|
|
139
|
+
*
|
|
140
|
+
* **The error message is deliberately vaguer than `checkBudget`'s.** It reports "more than N" rather than an
|
|
141
|
+
* exact total, because knowing the exact total requires finishing the scan — which is precisely the cost being
|
|
142
|
+
* refused. An honest approximation beats an expensive exact number.
|
|
143
|
+
*
|
|
144
|
+
* A `null` budget (`budget: false`) means the caller has explicitly opted out of fan-out limits, so this
|
|
145
|
+
* degrades to a plain drain.
|
|
146
|
+
*/
|
|
147
|
+
declare function collectWithinBudget<T>(source: AsyncIterable<T>, budget: Budget | null, op: string): Promise<T[]>;
|
|
125
148
|
|
|
126
149
|
/**
|
|
127
150
|
* The determinism seam.
|
|
@@ -143,6 +166,21 @@ interface Clock {
|
|
|
143
166
|
* replayable. `ms <= 0` resolves on a microtask without scheduling a timer.
|
|
144
167
|
*/
|
|
145
168
|
sleep(ms: number): Promise<void>;
|
|
169
|
+
/**
|
|
170
|
+
* Hand the event loop back once, without asking for any elapsed time.
|
|
171
|
+
*
|
|
172
|
+
* This is **not** `sleep(0)`, and the difference is the reason the member exists. `sleep(0)` resolves on a
|
|
173
|
+
* microtask, and microtasks drain before the loop advances a phase — so awaiting it inside a CPU-bound loop
|
|
174
|
+
* yields to nothing at all, and a co-resident server stays blocked for the loop's full duration. A real yield
|
|
175
|
+
* needs a macrotask. `sleep(1)` is one, but it buys the relief at ~1 ms of dead wall-clock per yield.
|
|
176
|
+
*
|
|
177
|
+
* **Optional**, so every `Clock` written before this member still satisfies the interface. Callers must
|
|
178
|
+
* therefore degrade rather than assume: `clock.yieldNow?.() ?? clock.sleep(1)` — correct on any clock, cheap
|
|
179
|
+
* on one that implements this. `core/` cannot supply it itself (it is timer-free, lint-enforced); production
|
|
180
|
+
* wiring backs it with `setImmediate`, and a simulator can make it a no-op so virtual time is not perturbed
|
|
181
|
+
* by what is purely a scheduling courtesy.
|
|
182
|
+
*/
|
|
183
|
+
yieldNow?(): Promise<void>;
|
|
146
184
|
}
|
|
147
185
|
/** Injected randomness. `core/` must never call `Math.random()` directly. Seedable for simulation. */
|
|
148
186
|
interface Rng {
|
|
@@ -230,7 +268,7 @@ declare class BoundedLru<K, V> {
|
|
|
230
268
|
* "Observability" section.
|
|
231
269
|
*/
|
|
232
270
|
/** The segment operations that emit an `op` latency event (timed with the injected clock, at the facade). */
|
|
233
|
-
type MetricOpName = 'add' | 'remove' | 'addMany' | 'removeMany' | 'has' | 'count' | 'intersectInto';
|
|
271
|
+
type MetricOpName = 'add' | 'remove' | 'addMany' | 'removeMany' | 'has' | 'count' | 'intersectInto' | 'unionInto' | 'andNotInto';
|
|
234
272
|
/**
|
|
235
273
|
* One observability event. A discriminated union on `kind` — new variants can be added over time without
|
|
236
274
|
* breaking consumers (they switch on the kinds they care about). `namespace` is the segment's namespace
|
|
@@ -272,8 +310,18 @@ type MetricEvent = {
|
|
|
272
310
|
readonly delayMs: number;
|
|
273
311
|
} | {
|
|
274
312
|
readonly kind: 'intersect';
|
|
313
|
+
/**
|
|
314
|
+
* Which chunk-aligned combine this was. **Optional for backward compatibility** — absent means
|
|
315
|
+
* `'intersect'`, which is all this event reported before `union`/`andNot` existed.
|
|
316
|
+
*
|
|
317
|
+
* `skippedChunks` is only ever non-zero for `'intersect'` and `'andNot'`: union has to read every chunk
|
|
318
|
+
* of every operand, so there is nothing it can prune. A steady stream of `op: 'union'` events with
|
|
319
|
+
* `skippedChunks: 0` is the expected shape, not a regression.
|
|
320
|
+
*/
|
|
321
|
+
readonly op?: 'intersect' | 'union' | 'andNot';
|
|
322
|
+
/** Operand count, including any `exclude` (suppression) operands. */
|
|
275
323
|
readonly operands: number;
|
|
276
|
-
/**
|
|
324
|
+
/** Distinct chunk-keys selected to fetch (for `'intersect'`, those present in every operand). */
|
|
277
325
|
readonly fetchedChunks: number;
|
|
278
326
|
/**
|
|
279
327
|
* **Distinct** chunk-keys pruned — never fetched (the chunk-skipping saving). Counts distinct keys,
|
|
@@ -430,6 +478,46 @@ declare function isTransient(err: unknown): boolean;
|
|
|
430
478
|
*/
|
|
431
479
|
declare function withRetry<T>(op: () => Promise<T>, policy: RetryPolicy, deps: RetryDeps): Promise<T>;
|
|
432
480
|
|
|
481
|
+
/**
|
|
482
|
+
* Default ceiling on resident warm-delta bytes for a single segment scan: 64 MiB.
|
|
483
|
+
*
|
|
484
|
+
* Chosen to be generous enough that no realistic segment trips it — a maximally-wide segment of 65,536 chunks
|
|
485
|
+
* would need ~1 KiB of warm delta per chunk to reach it — while still being small enough to keep a modest
|
|
486
|
+
* container (a 512 MiB Lambda, say) alive rather than OOM-killed when a pathological segment or a hostile
|
|
487
|
+
* caller shows up.
|
|
488
|
+
*/
|
|
489
|
+
declare const DEFAULT_MAX_WARM_SCAN_BYTES: number;
|
|
490
|
+
/**
|
|
491
|
+
* Default number of Warm chunk writes in flight per `addMany`/`removeMany`: **4**.
|
|
492
|
+
*
|
|
493
|
+
* The flusher was serial by default, which meant a 100-chunk `addMany` against a backend with ~10 ms of
|
|
494
|
+
* round-trip latency spent a full second doing nothing but waiting — one round-trip at a time, on work that has
|
|
495
|
+
* no ordering requirement between chunks. Distinct chunks are independent OCC rows.
|
|
496
|
+
*
|
|
497
|
+
* **Why 4 and not more.** The number is chosen against the *backend's* tolerance, not the client's appetite. A
|
|
498
|
+
* write here is a read-modify-write, so 4 in flight is up to 8 concurrent requests per `addMany` call — and a
|
|
499
|
+
* server handling many concurrent calls multiplies that again. Provisioned-capacity backends answer a burst
|
|
500
|
+
* with throttling, which is free only while retries can absorb it (4 attempts, exponential backoff, full
|
|
501
|
+
* jitter). Swept against a backend that throttles on concurrent requests in flight, 64 chunks, 5 trials each,
|
|
502
|
+
* counting a lost id or a surfaced error as a failure:
|
|
503
|
+
*
|
|
504
|
+
* ```text
|
|
505
|
+
* backend capacity: 1 2 4 8
|
|
506
|
+
* writeConcurrency 4: 0/5 0/5 0/5 0/5
|
|
507
|
+
* writeConcurrency 8: 0/5 0/5 0/5 0/5
|
|
508
|
+
* writeConcurrency 16: 0/5 0/5 0/5 0/5
|
|
509
|
+
* writeConcurrency 32: 3/5 0/5 0/5 0/5 ← retry budget exhausted
|
|
510
|
+
* ```
|
|
511
|
+
*
|
|
512
|
+
* So the retry machinery holds far past 4; 4 is deliberately conservative, sitting 8x below the first observed
|
|
513
|
+
* failure against the harshest backend modelled. The remaining headroom is spent on the multiplier this table
|
|
514
|
+
* does not capture — many concurrent `addMany` calls sharing one backend — rather than on more throughput per
|
|
515
|
+
* call. A caller who knows their capacity can raise it.
|
|
516
|
+
*
|
|
517
|
+
* Raise it if your backend is provisioned for it (on-demand DynamoDB, or a Postgres pool sized to match); set
|
|
518
|
+
* it to `1` for the previous strictly-serial behaviour.
|
|
519
|
+
*/
|
|
520
|
+
declare const DEFAULT_WRITE_CONCURRENCY = 4;
|
|
433
521
|
interface EngineDeps {
|
|
434
522
|
readonly warm: IWarmDriver;
|
|
435
523
|
readonly cold: ColdChunkSource;
|
|
@@ -458,11 +546,31 @@ interface EngineDeps {
|
|
|
458
546
|
*/
|
|
459
547
|
readonly warmReadConsistency?: 'strong' | 'eventual';
|
|
460
548
|
/**
|
|
461
|
-
* Max Warm chunk writes in flight per `addMany`/`removeMany` (the bounded flusher).
|
|
462
|
-
*
|
|
463
|
-
*
|
|
549
|
+
* Max Warm chunk writes in flight per `addMany`/`removeMany` (the bounded flusher).
|
|
550
|
+
* Default {@link DEFAULT_WRITE_CONCURRENCY}.
|
|
551
|
+
*
|
|
552
|
+
* Each chunk is its own OCC row, so distinct chunks never conflict with each other and fanning them out is
|
|
553
|
+
* safe. Raise it for more throughput against a backend with capacity to spare; set it to `1` for strictly
|
|
554
|
+
* serial writes.
|
|
555
|
+
*
|
|
556
|
+
* As before, `addMany`/`removeMany` are **not atomic**: a mid-flush failure can leave a partial result. What
|
|
557
|
+
* concurrency changes is only *how much* of the batch may already have landed when the first error surfaces —
|
|
558
|
+
* the flusher stops scheduling on the first failure, but writes already in flight still settle.
|
|
464
559
|
*/
|
|
465
560
|
readonly writeConcurrency?: number;
|
|
561
|
+
/**
|
|
562
|
+
* Hard ceiling on the bytes one warm scan may hold resident, in bytes. Default {@link DEFAULT_MAX_WARM_SCAN_BYTES}.
|
|
563
|
+
*
|
|
564
|
+
* **Separate from `budget`, and enforced even when `budget: false`.** The budget bounds *cost* — billable
|
|
565
|
+
* requests — which is a different axis from memory, and conflating the two is what let a segment with
|
|
566
|
+
* thousands of warm chunks materialise ~12 MB before a `maxRequests: 2` budget could refuse it. It is also
|
|
567
|
+
* why `intersect` cannot use the budget as a memory bound: its budget is `common keys × operands`, a product
|
|
568
|
+
* that one wide operand can legitimately exceed in row count while staying entirely legal.
|
|
569
|
+
*
|
|
570
|
+
* Always on, because a ceiling that `budget: false` switches off is missing at exactly the moment it is
|
|
571
|
+
* needed. Configurable, because a ceiling you cannot raise is a landmine for a legitimately large segment.
|
|
572
|
+
*/
|
|
573
|
+
readonly maxWarmScanBytes?: number;
|
|
466
574
|
/**
|
|
467
575
|
* Per-op denial-of-wallet budget (07 Decision #3 / T3), already resolved by the facade: a {@link Budget} to
|
|
468
576
|
* enforce, or `null` to disable. Undefined ⇒ {@link DEFAULT_BUDGET} (a direct engine construction still gets
|
|
@@ -485,13 +593,14 @@ declare class SegmentEngine {
|
|
|
485
593
|
/** Consistency for READ-path Warm fetches (precomputed; OCC RMW passes strong explicitly, see `warmGet`). */
|
|
486
594
|
private readonly readOpts;
|
|
487
595
|
private readonly writeConcurrency;
|
|
596
|
+
private readonly maxWarmScanBytes;
|
|
488
597
|
/** Resolved per-op budget (null = disabled); undefined deps ⇒ the generous default. See {@link checkBudget}. */
|
|
489
598
|
private readonly budget;
|
|
490
599
|
constructor(deps: EngineDeps);
|
|
491
600
|
add(seg: SegmentRef, id: number): Promise<void>;
|
|
492
601
|
remove(seg: SegmentRef, id: number): Promise<void>;
|
|
493
|
-
addMany(seg: SegmentRef, ids: Iterable<number>): Promise<void>;
|
|
494
|
-
removeMany(seg: SegmentRef, ids: Iterable<number>): Promise<void>;
|
|
602
|
+
addMany(seg: SegmentRef, ids: Iterable<number> | AsyncIterable<number>): Promise<void>;
|
|
603
|
+
removeMany(seg: SegmentRef, ids: Iterable<number> | AsyncIterable<number>): Promise<void>;
|
|
495
604
|
has(seg: SegmentRef, id: number, opts?: WarmReadOptions): Promise<boolean>;
|
|
496
605
|
count(seg: SegmentRef): Promise<number>;
|
|
497
606
|
/** Whether the Cold source can measure segment size (for grounded cost); false ⇒ storage isn't grounded. */
|
|
@@ -524,17 +633,51 @@ declare class SegmentEngine {
|
|
|
524
633
|
* re-resolves fresh (bypassing the TTL), which can hop generations even sub-TTL — still whole/immutable per
|
|
525
634
|
* read, never torn. Full intra-op snapshot isolation is deferred (gap #4).
|
|
526
635
|
*/
|
|
636
|
+
/**
|
|
637
|
+
* `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
|
|
638
|
+
*
|
|
639
|
+
* The `exclude` operands are what make suppression **compose without materialising**. Applying suppression
|
|
640
|
+
* afterwards (intersect into a temp segment, then remove) writes an intermediate segment that nobody wants;
|
|
641
|
+
* applying it here folds it into the same chunk-aligned pass. It is also cheaper than it looks: an exclude
|
|
642
|
+
* operand is read **only at the keys that survived the intersection**, so a 61,000-chunk global opt-out list
|
|
643
|
+
* intersected against a narrow audience costs a handful of reads, not 61,000.
|
|
644
|
+
*/
|
|
527
645
|
intersect(segs: readonly SegmentRef[], options?: {
|
|
528
646
|
concurrency?: number;
|
|
529
647
|
budget?: BudgetOption;
|
|
648
|
+
exclude?: readonly SegmentRef[];
|
|
649
|
+
}): AsyncGenerator<number>;
|
|
650
|
+
/**
|
|
651
|
+
* `segs[0] ∪ segs[1] ∪ …`, minus every id in `options.exclude` — streamed ascending.
|
|
652
|
+
*
|
|
653
|
+
* **Union is the one composite read with no chunk-skipping**, and that is a property of the operation rather
|
|
654
|
+
* than of this implementation: an id in *any* operand belongs to the result, so every chunk of every operand
|
|
655
|
+
* has to be read. `intersect` can prune a key absent from any operand; union cannot prune anything. It is
|
|
656
|
+
* budgeted exactly like `intersect` so a wide union is refused rather than quietly billed.
|
|
657
|
+
*/
|
|
658
|
+
union(segs: readonly SegmentRef[], options?: {
|
|
659
|
+
concurrency?: number;
|
|
660
|
+
budget?: BudgetOption;
|
|
661
|
+
exclude?: readonly SegmentRef[];
|
|
662
|
+
}): AsyncGenerator<number>;
|
|
663
|
+
/**
|
|
664
|
+
* `seg \ (excludes[0] ∪ excludes[1] ∪ …)` — streamed ascending. Suppression on its own.
|
|
665
|
+
*
|
|
666
|
+
* Reads every chunk of `seg` (any of them may survive) but each exclude **only where it overlaps `seg`**, so
|
|
667
|
+
* the cost scales with the segment being filtered rather than with the size of the suppression list.
|
|
668
|
+
*/
|
|
669
|
+
andNot(seg: SegmentRef, excludes: readonly SegmentRef[], options?: {
|
|
670
|
+
concurrency?: number;
|
|
671
|
+
budget?: BudgetOption;
|
|
530
672
|
}): AsyncGenerator<number>;
|
|
673
|
+
private combine;
|
|
531
674
|
/**
|
|
532
675
|
* The AND of one chunk key across all operands (tier-merged per operand), or `null` if empty. Operand
|
|
533
676
|
* chunks are fetched **in parallel** (the spec's parallel byte-range reads). `acc` takes ownership of the
|
|
534
677
|
* first operand's effective chunk and mutates it in place — safe because `effectiveChunk` always returns
|
|
535
678
|
* a FRESH bitmap (never a cached/shared Cold instance), so this can't poison the HOT cache.
|
|
536
679
|
*/
|
|
537
|
-
private
|
|
680
|
+
private combineChunk;
|
|
538
681
|
/**
|
|
539
682
|
* Materialize `seg[0] ∩ seg[1] ∩ …` into `dest` via batched `addMany` (streaming, bounded memory).
|
|
540
683
|
* **Not atomic** — like `addMany`, ids land chunk-by-chunk; a failure mid-stream leaves a partial result.
|
|
@@ -543,7 +686,58 @@ declare class SegmentEngine {
|
|
|
543
686
|
intersectInto(dest: SegmentRef, segs: readonly SegmentRef[], options?: {
|
|
544
687
|
concurrency?: number;
|
|
545
688
|
batchSize?: number;
|
|
689
|
+
budget?: BudgetOption;
|
|
690
|
+
exclude?: readonly SegmentRef[];
|
|
691
|
+
}): Promise<void>;
|
|
692
|
+
/**
|
|
693
|
+
* Materialize `segs[0] ∪ segs[1] ∪ …` (minus `options.exclude`) into `dest`. **Not atomic**, like
|
|
694
|
+
* {@link intersectInto}. See {@link union} for why this one cannot skip any chunk.
|
|
695
|
+
*/
|
|
696
|
+
unionInto(dest: SegmentRef, segs: readonly SegmentRef[], options?: {
|
|
697
|
+
concurrency?: number;
|
|
698
|
+
batchSize?: number;
|
|
699
|
+
budget?: BudgetOption;
|
|
700
|
+
exclude?: readonly SegmentRef[];
|
|
701
|
+
}): Promise<void>;
|
|
702
|
+
/** Materialize `seg \ (excludes…)` into `dest`. **Not atomic**, like {@link intersectInto}. */
|
|
703
|
+
andNotInto(dest: SegmentRef, seg: SegmentRef, excludes: readonly SegmentRef[], options?: {
|
|
704
|
+
concurrency?: number;
|
|
705
|
+
batchSize?: number;
|
|
706
|
+
budget?: BudgetOption;
|
|
546
707
|
}): Promise<void>;
|
|
708
|
+
/**
|
|
709
|
+
* Stream a combine's results into `dest` in bounded batches.
|
|
710
|
+
*
|
|
711
|
+
* `dest` is **added to, not replaced**, and this is **not atomic** — ids land chunk-by-chunk, so a failure
|
|
712
|
+
* mid-stream leaves a partial result, exactly as `addMany` does. Batching keeps memory bounded no matter how
|
|
713
|
+
* large the result is.
|
|
714
|
+
*/
|
|
715
|
+
private drainInto;
|
|
716
|
+
/**
|
|
717
|
+
* Group ids by chunk, then apply each chunk's whole batch in a single OCC read-modify-write.
|
|
718
|
+
*
|
|
719
|
+
* Accepts a **sync or async** iterable, so an Athena/BigQuery cursor streams straight in without the caller
|
|
720
|
+
* hand-batching `page → addMany(page)`.
|
|
721
|
+
*
|
|
722
|
+
* **The pending ids are held as BITMAPS, not as JS numbers, and that is the load-bearing choice.** Streaming
|
|
723
|
+
* input is unbounded by definition, so something has to give. The obvious answer — buffer N ids, flush them to
|
|
724
|
+
* the backend, repeat — bounds memory and is quietly catastrophic on cost: with ids arriving in arbitrary
|
|
725
|
+
* order, every flush touches nearly every chunk again, so an 11M-id stream at a 1M-id buffer would issue
|
|
726
|
+
* **11x** the OCC round-trips of a single pass. In a library whose headline claim is honest cost, silently
|
|
727
|
+
* multiplying someone's write bill by the size of their input is not a bounded-memory strategy; it is a
|
|
728
|
+
* different bug.
|
|
729
|
+
*
|
|
730
|
+
* So the staging buffer is capped, but it folds into per-chunk **compressed bitmaps** rather than being
|
|
731
|
+
* written out. Peak memory is the roaring representation of the ids seen so far — which is what the segment
|
|
732
|
+
* itself costs, and the floor for any implementation that writes each chunk exactly once — while the backend
|
|
733
|
+
* still sees exactly one read-modify-write per distinct chunk regardless of stream length. This is strictly
|
|
734
|
+
* less memory than the previous sync-only version, which held every remainder as an uncompressed JS number.
|
|
735
|
+
*
|
|
736
|
+
* There is no way to do better without a sorted stream: a chunk can only be written once it is known that no
|
|
737
|
+
* further ids for it will arrive, and unsorted input never gives that guarantee. Callers streaming *very*
|
|
738
|
+
* large sets want `bulkLoadCrbmGeneration` instead — it writes one immutable Cold object rather than tens of
|
|
739
|
+
* thousands of warm rows. The crossover is documented in the guide.
|
|
740
|
+
*/
|
|
547
741
|
private mutateMany;
|
|
548
742
|
/** OCC read-modify-write on a chunk's Warm delta, retrying on conflict (bounded). */
|
|
549
743
|
private readModifyWrite;
|
|
@@ -1510,6 +1704,7 @@ declare function writeCrbmGeneration(driver: IColdDriver, key: GenKey, chunks: I
|
|
|
1510
1704
|
bitmap: CodecBitmap;
|
|
1511
1705
|
}>, options?: {
|
|
1512
1706
|
crypto?: CrbmCrypto;
|
|
1707
|
+
clock?: Clock;
|
|
1513
1708
|
}): Promise<{
|
|
1514
1709
|
size: number;
|
|
1515
1710
|
sha256: string;
|
|
@@ -1564,6 +1759,16 @@ declare function bulkLoadCrbmGeneration(driver: IColdDriver, key: GenKey, ids: I
|
|
|
1564
1759
|
audit?: IAuditSink;
|
|
1565
1760
|
/** Bitmap codec. Optional in the type; a **flavor** package binds it ({@link requireCodec}). */
|
|
1566
1761
|
codec?: CodecInterface;
|
|
1762
|
+
/**
|
|
1763
|
+
* Injected clock. Supplying one makes bulk-load **cooperative**: it yields the event loop periodically so a
|
|
1764
|
+
* long load does not stall everything else on the process. Without it the load still completes, just
|
|
1765
|
+
* without yielding — which is the pre-existing behaviour, kept so this is purely additive.
|
|
1766
|
+
*
|
|
1767
|
+
* `@cloudbitmaps/roaring` supplies a real clock by default, so flavor users get cooperative behaviour with
|
|
1768
|
+
* no wiring. `core/` cannot default it: it is timer-free by lint, which is exactly why waiting goes through
|
|
1769
|
+
* this seam rather than `setTimeout`.
|
|
1770
|
+
*/
|
|
1771
|
+
clock?: Clock;
|
|
1567
1772
|
}): Promise<BulkLoadResult>;
|
|
1568
1773
|
|
|
1569
1774
|
/**
|
|
@@ -1791,6 +1996,20 @@ declare function eraseNamespace(namespace: string, deps: EraseDeps, options: {
|
|
|
1791
1996
|
* to confirm a reported tear.
|
|
1792
1997
|
*/
|
|
1793
1998
|
|
|
1999
|
+
/** Default in-flight fan-out for the consistency scan — bounded, no thundering herd. */
|
|
2000
|
+
/**
|
|
2001
|
+
* Default ceiling on how many registry records one consistency check may hold resident: 250,000.
|
|
2002
|
+
*
|
|
2003
|
+
* **Named `maxScanSegments`, not `maxSegments`, on purpose.** Compaction already has a `maxSegments`
|
|
2004
|
+
* (`CR_COMPACT_MAX_SEGMENTS`) that caps how much work one cycle does and then *continues normally*. This one
|
|
2005
|
+
* caps how much a scan may hold and *refuses* past it. Same-sounding names with opposite behaviour at the
|
|
2006
|
+
* limit is a trap; pairing this with `maxWarmScanBytes` instead makes both read as what they are — ceilings on
|
|
2007
|
+
* a scan that fail closed.
|
|
2008
|
+
*
|
|
2009
|
+
* Generous — the compaction docs target 100K+ segment fleets — while still bounding a DR drill's memory to
|
|
2010
|
+
* something a modest operator box survives. Raisable, because a ceiling you cannot lift is a landmine.
|
|
2011
|
+
*/
|
|
2012
|
+
declare const DEFAULT_MAX_SCAN_SEGMENTS = 250000;
|
|
1794
2013
|
interface ConsistencyIssue {
|
|
1795
2014
|
readonly segment: string;
|
|
1796
2015
|
readonly namespace?: string;
|
|
@@ -1830,6 +2049,7 @@ declare function runConsistencyCheck(deps: {
|
|
|
1830
2049
|
}, options?: {
|
|
1831
2050
|
namespace?: string;
|
|
1832
2051
|
concurrency?: number;
|
|
2052
|
+
maxScanSegments?: number;
|
|
1833
2053
|
}): Promise<ConsistencyReport>;
|
|
1834
2054
|
|
|
1835
2055
|
/**
|
|
@@ -1963,4 +2183,4 @@ declare class CrbmWriter {
|
|
|
1963
2183
|
private buildFooter;
|
|
1964
2184
|
}
|
|
1965
2185
|
|
|
1966
|
-
export { AWS_US_EAST_1_ONDEMAND, Aead, AeadSealed, type AuditEvent, type AuditEventKind, BlobReader, BlobSink, BoundedLru, type Budget, BudgetExceededError, type BudgetOption, type BulkLoadResult, CapabilityError, ChunkRef, type Clock, CloudRoaringError, type CodecBitmap, type CodecInterface, ColdCaps, ColdChunkSource, type CompactionCandidate, type CompactionCycleResult, type CompactionDeps, type CompactionOptions, type CompactionResult, type ConsistencyErrorEntry, type ConsistencyIssue, type ConsistencyReport, type CostAdvisory, type CostReport, CountingMetricsSink, CrbmColdChunkSource, type CrbmColdChunkSourceOptions, CrbmCrypto, CrbmReader, type CrbmReaderOptions, CrbmWriter, type CrbmWriterOptions, DEFAULT_BUDGET, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETRY_POLICY, type DestroyResult, type DiscoveryOptions, type EngineDeps, type EraseDeps, type EstimateInput, type ExportFailure, type ExportFormat, type ExportManifest, type ExportOptions, type ExportSink, type ExportWriter, type ExportedSegment, GenKey, type IAuditSink, IColdDriver, IKeystore, type IMetricsSink, IRegistryDriver, IWarmDriver, InProcessKeystore, type InProcessKeystoreOptions, IntegrityError, KeyUnavailableError, LocalFsColdDriver, LocalFsRegistryDriver, type LocalFsRegistryDriverOptions, LocalFsWarmDriver, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, type MemoryRegistryDriverOptions, MemoryWarmDriver, type MetricEvent, type MetricOpName, type MetricsSnapshot, NOOP_AUDIT, NOOP_METRICS, NewRegistryRecord, NoRow, NodeAead, NotFoundError, type PricingProfile, RecordingAuditSink, RegCaps, RegistryPatch, RegistryRecord, type RetryDeps, type RetryPolicy, RetryingColdChunkSource, RetryingColdDriver, type RetryingOptions, RetryingRegistryDriver, RetryingWarmDriver, type Rng, SegmentEngine, SegmentRef, SegmentSize, type SegmentSizing, TimeoutError, Token, type Topology, TransientError, UnsupportedError, ValidationError, WarmReadOptions, WarmRow, type Workload, WrappedDek, WriteConflictError, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, compactSegment, destroySegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, groundedReport, isCloudRoaringError, isIntegrityError, isNotFoundError, isTransient, isTransientError, isValidationError, isWriteConflictError, joinId, mapWithConcurrency, publishGeneration, resolveBudget, resolvePerOpBudget, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, splitId, validateCompactionOptions, validateSegmentRef, withRetry, writeCrbmGeneration };
|
|
2186
|
+
export { AWS_US_EAST_1_ONDEMAND, Aead, AeadSealed, type AuditEvent, type AuditEventKind, BlobReader, BlobSink, BoundedLru, type Budget, BudgetExceededError, type BudgetOption, type BulkLoadResult, CapabilityError, ChunkRef, type Clock, CloudRoaringError, type CodecBitmap, type CodecInterface, ColdCaps, ColdChunkSource, type CompactionCandidate, type CompactionCycleResult, type CompactionDeps, type CompactionOptions, type CompactionResult, type ConsistencyErrorEntry, type ConsistencyIssue, type ConsistencyReport, type CostAdvisory, type CostReport, CountingMetricsSink, CrbmColdChunkSource, type CrbmColdChunkSourceOptions, CrbmCrypto, CrbmReader, type CrbmReaderOptions, CrbmWriter, type CrbmWriterOptions, DEFAULT_BUDGET, DEFAULT_MAX_SCAN_SEGMENTS, DEFAULT_MAX_WARM_SCAN_BYTES, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETRY_POLICY, DEFAULT_WRITE_CONCURRENCY, type DestroyResult, type DiscoveryOptions, type EngineDeps, type EraseDeps, type EstimateInput, type ExportFailure, type ExportFormat, type ExportManifest, type ExportOptions, type ExportSink, type ExportWriter, type ExportedSegment, GenKey, type IAuditSink, IColdDriver, IKeystore, type IMetricsSink, IRegistryDriver, IWarmDriver, InProcessKeystore, type InProcessKeystoreOptions, IntegrityError, KeyUnavailableError, LocalFsColdDriver, LocalFsRegistryDriver, type LocalFsRegistryDriverOptions, LocalFsWarmDriver, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, type MemoryRegistryDriverOptions, MemoryWarmDriver, type MetricEvent, type MetricOpName, type MetricsSnapshot, NOOP_AUDIT, NOOP_METRICS, NewRegistryRecord, NoRow, NodeAead, NotFoundError, type PricingProfile, RecordingAuditSink, RegCaps, RegistryPatch, RegistryRecord, type RetryDeps, type RetryPolicy, RetryingColdChunkSource, RetryingColdDriver, type RetryingOptions, RetryingRegistryDriver, RetryingWarmDriver, type Rng, SegmentEngine, SegmentRef, SegmentSize, type SegmentSizing, TimeoutError, Token, type Topology, TransientError, UnsupportedError, ValidationError, WarmReadOptions, WarmRow, type Workload, WrappedDek, WriteConflictError, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, collectWithinBudget, compactSegment, destroySegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, groundedReport, isCloudRoaringError, isIntegrityError, isNotFoundError, isTransient, isTransientError, isValidationError, isWriteConflictError, joinId, mapWithConcurrency, publishGeneration, resolveBudget, resolvePerOpBudget, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, splitId, validateCompactionOptions, validateSegmentRef, withRetry, writeCrbmGeneration };
|