@cloudbitmaps/core 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +296 -56
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +260 -11
- package/dist/index.d.ts +260 -11
- package/dist/index.js +296 -57
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -166,6 +166,21 @@ interface Clock {
|
|
|
166
166
|
* replayable. `ms <= 0` resolves on a microtask without scheduling a timer.
|
|
167
167
|
*/
|
|
168
168
|
sleep(ms: number): Promise<void>;
|
|
169
|
+
/**
|
|
170
|
+
* Hand the event loop back once, without asking for any elapsed time.
|
|
171
|
+
*
|
|
172
|
+
* This is **not** `sleep(0)`, and the difference is the reason the member exists. `sleep(0)` resolves on a
|
|
173
|
+
* microtask, and microtasks drain before the loop advances a phase — so awaiting it inside a CPU-bound loop
|
|
174
|
+
* yields to nothing at all, and a co-resident server stays blocked for the loop's full duration. A real yield
|
|
175
|
+
* needs a macrotask. `sleep(1)` is one, but it buys the relief at ~1 ms of dead wall-clock per yield.
|
|
176
|
+
*
|
|
177
|
+
* **Optional**, so every `Clock` written before this member still satisfies the interface. Callers must
|
|
178
|
+
* therefore degrade rather than assume: `clock.yieldNow?.() ?? clock.sleep(1)` — correct on any clock, cheap
|
|
179
|
+
* on one that implements this. `core/` cannot supply it itself (it is timer-free, lint-enforced); production
|
|
180
|
+
* wiring backs it with `setImmediate`, and a simulator can make it a no-op so virtual time is not perturbed
|
|
181
|
+
* by what is purely a scheduling courtesy.
|
|
182
|
+
*/
|
|
183
|
+
yieldNow?(): Promise<void>;
|
|
169
184
|
}
|
|
170
185
|
/** Injected randomness. `core/` must never call `Math.random()` directly. Seedable for simulation. */
|
|
171
186
|
interface Rng {
|
|
@@ -253,7 +268,7 @@ declare class BoundedLru<K, V> {
|
|
|
253
268
|
* "Observability" section.
|
|
254
269
|
*/
|
|
255
270
|
/** The segment operations that emit an `op` latency event (timed with the injected clock, at the facade). */
|
|
256
|
-
type MetricOpName = 'add' | 'remove' | 'addMany' | 'removeMany' | 'has' | 'count' | 'intersectInto';
|
|
271
|
+
type MetricOpName = 'add' | 'remove' | 'addMany' | 'removeMany' | 'has' | 'count' | 'intersectInto' | 'unionInto' | 'andNotInto';
|
|
257
272
|
/**
|
|
258
273
|
* One observability event. A discriminated union on `kind` — new variants can be added over time without
|
|
259
274
|
* breaking consumers (they switch on the kinds they care about). `namespace` is the segment's namespace
|
|
@@ -295,8 +310,26 @@ type MetricEvent = {
|
|
|
295
310
|
readonly delayMs: number;
|
|
296
311
|
} | {
|
|
297
312
|
readonly kind: 'intersect';
|
|
313
|
+
/**
|
|
314
|
+
* Which chunk-aligned combine this was. **Optional for backward compatibility** — absent means
|
|
315
|
+
* `'intersect'`, which is all this event reported before `union`/`andNot` existed.
|
|
316
|
+
*
|
|
317
|
+
* For `'union'` over its *include* operands alone, `skippedChunks` is 0 by construction — union reads
|
|
318
|
+
* every chunk of every operand, so there is nothing to prune. It can still be non-zero when a union
|
|
319
|
+
* carries `exclude` operands, because a suppression list may hold keys no include does and those are
|
|
320
|
+
* legitimately never fetched. So: 0 is the expected shape for a plain union, not a guarantee.
|
|
321
|
+
*/
|
|
322
|
+
readonly op?: 'intersect' | 'union' | 'andNot';
|
|
323
|
+
/** Operand count, including any `exclude` (suppression) operands. */
|
|
298
324
|
readonly operands: number;
|
|
299
|
-
/**
|
|
325
|
+
/**
|
|
326
|
+
* Distinct chunk **keys** selected to fetch — for `'intersect'`, those present in every operand.
|
|
327
|
+
*
|
|
328
|
+
* **Keys, not requests.** The actual cold GETs are roughly `fetchedChunks × operands`, plus one per
|
|
329
|
+
* `exclude` that holds a given key, which is why this number is smaller than what the per-op budget
|
|
330
|
+
* charges for the same call. The two are different units by design; if you are reconciling a bill, the
|
|
331
|
+
* budget's accounting is the one that models requests.
|
|
332
|
+
*/
|
|
300
333
|
readonly fetchedChunks: number;
|
|
301
334
|
/**
|
|
302
335
|
* **Distinct** chunk-keys pruned — never fetched (the chunk-skipping saving). Counts distinct keys,
|
|
@@ -462,6 +495,37 @@ declare function withRetry<T>(op: () => Promise<T>, policy: RetryPolicy, deps: R
|
|
|
462
495
|
* caller shows up.
|
|
463
496
|
*/
|
|
464
497
|
declare const DEFAULT_MAX_WARM_SCAN_BYTES: number;
|
|
498
|
+
/**
|
|
499
|
+
* Default number of Warm chunk writes in flight per `addMany`/`removeMany`: **4**.
|
|
500
|
+
*
|
|
501
|
+
* The flusher was serial by default, which meant a 100-chunk `addMany` against a backend with ~10 ms of
|
|
502
|
+
* round-trip latency spent a full second doing nothing but waiting — one round-trip at a time, on work that has
|
|
503
|
+
* no ordering requirement between chunks. Distinct chunks are independent OCC rows.
|
|
504
|
+
*
|
|
505
|
+
* **Why 4 and not more.** The number is chosen against the *backend's* tolerance, not the client's appetite. A
|
|
506
|
+
* write here is a read-modify-write, so 4 in flight is up to 8 concurrent requests per `addMany` call — and a
|
|
507
|
+
* server handling many concurrent calls multiplies that again. Provisioned-capacity backends answer a burst
|
|
508
|
+
* with throttling, which is free only while retries can absorb it (4 attempts, exponential backoff, full
|
|
509
|
+
* jitter). Swept against a **modelled** backend — an in-process fake that throttles on concurrent requests in
|
|
510
|
+
* flight, not a real DynamoDB — 64 chunks, 5 trials each, counting a lost id or a surfaced error as a failure:
|
|
511
|
+
*
|
|
512
|
+
* ```text
|
|
513
|
+
* backend capacity: 1 2 4 8
|
|
514
|
+
* writeConcurrency 4: 0/5 0/5 0/5 0/5
|
|
515
|
+
* writeConcurrency 8: 0/5 0/5 0/5 0/5
|
|
516
|
+
* writeConcurrency 16: 0/5 0/5 0/5 0/5
|
|
517
|
+
* writeConcurrency 32: 3/5 0/5 0/5 0/5 ← retry budget exhausted
|
|
518
|
+
* ```
|
|
519
|
+
*
|
|
520
|
+
* So the retry machinery holds far past 4; 4 is deliberately conservative, sitting 8x below the first observed
|
|
521
|
+
* failure against the harshest backend modelled. The remaining headroom is spent on the multiplier this table
|
|
522
|
+
* does not capture — many concurrent `addMany` calls sharing one backend — rather than on more throughput per
|
|
523
|
+
* call. A caller who knows their capacity can raise it.
|
|
524
|
+
*
|
|
525
|
+
* Raise it if your backend is provisioned for it (on-demand DynamoDB, or a Postgres pool sized to match); set
|
|
526
|
+
* it to `1` for the previous strictly-serial behaviour.
|
|
527
|
+
*/
|
|
528
|
+
declare const DEFAULT_WRITE_CONCURRENCY = 4;
|
|
465
529
|
interface EngineDeps {
|
|
466
530
|
readonly warm: IWarmDriver;
|
|
467
531
|
readonly cold: ColdChunkSource;
|
|
@@ -490,9 +554,16 @@ interface EngineDeps {
|
|
|
490
554
|
*/
|
|
491
555
|
readonly warmReadConsistency?: 'strong' | 'eventual';
|
|
492
556
|
/**
|
|
493
|
-
* Max Warm chunk writes in flight per `addMany`/`removeMany` (the bounded flusher).
|
|
494
|
-
*
|
|
495
|
-
*
|
|
557
|
+
* Max Warm chunk writes in flight per `addMany`/`removeMany` (the bounded flusher).
|
|
558
|
+
* Default {@link DEFAULT_WRITE_CONCURRENCY}.
|
|
559
|
+
*
|
|
560
|
+
* Each chunk is its own OCC row, so distinct chunks never conflict with each other and fanning them out is
|
|
561
|
+
* safe. Raise it for more throughput against a backend with capacity to spare; set it to `1` for strictly
|
|
562
|
+
* serial writes.
|
|
563
|
+
*
|
|
564
|
+
* As before, `addMany`/`removeMany` are **not atomic**: a mid-flush failure can leave a partial result. What
|
|
565
|
+
* concurrency changes is only *how much* of the batch may already have landed when the first error surfaces —
|
|
566
|
+
* the flusher stops scheduling on the first failure, but writes already in flight still settle.
|
|
496
567
|
*/
|
|
497
568
|
readonly writeConcurrency?: number;
|
|
498
569
|
/**
|
|
@@ -536,8 +607,8 @@ declare class SegmentEngine {
|
|
|
536
607
|
constructor(deps: EngineDeps);
|
|
537
608
|
add(seg: SegmentRef, id: number): Promise<void>;
|
|
538
609
|
remove(seg: SegmentRef, id: number): Promise<void>;
|
|
539
|
-
addMany(seg: SegmentRef, ids: Iterable<number>): Promise<void>;
|
|
540
|
-
removeMany(seg: SegmentRef, ids: Iterable<number>): Promise<void>;
|
|
610
|
+
addMany(seg: SegmentRef, ids: Iterable<number> | AsyncIterable<number>): Promise<void>;
|
|
611
|
+
removeMany(seg: SegmentRef, ids: Iterable<number> | AsyncIterable<number>): Promise<void>;
|
|
541
612
|
has(seg: SegmentRef, id: number, opts?: WarmReadOptions): Promise<boolean>;
|
|
542
613
|
count(seg: SegmentRef): Promise<number>;
|
|
543
614
|
/** Whether the Cold source can measure segment size (for grounded cost); false ⇒ storage isn't grounded. */
|
|
@@ -546,6 +617,17 @@ declare class SegmentEngine {
|
|
|
546
617
|
segmentSize(seg: SegmentRef): Promise<SegmentSize | null>;
|
|
547
618
|
iterate(seg: SegmentRef): AsyncGenerator<number>;
|
|
548
619
|
/**
|
|
620
|
+
* `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
|
|
621
|
+
*
|
|
622
|
+
* The `exclude` operands are what make suppression **compose without materialising**. Applying suppression
|
|
623
|
+
* afterwards (intersect into a temp segment, then remove) writes an intermediate segment that nobody wants;
|
|
624
|
+
* applying it here folds it into the same chunk-aligned pass. It is also cheaper than it looks: an exclude
|
|
625
|
+
* operand is read **only at the keys that survived the intersection** — the cost is bounded by the
|
|
626
|
+
* audience's surviving key count and **never scales with the size of the suppression list**. Subtracting a
|
|
627
|
+
* 61,000-chunk global opt-out list costs at most one read per surviving key, not 61,000.
|
|
628
|
+
*
|
|
629
|
+
* ---
|
|
630
|
+
*
|
|
549
631
|
* Chunk-skipping intersection. Computes
|
|
550
632
|
* `seg[0] ∩ seg[1] ∩ …` and streams the result ids ascending. It aligns each segment's *effective*
|
|
551
633
|
* chunk-key set (Warm ∪ Cold), keeps only keys present in **all** operands (a key missing from any
|
|
@@ -573,14 +655,39 @@ declare class SegmentEngine {
|
|
|
573
655
|
intersect(segs: readonly SegmentRef[], options?: {
|
|
574
656
|
concurrency?: number;
|
|
575
657
|
budget?: BudgetOption;
|
|
658
|
+
exclude?: readonly SegmentRef[];
|
|
659
|
+
}): AsyncGenerator<number>;
|
|
660
|
+
/**
|
|
661
|
+
* `segs[0] ∪ segs[1] ∪ …`, minus every id in `options.exclude` — streamed ascending.
|
|
662
|
+
*
|
|
663
|
+
* **Union is the one composite read with no chunk-skipping**, and that is a property of the operation rather
|
|
664
|
+
* than of this implementation: an id in *any* operand belongs to the result, so every chunk of every operand
|
|
665
|
+
* has to be read. `intersect` can prune a key absent from any operand; union cannot prune anything. It is
|
|
666
|
+
* budgeted exactly like `intersect` so a wide union is refused rather than quietly billed.
|
|
667
|
+
*/
|
|
668
|
+
union(segs: readonly SegmentRef[], options?: {
|
|
669
|
+
concurrency?: number;
|
|
670
|
+
budget?: BudgetOption;
|
|
671
|
+
exclude?: readonly SegmentRef[];
|
|
672
|
+
}): AsyncGenerator<number>;
|
|
673
|
+
/**
|
|
674
|
+
* `seg \ (excludes[0] ∪ excludes[1] ∪ …)` — streamed ascending. Suppression on its own.
|
|
675
|
+
*
|
|
676
|
+
* Reads every chunk of `seg` (any of them may survive) but each exclude **only where it overlaps `seg`**, so
|
|
677
|
+
* the cost scales with the segment being filtered rather than with the size of the suppression list.
|
|
678
|
+
*/
|
|
679
|
+
andNot(seg: SegmentRef, excludes: readonly SegmentRef[], options?: {
|
|
680
|
+
concurrency?: number;
|
|
681
|
+
budget?: BudgetOption;
|
|
576
682
|
}): AsyncGenerator<number>;
|
|
683
|
+
private combine;
|
|
577
684
|
/**
|
|
578
685
|
* The AND of one chunk key across all operands (tier-merged per operand), or `null` if empty. Operand
|
|
579
686
|
* chunks are fetched **in parallel** (the spec's parallel byte-range reads). `acc` takes ownership of the
|
|
580
687
|
* first operand's effective chunk and mutates it in place — safe because `effectiveChunk` always returns
|
|
581
688
|
* a FRESH bitmap (never a cached/shared Cold instance), so this can't poison the HOT cache.
|
|
582
689
|
*/
|
|
583
|
-
private
|
|
690
|
+
private combineChunk;
|
|
584
691
|
/**
|
|
585
692
|
* Materialize `seg[0] ∩ seg[1] ∩ …` into `dest` via batched `addMany` (streaming, bounded memory).
|
|
586
693
|
* **Not atomic** — like `addMany`, ids land chunk-by-chunk; a failure mid-stream leaves a partial result.
|
|
@@ -589,7 +696,60 @@ declare class SegmentEngine {
|
|
|
589
696
|
intersectInto(dest: SegmentRef, segs: readonly SegmentRef[], options?: {
|
|
590
697
|
concurrency?: number;
|
|
591
698
|
batchSize?: number;
|
|
699
|
+
budget?: BudgetOption;
|
|
700
|
+
exclude?: readonly SegmentRef[];
|
|
592
701
|
}): Promise<void>;
|
|
702
|
+
/**
|
|
703
|
+
* Materialize `segs[0] ∪ segs[1] ∪ …` (minus `options.exclude`) into `dest`. **Not atomic**, like
|
|
704
|
+
* {@link intersectInto}. See {@link union} for why this one cannot skip any chunk.
|
|
705
|
+
*/
|
|
706
|
+
unionInto(dest: SegmentRef, segs: readonly SegmentRef[], options?: {
|
|
707
|
+
concurrency?: number;
|
|
708
|
+
batchSize?: number;
|
|
709
|
+
budget?: BudgetOption;
|
|
710
|
+
exclude?: readonly SegmentRef[];
|
|
711
|
+
}): Promise<void>;
|
|
712
|
+
/** Materialize `seg \ (excludes…)` into `dest`. **Not atomic**, like {@link intersectInto}. */
|
|
713
|
+
andNotInto(dest: SegmentRef, seg: SegmentRef, excludes: readonly SegmentRef[], options?: {
|
|
714
|
+
concurrency?: number;
|
|
715
|
+
batchSize?: number;
|
|
716
|
+
budget?: BudgetOption;
|
|
717
|
+
}): Promise<void>;
|
|
718
|
+
/**
|
|
719
|
+
* Stream a combine's results into `dest` in bounded batches.
|
|
720
|
+
*
|
|
721
|
+
* `dest` is **added to, not replaced**, and this is **not atomic** — ids land chunk-by-chunk, so a failure
|
|
722
|
+
* mid-stream leaves a partial result, exactly as `addMany` does. Batching keeps memory bounded no matter how
|
|
723
|
+
* large the result is.
|
|
724
|
+
*/
|
|
725
|
+
private drainInto;
|
|
726
|
+
/**
|
|
727
|
+
* Group ids by chunk, then apply each chunk's whole batch in a single OCC read-modify-write.
|
|
728
|
+
*
|
|
729
|
+
* Accepts a **sync or async** iterable, so an Athena/BigQuery cursor streams straight in without the caller
|
|
730
|
+
* hand-batching `page → addMany(page)`.
|
|
731
|
+
*
|
|
732
|
+
* **The pending ids are held as BITMAPS, not as JS numbers, and that is the load-bearing choice.** Streaming
|
|
733
|
+
* input is unbounded by definition, so something has to give. The obvious answer — buffer N ids, flush them to
|
|
734
|
+
* the backend, repeat — bounds memory and is quietly catastrophic on cost: with ids arriving in arbitrary
|
|
735
|
+
* order, every flush touches nearly every chunk again, so an 11M-id stream at a 1M-id buffer would issue
|
|
736
|
+
* **11x** the OCC round-trips of a single pass. In a library whose headline claim is honest cost, silently
|
|
737
|
+
* multiplying someone's write bill by the size of their input is not a bounded-memory strategy; it is a
|
|
738
|
+
* different bug.
|
|
739
|
+
*
|
|
740
|
+
* So the staging buffer is capped, but it folds into per-chunk **compressed bitmaps** rather than being
|
|
741
|
+
* written out. Peak memory is the roaring representation of the ids seen so far — which is what the segment
|
|
742
|
+
* itself costs, and the floor for any implementation that writes each chunk exactly once — while the backend
|
|
743
|
+
* still sees exactly one read-modify-write per distinct chunk regardless of stream length. Above the staging cap
|
|
744
|
+
* this holds less than the previous sync-only version, which kept every remainder as an uncompressed JS
|
|
745
|
+
* number; below it — the common `addMany` — nothing is compacted until the end and the two are comparable.
|
|
746
|
+
* The guarantee is that a stream of *any* length stays bounded, not that every call got smaller.
|
|
747
|
+
*
|
|
748
|
+
* There is no way to do better without a sorted stream: a chunk can only be written once it is known that no
|
|
749
|
+
* further ids for it will arrive, and unsorted input never gives that guarantee. Callers streaming *very*
|
|
750
|
+
* large sets want `bulkLoadCrbmGeneration` instead — it writes one immutable Cold object rather than tens of
|
|
751
|
+
* thousands of warm rows. The crossover is documented in the guide.
|
|
752
|
+
*/
|
|
593
753
|
private mutateMany;
|
|
594
754
|
/** OCC read-modify-write on a chunk's Warm delta, retrying on conflict (bounded). */
|
|
595
755
|
private readModifyWrite;
|
|
@@ -907,8 +1067,16 @@ interface CompactionDeps {
|
|
|
907
1067
|
readonly cold: IColdDriver;
|
|
908
1068
|
readonly warm: IWarmDriver;
|
|
909
1069
|
readonly registry: IRegistryDriver;
|
|
910
|
-
/**
|
|
911
|
-
|
|
1070
|
+
/**
|
|
1071
|
+
* Injected time source for lease expiry (the determinism seam — `core/` never reads ambient time).
|
|
1072
|
+
*
|
|
1073
|
+
* `now` is required; `sleep`/`yieldNow` are **optional and used only for cooperative yielding** while writing
|
|
1074
|
+
* a generation. Supplying a full {@link Clock} (the facade does) makes compaction hand the event loop back
|
|
1075
|
+
* periodically instead of blocking it for the whole write; supplying a bare `{ now }` keeps the previous
|
|
1076
|
+
* uninterrupted behaviour. Optional rather than required because every existing caller passes `{ now }`, and
|
|
1077
|
+
* requiring more would be a breaking change — the reason the yield below shipped unreachable the first time.
|
|
1078
|
+
*/
|
|
1079
|
+
readonly clock: Pick<Clock, 'now'> & Partial<Pick<Clock, 'sleep' | 'yieldNow'>>;
|
|
912
1080
|
readonly maxBitmapBytes?: number;
|
|
913
1081
|
/**
|
|
914
1082
|
* Bitmap codec — compaction only merges bitmaps through it. Optional in the type so this
|
|
@@ -1367,6 +1535,56 @@ declare class LocalFsRegistryDriver implements IRegistryDriver {
|
|
|
1367
1535
|
private withRowLock;
|
|
1368
1536
|
}
|
|
1369
1537
|
|
|
1538
|
+
/**
|
|
1539
|
+
* Cooperative yielding for the long CPU-bound loops.
|
|
1540
|
+
*
|
|
1541
|
+
* Everything this library does on the hot read path is small. Bulk-load is the exception: it is the one place
|
|
1542
|
+
* where a single call legitimately occupies the CPU for hundreds of milliseconds, and Node has exactly one thread
|
|
1543
|
+
* to occupy. Measured on a 1M-id load, 62,455 chunks: **442 ms wall, and 450 ms during which the event loop did
|
|
1544
|
+
* not turn at all**. Wire that to a request handler and every other in-flight request on the instance — health
|
|
1545
|
+
* checks included — waits out the whole load.
|
|
1546
|
+
*
|
|
1547
|
+
* The fix is not to make the work asynchronous. That was measured too, and it is a 7x *regression*: handing each
|
|
1548
|
+
* chunk's insert to the threadpool costs ~9 µs of dispatch against ~1.5 µs of actual work once ids are spread
|
|
1549
|
+
* across ~62,000 chunks (636 ms versus 92 ms). The work is genuinely CPU-bound and genuinely small per chunk;
|
|
1550
|
+
* the only thing wrong with it is that it never stops to let anyone else go. So the fix is to keep the work
|
|
1551
|
+
* synchronous and periodically hand the loop back.
|
|
1552
|
+
*
|
|
1553
|
+
* TWO TRAPS, both of which make a plausible implementation measure as no change at all:
|
|
1554
|
+
*
|
|
1555
|
+
* 1. **`await Promise.resolve()` — and equally `Clock.sleep(0)` — do nothing.** Both resolve on a microtask, and
|
|
1556
|
+
* the microtask queue drains to empty before the event loop advances a single phase. A loop awaiting them
|
|
1557
|
+
* yields to other *microtasks* and to nothing else; no pending I/O callback runs. Measured: 555 ms of
|
|
1558
|
+
* starvation against a 568 ms unyielded baseline **on the synthetic yield-primitive benchmark below** —
|
|
1559
|
+
* a different, smaller workload than the 442 ms end-to-end load above, which is why the two baselines
|
|
1560
|
+
* differ. The yield must be a **macrotask** — hence
|
|
1561
|
+
* `Clock.yieldNow`, backed by `setImmediate` in production wiring.
|
|
1562
|
+
* 2. **Yielding per unit of work costs more than the work.** The yield has to be periodic, and `everyN` has to be
|
|
1563
|
+
* large enough that its cost disappears into the batch it interrupts.
|
|
1564
|
+
*
|
|
1565
|
+
* At `everyN = 1024` over that same synthetic workload the trade is close to free: 569 ms wall against a 568 ms
|
|
1566
|
+
* baseline, with the worst event-loop gap down from 568 ms to 13.8 ms.
|
|
1567
|
+
*
|
|
1568
|
+
* **Three experiments, three sets of numbers — do not mix them.** (a) The end-to-end 1M-id bulk load:
|
|
1569
|
+
* 442 ms wall / 450 ms blocked before, 256 ms / 19 ms after. (b) This synthetic yield-primitive comparison:
|
|
1570
|
+
* 568 ms / 568 ms unyielded, 569 ms / 13.8 ms with `setImmediate`. (c) A per-chunk insert microbenchmark:
|
|
1571
|
+
* 92 ms sync versus 636 ms via the threadpool. Quoting a baseline from one against a result from another is
|
|
1572
|
+
* how this file previously published a 92 ms operation with 819 ms of starvation inside it.
|
|
1573
|
+
*/
|
|
1574
|
+
/**
|
|
1575
|
+
* The only part of a `Clock` that yielding actually needs.
|
|
1576
|
+
*
|
|
1577
|
+
* Deliberately looser than `Clock` so the compaction path can opt in: `CompactionDeps.clock` is declared as
|
|
1578
|
+
* `Pick<Clock, 'now'>` plus optional yield members, because its callers (including the `compact-segments` CLI)
|
|
1579
|
+
* have always supplied a bare `{ now }`. Requiring a full `Clock` there would have been a breaking change for
|
|
1580
|
+
* every one of them — which is exactly why the compaction yield shipped as dead code: the type could not carry
|
|
1581
|
+
* a clock capable of yielding.
|
|
1582
|
+
*/
|
|
1583
|
+
interface Yielder {
|
|
1584
|
+
yieldNow?(): Promise<void>;
|
|
1585
|
+
sleep?(ms: number): Promise<void>;
|
|
1586
|
+
}
|
|
1587
|
+
|
|
1370
1588
|
interface CrbmReaderOptions {
|
|
1371
1589
|
/** Speculative tail size in bytes (default 256 KB; clamped up to at least the footer size). */
|
|
1372
1590
|
readonly tailBytes?: number;
|
|
@@ -1556,6 +1774,7 @@ declare function writeCrbmGeneration(driver: IColdDriver, key: GenKey, chunks: I
|
|
|
1556
1774
|
bitmap: CodecBitmap;
|
|
1557
1775
|
}>, options?: {
|
|
1558
1776
|
crypto?: CrbmCrypto;
|
|
1777
|
+
clock?: Yielder;
|
|
1559
1778
|
}): Promise<{
|
|
1560
1779
|
size: number;
|
|
1561
1780
|
sha256: string;
|
|
@@ -1610,6 +1829,16 @@ declare function bulkLoadCrbmGeneration(driver: IColdDriver, key: GenKey, ids: I
|
|
|
1610
1829
|
audit?: IAuditSink;
|
|
1611
1830
|
/** Bitmap codec. Optional in the type; a **flavor** package binds it ({@link requireCodec}). */
|
|
1612
1831
|
codec?: CodecInterface;
|
|
1832
|
+
/**
|
|
1833
|
+
* Injected clock. Supplying one makes bulk-load **cooperative**: it yields the event loop periodically so a
|
|
1834
|
+
* long load does not stall everything else on the process. Without it the load still completes, just
|
|
1835
|
+
* without yielding — which is the pre-existing behaviour, kept so this is purely additive.
|
|
1836
|
+
*
|
|
1837
|
+
* `@cloudbitmaps/roaring` supplies a real clock by default, so flavor users get cooperative behaviour with
|
|
1838
|
+
* no wiring. `core/` cannot default it: it is timer-free by lint, which is exactly why waiting goes through
|
|
1839
|
+
* this seam rather than `setTimeout`.
|
|
1840
|
+
*/
|
|
1841
|
+
clock?: Clock;
|
|
1613
1842
|
}): Promise<BulkLoadResult>;
|
|
1614
1843
|
|
|
1615
1844
|
/**
|
|
@@ -1938,6 +2167,26 @@ declare class RetryingWarmDriver implements IWarmDriver {
|
|
|
1938
2167
|
token: Token;
|
|
1939
2168
|
}>;
|
|
1940
2169
|
deleteConditional(ref: ChunkRef, expected: Token): Promise<void>;
|
|
2170
|
+
/**
|
|
2171
|
+
* Stream the inner driver's rows, retrying only until the first one arrives.
|
|
2172
|
+
*
|
|
2173
|
+
* **This used to buffer the whole scan** — `for await (…) out.push(row)` inside `withRetry`, then
|
|
2174
|
+
* `yield* out` — which made the retry trivially safe and quietly defeated every bound above it. The engine's
|
|
2175
|
+
* `collectWarm` refuses *during* enumeration precisely so resident memory is `O(ceiling)` rather than
|
|
2176
|
+
* `O(segment)`; with the buffering wrapper in place (and it is wired by default) it saw its first row only
|
|
2177
|
+
* after the entire segment was already in memory. Measured on a 500-row segment under
|
|
2178
|
+
* `budget: { maxRequests: 3 }`: **500 rows materialised with the default wiring, 4 with `retry: false`**.
|
|
2179
|
+
* Both paths threw the same `BudgetExceededError`, which is why no test caught it — the error was never the
|
|
2180
|
+
* distinguishing observable, the row count was.
|
|
2181
|
+
*
|
|
2182
|
+
* **The trade this makes.** A mid-stream fault now propagates instead of being retried. It cannot be retried
|
|
2183
|
+
* honestly: rows have already been handed to the consumer, so restarting the iterator would re-yield them,
|
|
2184
|
+
* and the driver interface exposes no resumption token to continue from. Retry is kept where it pays —
|
|
2185
|
+
* establishing the scan, which is where transient connect/throttle failures actually land — and bounded
|
|
2186
|
+
* memory wins the conflict, because invariant 6 ("bounded memory & cost, always") is a hard invariant and
|
|
2187
|
+
* mid-scan retry is not. A caller who needs whole-scan retry can wrap the operation at their own level,
|
|
2188
|
+
* where re-reading from the start is safe.
|
|
2189
|
+
*/
|
|
1941
2190
|
listChunks(ref: SegmentRef, opts?: WarmReadOptions): AsyncIterable<{
|
|
1942
2191
|
chunkKey: number;
|
|
1943
2192
|
} & WarmRow>;
|
|
@@ -2024,4 +2273,4 @@ declare class CrbmWriter {
|
|
|
2024
2273
|
private buildFooter;
|
|
2025
2274
|
}
|
|
2026
2275
|
|
|
2027
|
-
export { AWS_US_EAST_1_ONDEMAND, Aead, AeadSealed, type AuditEvent, type AuditEventKind, BlobReader, BlobSink, BoundedLru, type Budget, BudgetExceededError, type BudgetOption, type BulkLoadResult, CapabilityError, ChunkRef, type Clock, CloudRoaringError, type CodecBitmap, type CodecInterface, ColdCaps, ColdChunkSource, type CompactionCandidate, type CompactionCycleResult, type CompactionDeps, type CompactionOptions, type CompactionResult, type ConsistencyErrorEntry, type ConsistencyIssue, type ConsistencyReport, type CostAdvisory, type CostReport, CountingMetricsSink, CrbmColdChunkSource, type CrbmColdChunkSourceOptions, CrbmCrypto, CrbmReader, type CrbmReaderOptions, CrbmWriter, type CrbmWriterOptions, DEFAULT_BUDGET, DEFAULT_MAX_SCAN_SEGMENTS, DEFAULT_MAX_WARM_SCAN_BYTES, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETRY_POLICY, type DestroyResult, type DiscoveryOptions, type EngineDeps, type EraseDeps, type EstimateInput, type ExportFailure, type ExportFormat, type ExportManifest, type ExportOptions, type ExportSink, type ExportWriter, type ExportedSegment, GenKey, type IAuditSink, IColdDriver, IKeystore, type IMetricsSink, IRegistryDriver, IWarmDriver, InProcessKeystore, type InProcessKeystoreOptions, IntegrityError, KeyUnavailableError, LocalFsColdDriver, LocalFsRegistryDriver, type LocalFsRegistryDriverOptions, LocalFsWarmDriver, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, type MemoryRegistryDriverOptions, MemoryWarmDriver, type MetricEvent, type MetricOpName, type MetricsSnapshot, NOOP_AUDIT, NOOP_METRICS, NewRegistryRecord, NoRow, NodeAead, NotFoundError, type PricingProfile, RecordingAuditSink, RegCaps, RegistryPatch, RegistryRecord, type RetryDeps, type RetryPolicy, RetryingColdChunkSource, RetryingColdDriver, type RetryingOptions, RetryingRegistryDriver, RetryingWarmDriver, type Rng, SegmentEngine, SegmentRef, SegmentSize, type SegmentSizing, TimeoutError, Token, type Topology, TransientError, UnsupportedError, ValidationError, WarmReadOptions, WarmRow, type Workload, WrappedDek, WriteConflictError, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, collectWithinBudget, compactSegment, destroySegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, groundedReport, isCloudRoaringError, isIntegrityError, isNotFoundError, isTransient, isTransientError, isValidationError, isWriteConflictError, joinId, mapWithConcurrency, publishGeneration, resolveBudget, resolvePerOpBudget, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, splitId, validateCompactionOptions, validateSegmentRef, withRetry, writeCrbmGeneration };
|
|
2276
|
+
export { AWS_US_EAST_1_ONDEMAND, Aead, AeadSealed, type AuditEvent, type AuditEventKind, BlobReader, BlobSink, BoundedLru, type Budget, BudgetExceededError, type BudgetOption, type BulkLoadResult, CapabilityError, ChunkRef, type Clock, CloudRoaringError, type CodecBitmap, type CodecInterface, ColdCaps, ColdChunkSource, type CompactionCandidate, type CompactionCycleResult, type CompactionDeps, type CompactionOptions, type CompactionResult, type ConsistencyErrorEntry, type ConsistencyIssue, type ConsistencyReport, type CostAdvisory, type CostReport, CountingMetricsSink, CrbmColdChunkSource, type CrbmColdChunkSourceOptions, CrbmCrypto, CrbmReader, type CrbmReaderOptions, CrbmWriter, type CrbmWriterOptions, DEFAULT_BUDGET, DEFAULT_MAX_SCAN_SEGMENTS, DEFAULT_MAX_WARM_SCAN_BYTES, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETRY_POLICY, DEFAULT_WRITE_CONCURRENCY, type DestroyResult, type DiscoveryOptions, type EngineDeps, type EraseDeps, type EstimateInput, type ExportFailure, type ExportFormat, type ExportManifest, type ExportOptions, type ExportSink, type ExportWriter, type ExportedSegment, GenKey, type IAuditSink, IColdDriver, IKeystore, type IMetricsSink, IRegistryDriver, IWarmDriver, InProcessKeystore, type InProcessKeystoreOptions, IntegrityError, KeyUnavailableError, LocalFsColdDriver, LocalFsRegistryDriver, type LocalFsRegistryDriverOptions, LocalFsWarmDriver, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, type MemoryRegistryDriverOptions, MemoryWarmDriver, type MetricEvent, type MetricOpName, type MetricsSnapshot, NOOP_AUDIT, NOOP_METRICS, NewRegistryRecord, NoRow, NodeAead, NotFoundError, type PricingProfile, RecordingAuditSink, RegCaps, RegistryPatch, RegistryRecord, type RetryDeps, type RetryPolicy, RetryingColdChunkSource, RetryingColdDriver, type RetryingOptions, RetryingRegistryDriver, RetryingWarmDriver, type Rng, SegmentEngine, SegmentRef, SegmentSize, type SegmentSizing, TimeoutError, Token, type Topology, TransientError, UnsupportedError, ValidationError, WarmReadOptions, WarmRow, type Workload, WrappedDek, WriteConflictError, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, collectWithinBudget, compactSegment, destroySegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, groundedReport, isCloudRoaringError, isIntegrityError, isNotFoundError, isTransient, isTransientError, isValidationError, isWriteConflictError, joinId, mapWithConcurrency, publishGeneration, resolveBudget, resolvePerOpBudget, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, splitId, validateCompactionOptions, validateSegmentRef, withRetry, writeCrbmGeneration };
|