@cloudbitmaps/roaring 0.11.2 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +50 -15
- package/README.md +9 -7
- package/dist/bin/export-segments.js +238 -61
- package/dist/bin/export-segments.js.map +3 -3
- package/dist/index.d.ts +151 -35
- package/dist/index.js +238 -61
- package/dist/index.js.map +3 -3
- package/dist/portable/layout.d.ts +20 -1
- package/dist/roaring-codec.d.ts +28 -3
- package/dist/testing/conformance.d.ts +35 -3
- package/dist/testing/fuzz-codec.d.ts +1 -1
- package/package.json +2 -2
package/dist/index.d.ts
CHANGED
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
* `checkConsistency`, `exportSegments` — reuse the store's own drivers, so you never re-pass them (they need the
|
|
24
24
|
* store built with a backend). See the README and the getting-started guide.
|
|
25
25
|
*/
|
|
26
|
-
import type { BudgetOption, GenerationEntry, LoadGuard, LoadOptions, LoadRefusal, LoadResult, RollbackResult, Clock, StorageChunkSource, ConsistencyReport, CostReport, DropResult, EstimateInput, ExportManifest, ExportOptions, ExportSink, IAuditSink, IStorageDriver, IKeystore, IMetricsSink, StorageBackend, PricingProfile, RetentionPolicy, RetireExpiredOptions, RetireExpiredResult, RetryPolicy, Rng, SetRetentionResult, PinnedAt, IdRange, SegmentRef, Workload } from '@cloudbitmaps/core';
|
|
26
|
+
import type { BudgetOption, GenerationEntry, GenerationMetadata, LoadGuard, LoadInput, LoadOptions, LoadRefusal, LoadResult, RollbackResult, Clock, StorageChunkSource, ConsistencyReport, CostReport, DropResult, EstimateInput, ExportManifest, ExportOptions, ExportSink, IAuditSink, IStorageDriver, IKeystore, IMetricsSink, StorageBackend, PricingProfile, RetentionPolicy, RetireExpiredOptions, RetireExpiredResult, RetryPolicy, Rng, SetRetentionResult, PinnedAt, IdRange, SegmentRef, Workload } from '@cloudbitmaps/core';
|
|
27
27
|
import type { SegmentInfo } from '@cloudbitmaps/core';
|
|
28
28
|
/**
|
|
29
29
|
* Wiring for a {@link CloudRoaring} store. **Only `storage` is required** — the minimal call is
|
|
@@ -68,10 +68,14 @@ export interface CloudRoaringOptions {
|
|
|
68
68
|
* Resilience: by default every read that answers a query retries **transient** faults (throttling, 5xx, dropped
|
|
69
69
|
* connections) with bounded, jittered exponential backoff (see {@link DEFAULT_RETRY_POLICY}): `has`, `count`,
|
|
70
70
|
* `iterate` and the combines, the `*Into` verbs' reads of their operands included, a pinned handle's reads and
|
|
71
|
-
* `pin()` itself
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
*
|
|
71
|
+
* `pin()` itself, and the reads a write makes along the way: a load's guard read of the current generation, and an
|
|
72
|
+
* erasure's reads of the generation it rewrites, of the one it wrote and of any other that may still hold the id.
|
|
73
|
+
* This option does not govern writes, nor the calls that read the registry or list the bucket directly (`exists`,
|
|
74
|
+
* `segments`, `generations`, `getRetention`, and the registry scan `subjectReport`, `exportSegments` and
|
|
75
|
+
* `checkConsistency` start from): they report a transient fault to their caller, because a conditional write that
|
|
76
|
+
* lands and then loses its response would, replayed blindly, report its own write as a conflict. A write is
|
|
77
|
+
* retried only where that is safe: a load's registry write that gets no answer is settled by reading the row,
|
|
78
|
+
* and a throttled write-once object is sent again by the S3 and GCS drivers. Pass a partial policy to tune it — anything you leave out keeps its
|
|
75
79
|
* default — or `false` to turn the read retry off (e.g. if your injected client already retries). A GCS download is retried by
|
|
76
80
|
* the GCS driver whatever this says.
|
|
77
81
|
* Deterministic errors (`ValidationError`/`IntegrityError`/`WriteConflictError`/…) are never retried by this
|
|
@@ -138,7 +142,8 @@ export interface CacheOptions {
|
|
|
138
142
|
*/
|
|
139
143
|
readonly readerMax?: number;
|
|
140
144
|
/**
|
|
141
|
-
* Aggregate byte ceiling on
|
|
145
|
+
* Aggregate byte ceiling on what the open readers hold, their parsed `.crbm` indices and the metadata a
|
|
146
|
+
* generation carries (default 64 MiB) — the byte half
|
|
142
147
|
* of the memory bound, complementing the {@link CacheOptions.readerMax} *count* bound. A wide/dense segment's
|
|
143
148
|
* parsed index can reach about 1.3 MB, so a count-only bound could let the open readers pin over a GB and blow a small
|
|
144
149
|
* heap (e.g. a 128 MB Lambda); this evicts the least-recently-used reader once the summed index footprint
|
|
@@ -310,9 +315,10 @@ export interface MaterializeResult {
|
|
|
310
315
|
readonly cardinality: number;
|
|
311
316
|
/**
|
|
312
317
|
* What the destination held when the guard judged it — `null` when it had no current generation, **or when
|
|
313
|
-
* no bound needed
|
|
314
|
-
*
|
|
315
|
-
*
|
|
318
|
+
* no bound needed it**. It is taken only when a bound will use it: `allowEmpty: true` with no
|
|
319
|
+
* `guard.minRetained` skips it, and this is `null` even though `dest` was non-empty. When the destination's row
|
|
320
|
+
* carries a summary of its current generation the count comes from the row and the object is not read; a row
|
|
321
|
+
* written before rows carried a summary costs one object-header fetch.
|
|
316
322
|
*/
|
|
317
323
|
readonly cardinalityBefore: number | null;
|
|
318
324
|
/** Non-empty chunks in the generation. */
|
|
@@ -333,6 +339,7 @@ export declare class CloudRoaring {
|
|
|
333
339
|
private readonly retryOptions;
|
|
334
340
|
private readonly crbmSource;
|
|
335
341
|
private readonly clock;
|
|
342
|
+
private readonly rng;
|
|
336
343
|
private readonly metrics;
|
|
337
344
|
private readonly storageDriver;
|
|
338
345
|
private readonly registry;
|
|
@@ -360,7 +367,7 @@ export declare class CloudRoaring {
|
|
|
360
367
|
/**
|
|
361
368
|
* Write `ids` as a **new generation of `dest`** and publish it — the shared body of the `*Into` verbs. An `*Into`
|
|
362
369
|
* materialisation is a load, and publishes the same way (hard invariant 1): every load that finds a row fences its
|
|
363
|
-
* publish on the row's token; a guarded load (the default, since the empty refusal
|
|
370
|
+
* publish on the row's token; a guarded load (the default, since the empty refusal needs the size of the current generation)
|
|
364
371
|
* also fences on the pointer it judged (`expectFrom`), and one that found no row fences on that absence instead.
|
|
365
372
|
* Only an unguarded load (`allowEmpty: true` and no `guard.minRetained`) onto a segment with no row publishes bare
|
|
366
373
|
* forward-only. The destination's previous generation stays readable until the publish lands (readers re-resolve
|
|
@@ -512,7 +519,8 @@ export declare class CloudRoaring {
|
|
|
512
519
|
* for a `destroyed` tombstone, because a read answers empty in both cases.
|
|
513
520
|
*
|
|
514
521
|
* Two states answer `true` where a read still gives you nothing: a torn restore (a live pointer whose object
|
|
515
|
-
* was deleted) makes reads *throw* rather than answer empty
|
|
522
|
+
* was deleted) makes reads of the object *throw* rather than answer empty, while a cold `count()` answers the number
|
|
523
|
+
* its row records — `checkConsistency` is the call for that — and
|
|
516
524
|
* a handle carrying an expired `expiresAt` reads empty by a rule that lives on the handle, not the row.
|
|
517
525
|
*
|
|
518
526
|
* Not a lock: the answer can change the moment it returns. If it has to hold, use the fence built for that —
|
|
@@ -567,13 +575,20 @@ export declare class CloudRoaring {
|
|
|
567
575
|
* one cannot.
|
|
568
576
|
*
|
|
569
577
|
* It refuses rather than guesses: a generation not in the bucket (collected, or never written) throws
|
|
570
|
-
* `NotFoundError` naming what *is* available,
|
|
571
|
-
* {@link ValidationError} because every generation of it is unreadable
|
|
572
|
-
*
|
|
578
|
+
* `NotFoundError` naming what *is* available, a crypto-shredded segment throws
|
|
579
|
+
* {@link ValidationError} because every generation of it is unreadable, and a target that is cleartext under an
|
|
580
|
+
* encrypted segment, or encrypted under a cleartext one, throws {@link IntegrityError} from one tail read of it,
|
|
581
|
+
* because every read would refuse it. Rolling to the generation already current is a no-op that reports itself.
|
|
582
|
+
*
|
|
583
|
+
* The same tail read (and a range read when the target's index is longer than it) gives the target's id count and
|
|
584
|
+
* metadata, which the rollback writes into the row with the pointer, so a reader that sees the target as current sees
|
|
585
|
+
* what describes it. The store's keystore opens an encrypted target for this; without it, or when it cannot open the
|
|
586
|
+
* key for any reason, the segment still rolls back and the row carries no summary of the target.
|
|
573
587
|
*
|
|
574
588
|
* It deletes nothing. The generations above the new pointer stay put — which is what makes this reversible —
|
|
575
|
-
* and are then *above* `currentGen`, where collection never looks. They remain until
|
|
576
|
-
*
|
|
589
|
+
* and are then *above* `currentGen`, where collection never looks. They remain until loads pass them (the first
|
|
590
|
+
* load whose number one of them holds numbers above them all, and collection then keeps the newest `keep` of what is
|
|
591
|
+
* below its pointer), {@link CloudRoaring.dropSegment}
|
|
577
592
|
* deletes them, or {@link CloudRoaring.eraseSubject} does: all of those present when it rewrites, and only those holding
|
|
578
593
|
* the id when the current generation does not. An operator who has just undone a bad load should not have the
|
|
579
594
|
* evidence collected out from under them, while a rollback target that still holds erased data would make the
|
|
@@ -590,7 +605,14 @@ export declare class CloudRoaring {
|
|
|
590
605
|
allowForward?: boolean;
|
|
591
606
|
}): Promise<RollbackResult>;
|
|
592
607
|
/**
|
|
593
|
-
* **Replace this segment's contents** with `
|
|
608
|
+
* **Replace this segment's contents** with `input`, as one new immutable generation, and make it current.
|
|
609
|
+
*
|
|
610
|
+
* `input` is ids (any sync or async iterable of integers in `[0, 2^32)`), or a whole bitmap: `{ bitmap }`, anything
|
|
611
|
+
* with `serialize('portable')` such as `roaring`'s `RoaringBitmap32`, or `{ serialized }`, portable Roaring bytes.
|
|
612
|
+
* A bitmap is checked (size cap, structure, safe deserializer) before the first request, written from its own
|
|
613
|
+
* containers with no per-id work, and gives the generation byte for byte the one its ids would. A bare
|
|
614
|
+
* `RoaringBitmap32` passed as ids loads as `{ bitmap }`. A `Uint8Array` or `Buffer` passed as ids is refused with
|
|
615
|
+
* `ValidationError`, since each byte would load as an id: pass bytes as `{ serialized }`.
|
|
594
616
|
*
|
|
595
617
|
* The whole write path in one call: take the next generation number, write the object, check the result is
|
|
596
618
|
* plausible, move the pointer, collect what the move superseded. Composed by hand those are four functions and
|
|
@@ -604,6 +626,14 @@ export declare class CloudRoaring {
|
|
|
604
626
|
* if (!r.published) console.warn(`load refused: ${r.reason}`);
|
|
605
627
|
* ```
|
|
606
628
|
*
|
|
629
|
+
* `metadata` attaches a small record of your own to the generation: a flat object of string keys and string or finite
|
|
630
|
+
* number values, at most 1,024 bytes as canonical JSON and no key over 128 bytes. A record that breaks a rule throws
|
|
631
|
+
* `ValidationError` before the load makes a request. It is copied when you call, stored in the generation's object,
|
|
632
|
+
* and written to the segment's row with the generation's id count by the write that moves the pointer, so a reader
|
|
633
|
+
* that sees the generation as current sees its metadata. It never changes after that, a rollback puts the target's own
|
|
634
|
+
* back, and an erasure carries it over without scanning it, so keep a subject's id out of it. It is sealed under the
|
|
635
|
+
* segment's key on an encrypted segment.
|
|
636
|
+
*
|
|
607
637
|
* **A load REPLACES.** Whatever the stream contains is what the segment contains afterwards, so an upstream
|
|
608
638
|
* query that returns fewer rows than usual is a shrink nobody asked for and an empty one is a wipe — both
|
|
609
639
|
* reported as a successful write, because at the storage layer they are one. That is what `guard` and the
|
|
@@ -627,16 +657,35 @@ export declare class CloudRoaring {
|
|
|
627
657
|
* so it leaves the orphan rather than risk deleting live data. The orphan is an ordinary generation once a later
|
|
628
658
|
* one is current above it, and collection counts it within `keep`.
|
|
629
659
|
*
|
|
660
|
+
* **Collection is by name for the default `keep`.** With `keep` of 0 or 1, a load that found nothing above the
|
|
661
|
+
* pointer deletes the one generation its publish pushed out of the window and lists nothing; it lists the segment's
|
|
662
|
+
* objects on every sixteenth generation, and on any load that met an object above the pointer or whose check found
|
|
663
|
+
* the current generation's object gone, to take what the name-only passes leave, such as the generations an
|
|
664
|
+
* earlier, wider `keep` held. `keep` of 2 or more lists on every load. {@link LoadResult.collected} then names what
|
|
665
|
+
* the pass deleted by name, and that generation may have been gone already.
|
|
666
|
+
*
|
|
630
667
|
* What it **throws** for is a fault rather than an outcome: invalid options or ids, and a crypto-shredded
|
|
631
668
|
* segment (`ValidationError`); a key the keystore cannot provide (`KeyUnavailableError`); a current generation
|
|
632
|
-
* that will not open when a guard
|
|
633
|
-
*
|
|
634
|
-
*
|
|
635
|
-
*
|
|
669
|
+
* that will not open when a guard has to read its size, which it does only when the row has no usable summary of it
|
|
670
|
+
* (`IntegrityError`); a driver failure; and a collection pass
|
|
671
|
+
* by listing that could not prove the segment was still the same one (`WriteConflictError`). That one, and a
|
|
672
|
+
* failure in the collection's own reads or deletes, can be raised **after** the publish already landed, so a throw
|
|
673
|
+
* does not by itself mean the load did not take effect — re-read the pointer rather than assuming. A collection by
|
|
674
|
+
* name that finds the segment changed returns an empty `collected` instead.
|
|
675
|
+
*
|
|
676
|
+
* **A `TransientError` from the registry write can leave the publish unsettled, and deletes nothing.** The
|
|
677
|
+
* generation's object is sent again after a throttle where the backend allows it (a write id tells a first send
|
|
678
|
+
* that landed from another writer's object), and a driver sends each registry write once. When that write ends
|
|
679
|
+
* without an answer, the load reads the row: its own landed write is `published: true`, a row that has moved on is
|
|
680
|
+
* `superseded`, and a row still as the write found it gets a fresh compare-and-swap from the version just read, at
|
|
681
|
+
* most three times, after a wait on the store's clock. Still unanswered, the load throws the registry's `TransientError` and keeps its object, which
|
|
682
|
+
* a write may still point the row at. Re-run the load: it numbers past that object, and collection removes it once a
|
|
683
|
+
* generation above it is current. A `'superseded'` refusal that follows an unanswered write sets `unanswered: true`
|
|
684
|
+
* on its audit event: that write may have landed, and the generation been current for a while, first.
|
|
636
685
|
*
|
|
637
686
|
* Needs a backend (throws {@link UnsupportedError} otherwise).
|
|
638
687
|
*/
|
|
639
|
-
load(ref: SegmentRef,
|
|
688
|
+
load(ref: SegmentRef, input: LoadInput, options?: LoadOptions): Promise<LoadResult>;
|
|
640
689
|
/**
|
|
641
690
|
* **Dispose of a segment — tombstone it, then delete its storage objects.** Irreversible.
|
|
642
691
|
*
|
|
@@ -662,9 +711,12 @@ export declare class CloudRoaring {
|
|
|
662
711
|
* forever and quietly. Branch on `dropped`, and treat `reason: 'absent'` as the alert.
|
|
663
712
|
*
|
|
664
713
|
* **Inspect `generationsRemaining`.** Empty is the normal outcome; non-empty means the storage was NOT fully
|
|
665
|
-
* reclaimed and the drop should be re-run. A load that
|
|
666
|
-
*
|
|
667
|
-
* could not remove rather than returning a result that looks like a
|
|
714
|
+
* reclaimed and the drop should be re-run. A load that had read the segment before the tombstone landed, whether it
|
|
715
|
+
* was writing or still consuming its ids, can still write its object, so a single sweep can miss it — this call
|
|
716
|
+
* re-sweeps and then reports whatever it still could not remove rather than returning a result that looks like a
|
|
717
|
+
* clean drop. A load that writes after the last sweep deletes its own object once its publish is refused; only one
|
|
718
|
+
* whose process stops in between, or whose publish fails without a definite answer (a lost response, a timeout),
|
|
719
|
+
* leaves it, for a re-run of the drop.
|
|
668
720
|
*
|
|
669
721
|
* Reads become empty within `cache.genTtlMs` (default 2 s), not instantly: a store that had already read this
|
|
670
722
|
* segment may answer from its cached generation + cached chunks until that window lapses, or, while the registry
|
|
@@ -759,6 +811,12 @@ export declare class CloudRoaring {
|
|
|
759
811
|
* wall-clock knob too, and `retired` counts deletions only — a dry run reports `wouldRetire` instead, so a
|
|
760
812
|
* dashboard summing `retired` can never show a phantom deletion.
|
|
761
813
|
*
|
|
814
|
+
* **Check `purgeFaults`.** A delete the registry refuses for a reason other than a lost race (a policy that denies
|
|
815
|
+
* delete, an Azure blob with a snapshot, a raw provider error), whether of a tombstone or of a due-index pointer, is
|
|
816
|
+
* counted there, with the first one's reason in `firstPurgeFault`. A refused purge is `skipped` in the ledger, is not
|
|
817
|
+
* charged to `limit`, so it never holds the retirements behind it, and purging stops for the rest of the call after
|
|
818
|
+
* three refused purges in a row (a purge that succeeds starts the count again).
|
|
819
|
+
*
|
|
762
820
|
* It also **deletes the tombstone rows its own past retirements left**, after `tombstoneGraceMs` (default 24 h)
|
|
763
821
|
* and only once that segment's Storage generations are provably gone — collecting a straggler generation itself
|
|
764
822
|
* first, since nothing else ever would for a tombstoned segment. Attribution is a **positive marker the sweep
|
|
@@ -766,7 +824,12 @@ export declare class CloudRoaring {
|
|
|
766
824
|
* `retention` untouched, so setting a policy and then honouring a right-to-erasure request mid-window produces
|
|
767
825
|
* exactly that row, and deleting it would destroy the Art. 17 attestation and un-fence the name. By default a
|
|
768
826
|
* segment that held nothing has its row deleted in the pass that retires it, since that row would only fence the name.
|
|
769
|
-
* Pass `purgeTombstones: false` to keep every tombstone, that row included.
|
|
827
|
+
* Pass `purgeTombstones: false` to keep every tombstone, that row included. On a backend whose registry reports
|
|
828
|
+
* `conditionalDelete` (AWS S3, Azure Blob, the local filesystem and memory, by default; GCS when you set `conditionalDelete: true`)
|
|
829
|
+
* the purge removes the row from the bucket for good, by a delete the store applies only to the version it judged,
|
|
830
|
+
* so a full sweep reads what is live or inside its grace rather than every name a namespace ever held; elsewhere,
|
|
831
|
+
* and for a row a release before 0.12 wrote, it leaves a tombstone. Each retirement files a pointer in the due index
|
|
832
|
+
* under the day its tombstone's grace ends, so `scan: 'index'` purges as well as retires.
|
|
770
833
|
*
|
|
771
834
|
* Needs the store built with a **backend** (throws {@link UnsupportedError} otherwise),
|
|
772
835
|
* because retiring a segment deletes its storage objects. `now` defaults to the store's clock.
|
|
@@ -852,6 +915,13 @@ export declare class CloudRoaring {
|
|
|
852
915
|
* (throws {@link UnsupportedError} otherwise). `destroyed` (crypto-shredded) segments are skipped. Pair it with
|
|
853
916
|
* the DR runbook (docs/guide/disaster-recovery.md).
|
|
854
917
|
*
|
|
918
|
+
* A segment whose pointer names a missing object is the torn restore this reports; a cold `count()` of it still
|
|
919
|
+
* answers the number its row records, while a read of the object throws, so the count alone never shows it. With
|
|
920
|
+
* `summaries: true` it also opens each segment's current object (one tail read each) and reports a segment whose
|
|
921
|
+
* row's summary says another id count or metadata than the object holds as `summary-mismatch`, which is what a
|
|
922
|
+
* restore of the registry from another point than the bucket leaves a count to answer. A sealed summary is held
|
|
923
|
+
* against its object when this store has the key, and counted in `summariesUnchecked` when it does not.
|
|
924
|
+
*
|
|
855
925
|
* It holds the registry rows it enumerates resident, at most 250,000 of them, and throws
|
|
856
926
|
* `BudgetExceededError` past that rather than report a partial scan as a whole one. This method takes no
|
|
857
927
|
* ceiling of its own: narrow the scan with `namespace`, or call `runConsistencyCheck` over the backend's
|
|
@@ -860,6 +930,7 @@ export declare class CloudRoaring {
|
|
|
860
930
|
checkConsistency(options?: {
|
|
861
931
|
namespace?: string;
|
|
862
932
|
concurrency?: number;
|
|
933
|
+
summaries?: boolean;
|
|
863
934
|
}): Promise<ConsistencyReport>;
|
|
864
935
|
/**
|
|
865
936
|
* **Export ("eject") every registered segment's current generation** through the injected `sink`, using only
|
|
@@ -887,7 +958,13 @@ export declare class CloudRoaring {
|
|
|
887
958
|
* in `(after, through]` and fetches only the chunks the range overlaps, as {@link Segment.iterate} does.
|
|
888
959
|
*/
|
|
889
960
|
export interface BaseCombineOptions extends IdRange {
|
|
890
|
-
/**
|
|
961
|
+
/**
|
|
962
|
+
* Max chunk keys resolved concurrently — bounds the Storage footprint (about `concurrency × operands` chunks in
|
|
963
|
+
* flight). A positive integer; default 32. The window opens 8 keys wide (or `concurrency`, if lower) and widens
|
|
964
|
+
* as keys are taken, so a read that stops early fetches no more than a few keys ahead; one that runs to the end
|
|
965
|
+
* spends nearly all of its round trips at the full width. A read that stops early has requested up to
|
|
966
|
+
* `concurrency` keys per operand past the last one it used.
|
|
967
|
+
*/
|
|
891
968
|
readonly concurrency?: number;
|
|
892
969
|
/** Override the store's per-op denial-of-wallet budget for this call (`false` lifts it). */
|
|
893
970
|
readonly budget?: BudgetOption;
|
|
@@ -955,14 +1032,22 @@ export interface MaterializeOptions extends CombineOptions {
|
|
|
955
1032
|
readonly allowEmpty?: boolean;
|
|
956
1033
|
/** Refuse an implausible result rather than publish it. Same bounds, and same meaning, as on `load()`. */
|
|
957
1034
|
readonly guard?: LoadGuard;
|
|
1035
|
+
/**
|
|
1036
|
+
* Metadata for the generation this call publishes, under the rules and with the meaning of
|
|
1037
|
+
* {@link LoadOptions.metadata}: small, immutable, published with the pointer, never a subject's id. A value that
|
|
1038
|
+
* breaks a rule throws {@link ValidationError} before any request is made.
|
|
1039
|
+
*/
|
|
1040
|
+
readonly metadata?: GenerationMetadata;
|
|
958
1041
|
/**
|
|
959
1042
|
* Generations to keep below the new pointer — see {@link LoadOptions.keep}. A value that is not a non-negative
|
|
960
1043
|
* integer throws `ValidationError`.
|
|
961
1044
|
*
|
|
962
|
-
* **Defaults to keeping everything**, unlike `load()`, which keeps 1 and collects
|
|
1045
|
+
* **Defaults to keeping everything**, unlike `load()`, which keeps 1 and collects what it supersedes. An operator's
|
|
963
1046
|
* recovery story can depend on a materialisation collecting nothing: `rollbackSegment` refuses a target that
|
|
964
1047
|
* has been collected. Pass a number to collect on the way through; `0` keeps only the
|
|
965
|
-
* generation this call publishes.
|
|
1048
|
+
* generation this call publishes. It collects by listing the destination, so it clears every generation below the
|
|
1049
|
+
* new one beyond `keep`, however many earlier calls kept, where a `load()` deletes by name the one generation its
|
|
1050
|
+
* publish pushes out of the window.
|
|
966
1051
|
*/
|
|
967
1052
|
readonly keep?: number;
|
|
968
1053
|
}
|
|
@@ -974,6 +1059,14 @@ export interface MaterializeOptions extends CombineOptions {
|
|
|
974
1059
|
* solves the same problem the same way, by taking {@link BaseCombineOptions}.
|
|
975
1060
|
*/
|
|
976
1061
|
export type AndNotIntoOptions = Omit<MaterializeOptions, 'exclude'>;
|
|
1062
|
+
/** What {@link Segment.stat} answers: the generation a handle reads, its id count and its metadata. */
|
|
1063
|
+
export interface SegmentStat {
|
|
1064
|
+
/** The generation read, or `null` when the segment has none. */
|
|
1065
|
+
readonly generation: number | null;
|
|
1066
|
+
readonly cardinality: number;
|
|
1067
|
+
/** The metadata the generation was loaded with; absent when it has none. */
|
|
1068
|
+
readonly metadata?: GenerationMetadata;
|
|
1069
|
+
}
|
|
977
1070
|
/**
|
|
978
1071
|
* A handle bound to one segment — the read verbs, plus the three `*Into` verbs that write a **new generation**
|
|
979
1072
|
* of another segment.
|
|
@@ -1122,16 +1215,39 @@ export declare class Segment {
|
|
|
1122
1215
|
/** Membership: one chunk — the cache, else one ranged GET. Throws {@link ValidationError} on a bad id. */
|
|
1123
1216
|
has(id: number): Promise<boolean>;
|
|
1124
1217
|
/**
|
|
1125
|
-
* Cardinality
|
|
1126
|
-
*
|
|
1127
|
-
*
|
|
1128
|
-
*
|
|
1218
|
+
* Cardinality of the generation this handle reads: **one registry read when cold, none when warm, and no read of the
|
|
1219
|
+
* object**, with **zero payload reads**. The registry row records the id count of the generation it names, written
|
|
1220
|
+
* by the write that made it current, and a count answers from that. A row with no summary it can use (one written
|
|
1221
|
+
* before rows carried it, one that names another generation, a sealed one that does not open) sends the count to the
|
|
1222
|
+
* `.crbm` index, summed with a tail read of the object. A segment whose pointer names a missing object (a torn
|
|
1223
|
+
* restore) still counts the row's number, while a read of the object throws; `checkConsistency` is what finds it.
|
|
1224
|
+
*
|
|
1225
|
+
* What this trusts: the row's summary, or the index's sum, and no payload is decoded to confirm it. The summary is
|
|
1226
|
+
* used only for the generation it names, on an active row, and a sealed one only if it opens under the segment's
|
|
1227
|
+
* key; it is held against the object whenever the object is opened anyway (a `has`, an `iterate`, a combine, a
|
|
1228
|
+
* `pin()`), at no extra request, and a disagreement stops this store using it, and fails nothing. It is not
|
|
1229
|
+
* confirmed on the cold path, so a party who can write the registry row can make a count wrong. The index is checked for internal consistency when the object is opened (each key in
|
|
1129
1230
|
* range and ascending, each cardinality in `1..65536`, each payload inside the payload region, and, on an
|
|
1130
1231
|
* unencrypted object, the footer's chunk count and total agreeing with the index), and a corrupt index that is
|
|
1131
1232
|
* still internally consistent yields a wrong count. `iterate()` and the combines decode the payloads, whose
|
|
1132
1233
|
* structure is checked.
|
|
1133
1234
|
*/
|
|
1134
1235
|
count(): Promise<number>;
|
|
1236
|
+
/**
|
|
1237
|
+
* What the generation this handle reads is, from one resolution: its number, its id count and the metadata it
|
|
1238
|
+
* was loaded with (absent when it has none). It is what answers {@link Segment.count}, so the three describe one
|
|
1239
|
+
* generation and cannot straddle a publish. One registry read when cold, none while warm, and none on a pinned
|
|
1240
|
+
* handle, which answers for the generation it pinned. A segment with no generation, and an expired handle,
|
|
1241
|
+
* answer `{ generation: null, cardinality: 0 }`.
|
|
1242
|
+
*
|
|
1243
|
+
* Trust is as for `count()`: the registry row's word, not confirmed against the object until the object is
|
|
1244
|
+
* opened, when a disagreement makes this process stop using that row's summary.
|
|
1245
|
+
*
|
|
1246
|
+
* ```ts
|
|
1247
|
+
* const { generation, cardinality, metadata } = await store.segment('active-30d').stat();
|
|
1248
|
+
* ```
|
|
1249
|
+
*/
|
|
1250
|
+
stat(): Promise<SegmentStat>;
|
|
1135
1251
|
/**
|
|
1136
1252
|
* Every id, ascending, streamed one chunk at a time. Pass a range to read part of the segment: the ids in
|
|
1137
1253
|
* `(after, through]`, fetching only the chunks the range overlaps. The per-op budget is charged for every chunk in
|
|
@@ -1173,7 +1289,7 @@ export declare class Segment {
|
|
|
1173
1289
|
* Chunk-skipping intersection: stream the ids in **this** segment AND every segment in `others`, ascending.
|
|
1174
1290
|
* Fetches only the Storage chunks present in *all* operands (a key absent from any operand contributes nothing
|
|
1175
1291
|
* and is never downloaded), streaming under a bounded in-flight window — so the Storage footprint stays small
|
|
1176
|
-
* (Lambda-friendly) regardless of segment size. Pass `concurrency` to tune that window (a positive integer).
|
|
1292
|
+
* (Lambda-friendly) regardless of segment size. Pass `concurrency` to tune that window (a positive integer, 32 by default).
|
|
1177
1293
|
* AND is commutative, so `a.intersect([b])` and `b.intersect([a])` yield the same ids. Pass `budget` to
|
|
1178
1294
|
* override the store's per-op denial-of-wallet budget for this call (or `false` to lift it).
|
|
1179
1295
|
*/
|
|
@@ -1243,4 +1359,4 @@ export declare class Segment {
|
|
|
1243
1359
|
}): Promise<CostReport>;
|
|
1244
1360
|
}
|
|
1245
1361
|
export { MemoryStorage, LocalFsStorage, CrbmStorageChunkSource, destroySegment, eraseNamespace, excludingReservedRows, readRetentionPolicy, MIN_EXPIRES_AT_MS, InProcessKeystore, NodeAead, aadFor, CloudRoaringError, ValidationError, WriteConflictError, IntegrityError, NotFoundError, UnsupportedError, CapabilityError, TransientError, KeyUnavailableError, BudgetExceededError, isCloudRoaringError, isWriteConflictError, isTransientError, isNotFoundError, isIntegrityError, isValidationError, CrbmReader, BufferReader, CountingMetricsSink, RecordingAuditSink, DEFAULT_RETRY_POLICY, AWS_US_EAST_1_ONDEMAND, ELASTICACHE_REDIS_US_EAST_1_ONDEMAND, ONE_REDIS_HA_CLUSTER, } from '@cloudbitmaps/core';
|
|
1246
|
-
export type { Aead, AeadSealed, AuditEvent, BlobReader, BlobSink, Budget, BudgetOption, ChunkRef, Clock, CodecBitmap, CodecInterface, ConsistencyErrorEntry, ConsistencyIssue, ConsistencyReport, CostReport, CrbmCrypto, CrbmReaderOptions, CrbmStorageChunkSourceOptions, DestroyResult, DropResult, EraseDeps, EstimateInput, ExportFailure, ExportFormat, ExportManifest, ExportOptions, ExportSink, ExportWriter, ExportedSegment, GenKey, GenerationEntry, GovernanceMeta, IAuditSink, IKeystore, IMetricsSink, IRegistryDriver, IStorageDriver, IdRange, InProcessKeystoreOptions, LoadGuard, LoadOptions, LoadRefusal, LoadResult, LocalFsStorageOptions, MemoryStorageOptions, MetricEvent, MetricOpName, MetricsSnapshot, NewRegistryRecord, PinnedAt, PinnedObject, PricingProfile, RedisNodeType, RedisSizing, RegCaps, RegistryPatch, RegistryRecord, RegistryStatus, RetentionPolicy, RetireEntry, RetireExpiredOptions, RetireExpiredResult, RetryPolicy, Rng, RollbackResult, SegmentInfo, SegmentRef, SegmentSize, SegmentSizing, SetRetentionResult, StorageBackend, StorageCaps, StorageChunkSource, Token, Workload, WrappedDek, } from '@cloudbitmaps/core';
|
|
1362
|
+
export type { Aead, AeadSealed, AuditEvent, BlobReader, BlobSink, Budget, BudgetOption, ChunkRef, Clock, CodecBitmap, CodecInterface, ConsistencyErrorEntry, ConsistencyIssue, ConsistencyReport, CostReport, CrbmCrypto, CrbmReaderOptions, CrbmStorageChunkSourceOptions, DestroyResult, ClearRegistrySummary, DropResult, EncodedChunk, EraseDeps, EstimateInput, ExportFailure, ExportFormat, ExportManifest, ExportOptions, ExportSink, ExportWriter, ExportedSegment, GenKey, GenerationEntry, GenerationMetadata, GovernanceMeta, IAuditSink, IKeystore, IMetricsSink, IRegistryDriver, IStorageDriver, IdRange, InProcessKeystoreOptions, LoadGuard, LoadInput, LoadOptions, LoadRefusal, LoadResult, LocalFsStorageOptions, MemoryStorageOptions, MetricEvent, MetricOpName, MetricsSnapshot, NewRegistryRecord, PinnedAt, PinnedObject, PortableBitmap, PricingProfile, RedisNodeType, RedisSizing, RegCaps, RegistryPatch, RegistryRecord, RegistryStatus, RegistrySummary, RetentionPolicy, RetireEntry, RetireExpiredOptions, RetireExpiredResult, RetryPolicy, Rng, RollbackResult, SealedRegistrySummary, SegmentInfo, SegmentRef, SegmentSize, SegmentSizing, SetRetentionResult, StorageBackend, StorageCaps, StorageChunkSource, Token, Workload, WrappedDek, } from '@cloudbitmaps/core';
|