@cloudbitmaps/roaring 0.11.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +53 -16
- package/README.md +6 -4
- package/dist/bin/export-segments.js +237 -60
- package/dist/bin/export-segments.js.map +3 -3
- package/dist/index.d.ts +146 -35
- package/dist/index.js +237 -60
- package/dist/index.js.map +3 -3
- package/dist/portable/layout.d.ts +20 -1
- package/dist/roaring-codec.d.ts +28 -3
- package/dist/testing/conformance.d.ts +35 -3
- package/package.json +2 -2
package/dist/index.d.ts
CHANGED
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
* `checkConsistency`, `exportSegments` — reuse the store's own drivers, so you never re-pass them (they need the
|
|
24
24
|
* store built with a backend). See the README and the getting-started guide.
|
|
25
25
|
*/
|
|
26
|
-
import type { BudgetOption, GenerationEntry, LoadGuard, LoadOptions, LoadRefusal, LoadResult, RollbackResult, Clock, StorageChunkSource, ConsistencyReport, CostReport, DropResult, EstimateInput, ExportManifest, ExportOptions, ExportSink, IAuditSink, IStorageDriver, IKeystore, IMetricsSink, StorageBackend, PricingProfile, RetentionPolicy, RetireExpiredOptions, RetireExpiredResult, RetryPolicy, Rng, SetRetentionResult, PinnedAt, IdRange, SegmentRef, Workload } from '@cloudbitmaps/core';
|
|
26
|
+
import type { BudgetOption, GenerationEntry, GenerationMetadata, LoadGuard, LoadInput, LoadOptions, LoadRefusal, LoadResult, RollbackResult, Clock, StorageChunkSource, ConsistencyReport, CostReport, DropResult, EstimateInput, ExportManifest, ExportOptions, ExportSink, IAuditSink, IStorageDriver, IKeystore, IMetricsSink, StorageBackend, PricingProfile, RetentionPolicy, RetireExpiredOptions, RetireExpiredResult, RetryPolicy, Rng, SetRetentionResult, PinnedAt, IdRange, SegmentRef, Workload } from '@cloudbitmaps/core';
|
|
27
27
|
import type { SegmentInfo } from '@cloudbitmaps/core';
|
|
28
28
|
/**
|
|
29
29
|
* Wiring for a {@link CloudRoaring} store. **Only `storage` is required** — the minimal call is
|
|
@@ -68,11 +68,16 @@ export interface CloudRoaringOptions {
|
|
|
68
68
|
* Resilience: by default every read that answers a query retries **transient** faults (throttling, 5xx, dropped
|
|
69
69
|
* connections) with bounded, jittered exponential backoff (see {@link DEFAULT_RETRY_POLICY}): `has`, `count`,
|
|
70
70
|
* `iterate` and the combines, the `*Into` verbs' reads of their operands included, a pinned handle's reads and
|
|
71
|
-
* `pin()` itself
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
71
|
+
* `pin()` itself, and the reads a write makes along the way: a load's guard read of the current generation, and an
|
|
72
|
+
* erasure's reads of the generation it rewrites, of the one it wrote and of any other that may still hold the id.
|
|
73
|
+
* This option does not govern writes, nor the calls that read the registry or list the bucket directly (`exists`,
|
|
74
|
+
* `segments`, `generations`, `getRetention`, and the registry scan `subjectReport`, `exportSegments` and
|
|
75
|
+
* `checkConsistency` start from): they report a transient fault to their caller, because a conditional write that
|
|
76
|
+
* lands and then loses its response would, replayed blindly, report its own write as a conflict. A write is
|
|
77
|
+
* retried only where that is safe: a load's registry write that gets no answer is settled by reading the row,
|
|
78
|
+
* and a throttled write-once object is sent again by the S3 and GCS drivers. Pass a partial policy to tune it — anything you leave out keeps its
|
|
79
|
+
* default — or `false` to turn the read retry off (e.g. if your injected client already retries). A GCS download is retried by
|
|
80
|
+
* the GCS driver whatever this says.
|
|
76
81
|
* Deterministic errors (`ValidationError`/`IntegrityError`/`WriteConflictError`/…) are never retried by this
|
|
77
82
|
* layer.
|
|
78
83
|
*/
|
|
@@ -137,9 +142,10 @@ export interface CacheOptions {
|
|
|
137
142
|
*/
|
|
138
143
|
readonly readerMax?: number;
|
|
139
144
|
/**
|
|
140
|
-
* Aggregate byte ceiling on
|
|
145
|
+
* Aggregate byte ceiling on what the open readers hold, their parsed `.crbm` indices and the metadata a
|
|
146
|
+
* generation carries (default 64 MiB) — the byte half
|
|
141
147
|
* of the memory bound, complementing the {@link CacheOptions.readerMax} *count* bound. A wide/dense segment's
|
|
142
|
-
* parsed index can
|
|
148
|
+
* parsed index can reach about 1.3 MB, so a count-only bound could let the open readers pin over a GB and blow a small
|
|
143
149
|
* heap (e.g. a 128 MB Lambda); this evicts the least-recently-used reader once the summed index footprint
|
|
144
150
|
* would exceed the ceiling — whichever of the count/byte bounds binds first. Lower it for memory-tight
|
|
145
151
|
* deployments that read across wide segments. Applies whenever the store builds its own read path.
|
|
@@ -309,9 +315,10 @@ export interface MaterializeResult {
|
|
|
309
315
|
readonly cardinality: number;
|
|
310
316
|
/**
|
|
311
317
|
* What the destination held when the guard judged it — `null` when it had no current generation, **or when
|
|
312
|
-
* no bound needed
|
|
313
|
-
*
|
|
314
|
-
*
|
|
318
|
+
* no bound needed it**. It is taken only when a bound will use it: `allowEmpty: true` with no
|
|
319
|
+
* `guard.minRetained` skips it, and this is `null` even though `dest` was non-empty. When the destination's row
|
|
320
|
+
* carries a summary of its current generation the count comes from the row and the object is not read; a row
|
|
321
|
+
* written before rows carried a summary costs one object-header fetch.
|
|
315
322
|
*/
|
|
316
323
|
readonly cardinalityBefore: number | null;
|
|
317
324
|
/** Non-empty chunks in the generation. */
|
|
@@ -332,6 +339,7 @@ export declare class CloudRoaring {
|
|
|
332
339
|
private readonly retryOptions;
|
|
333
340
|
private readonly crbmSource;
|
|
334
341
|
private readonly clock;
|
|
342
|
+
private readonly rng;
|
|
335
343
|
private readonly metrics;
|
|
336
344
|
private readonly storageDriver;
|
|
337
345
|
private readonly registry;
|
|
@@ -359,7 +367,7 @@ export declare class CloudRoaring {
|
|
|
359
367
|
/**
|
|
360
368
|
* Write `ids` as a **new generation of `dest`** and publish it — the shared body of the `*Into` verbs. An `*Into`
|
|
361
369
|
* materialisation is a load, and publishes the same way (hard invariant 1): every load that finds a row fences its
|
|
362
|
-
* publish on the row's token; a guarded load (the default, since the empty refusal
|
|
370
|
+
* publish on the row's token; a guarded load (the default, since the empty refusal needs the size of the current generation)
|
|
363
371
|
* also fences on the pointer it judged (`expectFrom`), and one that found no row fences on that absence instead.
|
|
364
372
|
* Only an unguarded load (`allowEmpty: true` and no `guard.minRetained`) onto a segment with no row publishes bare
|
|
365
373
|
* forward-only. The destination's previous generation stays readable until the publish lands (readers re-resolve
|
|
@@ -511,7 +519,8 @@ export declare class CloudRoaring {
|
|
|
511
519
|
* for a `destroyed` tombstone, because a read answers empty in both cases.
|
|
512
520
|
*
|
|
513
521
|
* Two states answer `true` where a read still gives you nothing: a torn restore (a live pointer whose object
|
|
514
|
-
* was deleted) makes reads *throw* rather than answer empty
|
|
522
|
+
* was deleted) makes reads of the object *throw* rather than answer empty, while a cold `count()` answers the number
|
|
523
|
+
* its row records — `checkConsistency` is the call for that — and
|
|
515
524
|
* a handle carrying an expired `expiresAt` reads empty by a rule that lives on the handle, not the row.
|
|
516
525
|
*
|
|
517
526
|
* Not a lock: the answer can change the moment it returns. If it has to hold, use the fence built for that —
|
|
@@ -566,13 +575,20 @@ export declare class CloudRoaring {
|
|
|
566
575
|
* one cannot.
|
|
567
576
|
*
|
|
568
577
|
* It refuses rather than guesses: a generation not in the bucket (collected, or never written) throws
|
|
569
|
-
* `NotFoundError` naming what *is* available,
|
|
570
|
-
* {@link ValidationError} because every generation of it is unreadable
|
|
571
|
-
*
|
|
578
|
+
* `NotFoundError` naming what *is* available, a crypto-shredded segment throws
|
|
579
|
+
* {@link ValidationError} because every generation of it is unreadable, and a target that is cleartext under an
|
|
580
|
+
* encrypted segment, or encrypted under a cleartext one, throws {@link IntegrityError} from one tail read of it,
|
|
581
|
+
* because every read would refuse it. Rolling to the generation already current is a no-op that reports itself.
|
|
582
|
+
*
|
|
583
|
+
* The same tail read (and a range read when the target's index is longer than it) gives the target's id count and
|
|
584
|
+
* metadata, which the rollback writes into the row with the pointer, so a reader that sees the target as current sees
|
|
585
|
+
* what describes it. The store's keystore opens an encrypted target for this; without it, or when it cannot open the
|
|
586
|
+
* key for any reason, the segment still rolls back and the row carries no summary of the target.
|
|
572
587
|
*
|
|
573
588
|
* It deletes nothing. The generations above the new pointer stay put — which is what makes this reversible —
|
|
574
|
-
* and are then *above* `currentGen`, where collection never looks. They remain until
|
|
575
|
-
*
|
|
589
|
+
* and are then *above* `currentGen`, where collection never looks. They remain until loads pass them (the first
|
|
590
|
+
* load whose number one of them holds numbers above them all, and collection then keeps the newest `keep` of what is
|
|
591
|
+
* below its pointer), {@link CloudRoaring.dropSegment}
|
|
576
592
|
* deletes them, or {@link CloudRoaring.eraseSubject} does: all of those present when it rewrites, and only those holding
|
|
577
593
|
* the id when the current generation does not. An operator who has just undone a bad load should not have the
|
|
578
594
|
* evidence collected out from under them, while a rollback target that still holds erased data would make the
|
|
@@ -589,7 +605,14 @@ export declare class CloudRoaring {
|
|
|
589
605
|
allowForward?: boolean;
|
|
590
606
|
}): Promise<RollbackResult>;
|
|
591
607
|
/**
|
|
592
|
-
* **Replace this segment's contents** with `
|
|
608
|
+
* **Replace this segment's contents** with `input`, as one new immutable generation, and make it current.
|
|
609
|
+
*
|
|
610
|
+
* `input` is ids (any sync or async iterable of integers in `[0, 2^32)`), or a whole bitmap: `{ bitmap }`, anything
|
|
611
|
+
* with `serialize('portable')` such as `roaring`'s `RoaringBitmap32`, or `{ serialized }`, portable Roaring bytes.
|
|
612
|
+
* A bitmap is checked (size cap, structure, safe deserializer) before the first request, written from its own
|
|
613
|
+
* containers with no per-id work, and gives the generation byte for byte the one its ids would. A bare
|
|
614
|
+
* `RoaringBitmap32` passed as ids loads as `{ bitmap }`. A `Uint8Array` or `Buffer` passed as ids is refused with
|
|
615
|
+
* `ValidationError`, since each byte would load as an id: pass bytes as `{ serialized }`.
|
|
593
616
|
*
|
|
594
617
|
* The whole write path in one call: take the next generation number, write the object, check the result is
|
|
595
618
|
* plausible, move the pointer, collect what the move superseded. Composed by hand those are four functions and
|
|
@@ -603,6 +626,14 @@ export declare class CloudRoaring {
|
|
|
603
626
|
* if (!r.published) console.warn(`load refused: ${r.reason}`);
|
|
604
627
|
* ```
|
|
605
628
|
*
|
|
629
|
+
* `metadata` attaches a small record of your own to the generation: a flat object of string keys and string or finite
|
|
630
|
+
* number values, at most 1,024 bytes as canonical JSON and no key over 128 bytes. A record that breaks a rule throws
|
|
631
|
+
* `ValidationError` before the load makes a request. It is copied when you call, stored in the generation's object,
|
|
632
|
+
* and written to the segment's row with the generation's id count by the write that moves the pointer, so a reader
|
|
633
|
+
* that sees the generation as current sees its metadata. It never changes after that, a rollback puts the target's own
|
|
634
|
+
* back, and an erasure carries it over without scanning it, so keep a subject's id out of it. It is sealed under the
|
|
635
|
+
* segment's key on an encrypted segment.
|
|
636
|
+
*
|
|
606
637
|
* **A load REPLACES.** Whatever the stream contains is what the segment contains afterwards, so an upstream
|
|
607
638
|
* query that returns fewer rows than usual is a shrink nobody asked for and an empty one is a wipe — both
|
|
608
639
|
* reported as a successful write, because at the storage layer they are one. That is what `guard` and the
|
|
@@ -626,16 +657,35 @@ export declare class CloudRoaring {
|
|
|
626
657
|
* so it leaves the orphan rather than risk deleting live data. The orphan is an ordinary generation once a later
|
|
627
658
|
* one is current above it, and collection counts it within `keep`.
|
|
628
659
|
*
|
|
660
|
+
* **Collection is by name for the default `keep`.** With `keep` of 0 or 1, a load that found nothing above the
|
|
661
|
+
* pointer deletes the one generation its publish pushed out of the window and lists nothing; it lists the segment's
|
|
662
|
+
* objects on every sixteenth generation, and on any load that met an object above the pointer or whose check found
|
|
663
|
+
* the current generation's object gone, to take what the name-only passes leave, such as the generations an
|
|
664
|
+
* earlier, wider `keep` held. `keep` of 2 or more lists on every load. {@link LoadResult.collected} then names what
|
|
665
|
+
* the pass deleted by name, and that generation may have been gone already.
|
|
666
|
+
*
|
|
629
667
|
* What it **throws** for is a fault rather than an outcome: invalid options or ids, and a crypto-shredded
|
|
630
668
|
* segment (`ValidationError`); a key the keystore cannot provide (`KeyUnavailableError`); a current generation
|
|
631
|
-
* that will not open when a guard
|
|
632
|
-
*
|
|
633
|
-
*
|
|
634
|
-
*
|
|
669
|
+
* that will not open when a guard has to read its size, which it does only when the row has no usable summary of it
|
|
670
|
+
* (`IntegrityError`); a driver failure; and a collection pass
|
|
671
|
+
* by listing that could not prove the segment was still the same one (`WriteConflictError`). That one, and a
|
|
672
|
+
* failure in the collection's own reads or deletes, can be raised **after** the publish already landed, so a throw
|
|
673
|
+
* does not by itself mean the load did not take effect — re-read the pointer rather than assuming. A collection by
|
|
674
|
+
* name that finds the segment changed returns an empty `collected` instead.
|
|
675
|
+
*
|
|
676
|
+
* **A `TransientError` from the registry write can leave the publish unsettled, and deletes nothing.** The
|
|
677
|
+
* generation's object is sent again after a throttle where the backend allows it (a write id tells a first send
|
|
678
|
+
* that landed from another writer's object), and a driver sends each registry write once. When that write ends
|
|
679
|
+
* without an answer, the load reads the row: its own landed write is `published: true`, a row that has moved on is
|
|
680
|
+
* `superseded`, and a row still as the write found it gets a fresh compare-and-swap from the version just read, at
|
|
681
|
+
* most three times, after a wait on the store's clock. Still unanswered, the load throws the registry's `TransientError` and keeps its object, which
|
|
682
|
+
* a write may still point the row at. Re-run the load: it numbers past that object, and collection removes it once a
|
|
683
|
+
* generation above it is current. A `'superseded'` refusal that follows an unanswered write sets `unanswered: true`
|
|
684
|
+
* on its audit event: that write may have landed, and the generation been current for a while, first.
|
|
635
685
|
*
|
|
636
686
|
* Needs a backend (throws {@link UnsupportedError} otherwise).
|
|
637
687
|
*/
|
|
638
|
-
load(ref: SegmentRef,
|
|
688
|
+
load(ref: SegmentRef, input: LoadInput, options?: LoadOptions): Promise<LoadResult>;
|
|
639
689
|
/**
|
|
640
690
|
* **Dispose of a segment — tombstone it, then delete its storage objects.** Irreversible.
|
|
641
691
|
*
|
|
@@ -661,9 +711,12 @@ export declare class CloudRoaring {
|
|
|
661
711
|
* forever and quietly. Branch on `dropped`, and treat `reason: 'absent'` as the alert.
|
|
662
712
|
*
|
|
663
713
|
* **Inspect `generationsRemaining`.** Empty is the normal outcome; non-empty means the storage was NOT fully
|
|
664
|
-
* reclaimed and the drop should be re-run. A load that
|
|
665
|
-
*
|
|
666
|
-
* could not remove rather than returning a result that looks like a
|
|
714
|
+
* reclaimed and the drop should be re-run. A load that had read the segment before the tombstone landed, whether it
|
|
715
|
+
* was writing or still consuming its ids, can still write its object, so a single sweep can miss it — this call
|
|
716
|
+
* re-sweeps and then reports whatever it still could not remove rather than returning a result that looks like a
|
|
717
|
+
* clean drop. A load that writes after the last sweep deletes its own object once its publish is refused; only one
|
|
718
|
+
* whose process stops in between, or whose publish fails without a definite answer (a lost response, a timeout),
|
|
719
|
+
* leaves it, for a re-run of the drop.
|
|
667
720
|
*
|
|
668
721
|
* Reads become empty within `cache.genTtlMs` (default 2 s), not instantly: a store that had already read this
|
|
669
722
|
* segment may answer from its cached generation + cached chunks until that window lapses, or, while the registry
|
|
@@ -758,6 +811,12 @@ export declare class CloudRoaring {
|
|
|
758
811
|
* wall-clock knob too, and `retired` counts deletions only — a dry run reports `wouldRetire` instead, so a
|
|
759
812
|
* dashboard summing `retired` can never show a phantom deletion.
|
|
760
813
|
*
|
|
814
|
+
* **Check `purgeFaults`.** A delete the registry refuses for a reason other than a lost race (a policy that denies
|
|
815
|
+
* delete, an Azure blob with a snapshot, a raw provider error), whether of a tombstone or of a due-index pointer, is
|
|
816
|
+
* counted there, with the first one's reason in `firstPurgeFault`. A refused purge is `skipped` in the ledger, is not
|
|
817
|
+
* charged to `limit`, so it never holds the retirements behind it, and purging stops for the rest of the call after
|
|
818
|
+
* three refused purges in a row (a purge that succeeds starts the count again).
|
|
819
|
+
*
|
|
761
820
|
* It also **deletes the tombstone rows its own past retirements left**, after `tombstoneGraceMs` (default 24 h)
|
|
762
821
|
* and only once that segment's Storage generations are provably gone — collecting a straggler generation itself
|
|
763
822
|
* first, since nothing else ever would for a tombstoned segment. Attribution is a **positive marker the sweep
|
|
@@ -765,7 +824,12 @@ export declare class CloudRoaring {
|
|
|
765
824
|
* `retention` untouched, so setting a policy and then honouring a right-to-erasure request mid-window produces
|
|
766
825
|
* exactly that row, and deleting it would destroy the Art. 17 attestation and un-fence the name. By default a
|
|
767
826
|
* segment that held nothing has its row deleted in the pass that retires it, since that row would only fence the name.
|
|
768
|
-
* Pass `purgeTombstones: false` to keep every tombstone, that row included.
|
|
827
|
+
* Pass `purgeTombstones: false` to keep every tombstone, that row included. On a backend whose registry reports
|
|
828
|
+
* `conditionalDelete` (AWS S3, Azure Blob, the local filesystem and memory, by default; GCS when you set `conditionalDelete: true`)
|
|
829
|
+
* the purge removes the row from the bucket for good, by a delete the store applies only to the version it judged,
|
|
830
|
+
* so a full sweep reads what is live or inside its grace rather than every name a namespace ever held; elsewhere,
|
|
831
|
+
* and for a row a release before 0.12 wrote, it leaves a tombstone. Each retirement files a pointer in the due index
|
|
832
|
+
* under the day its tombstone's grace ends, so `scan: 'index'` purges as well as retires.
|
|
769
833
|
*
|
|
770
834
|
* Needs the store built with a **backend** (throws {@link UnsupportedError} otherwise),
|
|
771
835
|
* because retiring a segment deletes its storage objects. `now` defaults to the store's clock.
|
|
@@ -851,6 +915,13 @@ export declare class CloudRoaring {
|
|
|
851
915
|
* (throws {@link UnsupportedError} otherwise). `destroyed` (crypto-shredded) segments are skipped. Pair it with
|
|
852
916
|
* the DR runbook (docs/guide/disaster-recovery.md).
|
|
853
917
|
*
|
|
918
|
+
* A segment whose pointer names a missing object is the torn restore this reports; a cold `count()` of it still
|
|
919
|
+
* answers the number its row records, while a read of the object throws, so the count alone never shows it. With
|
|
920
|
+
* `summaries: true` it also opens each segment's current object (one tail read each) and reports a segment whose
|
|
921
|
+
* row's summary says another id count or metadata than the object holds as `summary-mismatch`, which is what a
|
|
922
|
+
* restore of the registry from another point than the bucket leaves a count to answer. A sealed summary is held
|
|
923
|
+
* against its object when this store has the key, and counted in `summariesUnchecked` when it does not.
|
|
924
|
+
*
|
|
854
925
|
* It holds the registry rows it enumerates resident, at most 250,000 of them, and throws
|
|
855
926
|
* `BudgetExceededError` past that rather than report a partial scan as a whole one. This method takes no
|
|
856
927
|
* ceiling of its own: narrow the scan with `namespace`, or call `runConsistencyCheck` over the backend's
|
|
@@ -859,6 +930,7 @@ export declare class CloudRoaring {
|
|
|
859
930
|
checkConsistency(options?: {
|
|
860
931
|
namespace?: string;
|
|
861
932
|
concurrency?: number;
|
|
933
|
+
summaries?: boolean;
|
|
862
934
|
}): Promise<ConsistencyReport>;
|
|
863
935
|
/**
|
|
864
936
|
* **Export ("eject") every registered segment's current generation** through the injected `sink`, using only
|
|
@@ -954,14 +1026,22 @@ export interface MaterializeOptions extends CombineOptions {
|
|
|
954
1026
|
readonly allowEmpty?: boolean;
|
|
955
1027
|
/** Refuse an implausible result rather than publish it. Same bounds, and same meaning, as on `load()`. */
|
|
956
1028
|
readonly guard?: LoadGuard;
|
|
1029
|
+
/**
|
|
1030
|
+
* Metadata for the generation this call publishes, under the rules and with the meaning of
|
|
1031
|
+
* {@link LoadOptions.metadata}: small, immutable, published with the pointer, never a subject's id. A value that
|
|
1032
|
+
* breaks a rule throws {@link ValidationError} before any request is made.
|
|
1033
|
+
*/
|
|
1034
|
+
readonly metadata?: GenerationMetadata;
|
|
957
1035
|
/**
|
|
958
1036
|
* Generations to keep below the new pointer — see {@link LoadOptions.keep}. A value that is not a non-negative
|
|
959
1037
|
* integer throws `ValidationError`.
|
|
960
1038
|
*
|
|
961
|
-
* **Defaults to keeping everything**, unlike `load()`, which keeps 1 and collects
|
|
1039
|
+
* **Defaults to keeping everything**, unlike `load()`, which keeps 1 and collects what it supersedes. An operator's
|
|
962
1040
|
* recovery story can depend on a materialisation collecting nothing: `rollbackSegment` refuses a target that
|
|
963
1041
|
* has been collected. Pass a number to collect on the way through; `0` keeps only the
|
|
964
|
-
* generation this call publishes.
|
|
1042
|
+
* generation this call publishes. It collects by listing the destination, so it clears every generation below the
|
|
1043
|
+
* new one beyond `keep`, however many earlier calls kept, where a `load()` deletes by name the one generation its
|
|
1044
|
+
* publish pushes out of the window.
|
|
965
1045
|
*/
|
|
966
1046
|
readonly keep?: number;
|
|
967
1047
|
}
|
|
@@ -973,6 +1053,14 @@ export interface MaterializeOptions extends CombineOptions {
|
|
|
973
1053
|
* solves the same problem the same way, by taking {@link BaseCombineOptions}.
|
|
974
1054
|
*/
|
|
975
1055
|
export type AndNotIntoOptions = Omit<MaterializeOptions, 'exclude'>;
|
|
1056
|
+
/** What {@link Segment.stat} answers: the generation a handle reads, its id count and its metadata. */
|
|
1057
|
+
export interface SegmentStat {
|
|
1058
|
+
/** The generation read, or `null` when the segment has none. */
|
|
1059
|
+
readonly generation: number | null;
|
|
1060
|
+
readonly cardinality: number;
|
|
1061
|
+
/** The metadata the generation was loaded with; absent when it has none. */
|
|
1062
|
+
readonly metadata?: GenerationMetadata;
|
|
1063
|
+
}
|
|
976
1064
|
/**
|
|
977
1065
|
* A handle bound to one segment — the read verbs, plus the three `*Into` verbs that write a **new generation**
|
|
978
1066
|
* of another segment.
|
|
@@ -1121,16 +1209,39 @@ export declare class Segment {
|
|
|
1121
1209
|
/** Membership: one chunk — the cache, else one ranged GET. Throws {@link ValidationError} on a bad id. */
|
|
1122
1210
|
has(id: number): Promise<boolean>;
|
|
1123
1211
|
/**
|
|
1124
|
-
* Cardinality
|
|
1125
|
-
*
|
|
1126
|
-
*
|
|
1127
|
-
*
|
|
1212
|
+
* Cardinality of the generation this handle reads: **one registry read when cold, none when warm, and no read of the
|
|
1213
|
+
* object**, with **zero payload reads**. The registry row records the id count of the generation it names, written
|
|
1214
|
+
* by the write that made it current, and a count answers from that. A row with no summary it can use (one written
|
|
1215
|
+
* before rows carried it, one that names another generation, a sealed one that does not open) sends the count to the
|
|
1216
|
+
* `.crbm` index, summed with a tail read of the object. A segment whose pointer names a missing object (a torn
|
|
1217
|
+
* restore) still counts the row's number, while a read of the object throws; `checkConsistency` is what finds it.
|
|
1218
|
+
*
|
|
1219
|
+
* What this trusts: the row's summary, or the index's sum, and no payload is decoded to confirm it. The summary is
|
|
1220
|
+
* used only for the generation it names, on an active row, and a sealed one only if it opens under the segment's
|
|
1221
|
+
* key; it is held against the object whenever the object is opened anyway (a `has`, an `iterate`, a combine, a
|
|
1222
|
+
* `pin()`), at no extra request, and a disagreement stops this store using it, and fails nothing. It is not
|
|
1223
|
+
* confirmed on the cold path, so a party who can write the registry row can make a count wrong. The index is checked for internal consistency when the object is opened (each key in
|
|
1128
1224
|
* range and ascending, each cardinality in `1..65536`, each payload inside the payload region, and, on an
|
|
1129
1225
|
* unencrypted object, the footer's chunk count and total agreeing with the index), and a corrupt index that is
|
|
1130
1226
|
* still internally consistent yields a wrong count. `iterate()` and the combines decode the payloads, whose
|
|
1131
1227
|
* structure is checked.
|
|
1132
1228
|
*/
|
|
1133
1229
|
count(): Promise<number>;
|
|
1230
|
+
/**
|
|
1231
|
+
* What the generation this handle reads is, from one resolution: its number, its id count and the metadata it
|
|
1232
|
+
* was loaded with (absent when it has none). It is what answers {@link Segment.count}, so the three describe one
|
|
1233
|
+
* generation and cannot straddle a publish. One registry read when cold, none while warm, and none on a pinned
|
|
1234
|
+
* handle, which answers for the generation it pinned. A segment with no generation, and an expired handle,
|
|
1235
|
+
* answer `{ generation: null, cardinality: 0 }`.
|
|
1236
|
+
*
|
|
1237
|
+
* Trust is as for `count()`: the registry row's word, not confirmed against the object until the object is
|
|
1238
|
+
* opened, when a disagreement makes this process stop using that row's summary.
|
|
1239
|
+
*
|
|
1240
|
+
* ```ts
|
|
1241
|
+
* const { generation, cardinality, metadata } = await store.segment('active-30d').stat();
|
|
1242
|
+
* ```
|
|
1243
|
+
*/
|
|
1244
|
+
stat(): Promise<SegmentStat>;
|
|
1134
1245
|
/**
|
|
1135
1246
|
* Every id, ascending, streamed one chunk at a time. Pass a range to read part of the segment: the ids in
|
|
1136
1247
|
* `(after, through]`, fetching only the chunks the range overlaps. The per-op budget is charged for every chunk in
|
|
@@ -1242,4 +1353,4 @@ export declare class Segment {
|
|
|
1242
1353
|
}): Promise<CostReport>;
|
|
1243
1354
|
}
|
|
1244
1355
|
export { MemoryStorage, LocalFsStorage, CrbmStorageChunkSource, destroySegment, eraseNamespace, excludingReservedRows, readRetentionPolicy, MIN_EXPIRES_AT_MS, InProcessKeystore, NodeAead, aadFor, CloudRoaringError, ValidationError, WriteConflictError, IntegrityError, NotFoundError, UnsupportedError, CapabilityError, TransientError, KeyUnavailableError, BudgetExceededError, isCloudRoaringError, isWriteConflictError, isTransientError, isNotFoundError, isIntegrityError, isValidationError, CrbmReader, BufferReader, CountingMetricsSink, RecordingAuditSink, DEFAULT_RETRY_POLICY, AWS_US_EAST_1_ONDEMAND, ELASTICACHE_REDIS_US_EAST_1_ONDEMAND, ONE_REDIS_HA_CLUSTER, } from '@cloudbitmaps/core';
|
|
1245
|
-
export type { Aead, AeadSealed, AuditEvent, BlobReader, BlobSink, Budget, BudgetOption, ChunkRef, Clock, CodecBitmap, CodecInterface, ConsistencyErrorEntry, ConsistencyIssue, ConsistencyReport, CostReport, CrbmCrypto, CrbmReaderOptions, CrbmStorageChunkSourceOptions, DestroyResult, DropResult, EraseDeps, EstimateInput, ExportFailure, ExportFormat, ExportManifest, ExportOptions, ExportSink, ExportWriter, ExportedSegment, GenKey, GenerationEntry, GovernanceMeta, IAuditSink, IKeystore, IMetricsSink, IRegistryDriver, IStorageDriver, IdRange, InProcessKeystoreOptions, LoadGuard, LoadOptions, LoadRefusal, LoadResult, LocalFsStorageOptions, MemoryStorageOptions, MetricEvent, MetricOpName, MetricsSnapshot, NewRegistryRecord, PinnedAt, PinnedObject, PricingProfile, RedisNodeType, RedisSizing, RegCaps, RegistryPatch, RegistryRecord, RegistryStatus, RetentionPolicy, RetireEntry, RetireExpiredOptions, RetireExpiredResult, RetryPolicy, Rng, RollbackResult, SegmentInfo, SegmentRef, SegmentSize, SegmentSizing, SetRetentionResult, StorageBackend, StorageCaps, StorageChunkSource, Token, Workload, WrappedDek, } from '@cloudbitmaps/core';
|
|
1356
|
+
export type { Aead, AeadSealed, AuditEvent, BlobReader, BlobSink, Budget, BudgetOption, ChunkRef, Clock, CodecBitmap, CodecInterface, ConsistencyErrorEntry, ConsistencyIssue, ConsistencyReport, CostReport, CrbmCrypto, CrbmReaderOptions, CrbmStorageChunkSourceOptions, DestroyResult, ClearRegistrySummary, DropResult, EncodedChunk, EraseDeps, EstimateInput, ExportFailure, ExportFormat, ExportManifest, ExportOptions, ExportSink, ExportWriter, ExportedSegment, GenKey, GenerationEntry, GenerationMetadata, GovernanceMeta, IAuditSink, IKeystore, IMetricsSink, IRegistryDriver, IStorageDriver, IdRange, InProcessKeystoreOptions, LoadGuard, LoadInput, LoadOptions, LoadRefusal, LoadResult, LocalFsStorageOptions, MemoryStorageOptions, MetricEvent, MetricOpName, MetricsSnapshot, NewRegistryRecord, PinnedAt, PinnedObject, PortableBitmap, PricingProfile, RedisNodeType, RedisSizing, RegCaps, RegistryPatch, RegistryRecord, RegistryStatus, RegistrySummary, RetentionPolicy, RetireEntry, RetireExpiredOptions, RetireExpiredResult, RetryPolicy, Rng, RollbackResult, SealedRegistrySummary, SegmentInfo, SegmentRef, SegmentSize, SegmentSizing, SetRetentionResult, StorageBackend, StorageCaps, StorageChunkSource, Token, Workload, WrappedDek, } from '@cloudbitmaps/core';
|