@cloudbitmaps/roaring 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +16 -0
- package/README.md +4 -4
- package/dist/bin/export-segments.js +531 -59
- package/dist/bin/export-segments.js.map +4 -4
- package/dist/index.d.ts +223 -36
- package/dist/index.js +543 -60
- package/dist/index.js.map +4 -4
- package/dist/lease-guards.d.ts +34 -0
- package/dist/roaring-codec.d.ts +23 -0
- package/package.json +2 -2
package/dist/index.d.ts
CHANGED
|
@@ -23,8 +23,9 @@
|
|
|
23
23
|
* `checkConsistency`, `exportSegments` — reuse the store's own drivers, so you never re-pass them (they need the
|
|
24
24
|
* store built with a backend). See the README and the getting-started guide.
|
|
25
25
|
*/
|
|
26
|
-
import type { BudgetOption, GenerationEntry, GenerationMetadata, LoadGuard, LoadInput, LoadOptions, LoadRefusal, LoadResult, RollbackResult, Clock, StorageChunkSource, ConsistencyReport, CostReport, DropResult, EstimateInput, ExportManifest, ExportOptions, ExportSink, IAuditSink, IStorageDriver, IKeystore, IMetricsSink, StorageBackend, PricingProfile, RetentionPolicy, RetireExpiredOptions, RetireExpiredResult, RetryPolicy, Rng, SetRetentionResult, PinnedAt, IdRange, SegmentRef, Workload } from '@cloudbitmaps/core';
|
|
26
|
+
import type { BudgetOption, GenerationEntry, GenerationMetadata, LoadGuard, LoadInput, LoadOptions, LoadRefusal, LoadResult, ReapRegistryTombstonesOptions, ReapRegistryTombstonesResult, RollbackResult, Clock, StorageChunkSource, ConsistencyReport, CostReport, DropResult, EstimateInput, ExportManifest, ExportOptions, ExportSink, IAuditSink, IStorageDriver, IKeystore, IMetricsSink, StorageBackend, PricingProfile, RetentionPolicy, RetireExpiredOptions, RetireExpiredResult, RetryPolicy, Rng, SetRetentionResult, PinnedAt, IdRange, SegmentRef, Workload } from '@cloudbitmaps/core';
|
|
27
27
|
import type { SegmentInfo } from '@cloudbitmaps/core';
|
|
28
|
+
import { deserializePortable } from './roaring-codec.js';
|
|
28
29
|
/**
|
|
29
30
|
* Wiring for a {@link CloudRoaring} store. **Only `storage` is required** — the minimal call is
|
|
30
31
|
* `new CloudRoaring({ storage: backend })`. A backend resolves generations with one strong read, reads
|
|
@@ -657,24 +658,26 @@ export declare class CloudRoaring {
|
|
|
657
658
|
* A refused load deletes the object it wrote before returning — it sits above `currentGen`, where generation
|
|
658
659
|
* collection deliberately never looks — but only while the segment's registry row is unchanged or gone. Once
|
|
659
660
|
* another write has changed the row, the generation number it holds may name another incarnation's live object,
|
|
660
|
-
* so it leaves the orphan rather than risk deleting live data.
|
|
661
|
-
*
|
|
661
|
+
* so it leaves the orphan rather than risk deleting live data. No row's list names it, so it takes no place in a
|
|
662
|
+
* window, and a listing deletes it once a later generation is current above it.
|
|
662
663
|
*
|
|
663
|
-
* **Collection is by name
|
|
664
|
-
* pointer deletes the
|
|
665
|
-
* objects on every sixteenth generation, and on any load that met an object above the
|
|
666
|
-
* the current generation's object gone, to take what the name-only passes leave, such as
|
|
667
|
-
*
|
|
668
|
-
* the
|
|
664
|
+
* **Collection is by name at any `keep` up to 64.** The segment's row records the generations a load keeps, so a load
|
|
665
|
+
* that found nothing above the pointer deletes the generations its publish pushed out of the window and lists
|
|
666
|
+
* nothing. It lists the segment's objects on every sixteenth generation, and on any load that met an object above the
|
|
667
|
+
* pointer or whose check found the current generation's object gone, to take what the name-only passes leave, such as
|
|
668
|
+
* an object a crashed load left below the pointer. A `keep` above 64 records no list and lists on every load, and so
|
|
669
|
+
* does the first load of a row that records none, which then records it. {@link LoadResult.collected} then names
|
|
670
|
+
* what the pass deleted by name, and one name may have been gone already.
|
|
669
671
|
*
|
|
670
672
|
* What it **throws** for is a fault rather than an outcome: invalid options or ids, and a crypto-shredded
|
|
671
673
|
* segment (`ValidationError`); a key the keystore cannot provide (`KeyUnavailableError`); a current generation
|
|
672
674
|
* that will not open when a guard has to read its size, which it does only when the row has no usable summary of it
|
|
673
|
-
* (`IntegrityError`); a driver failure; and a collection
|
|
674
|
-
*
|
|
675
|
-
*
|
|
676
|
-
*
|
|
677
|
-
*
|
|
675
|
+
* (`IntegrityError`); a driver failure; and a fault in the collection's own reads, listing or deletes, or in the write
|
|
676
|
+
* that records the window (anything but a lost race). A collection fault can be raised **after** the publish already
|
|
677
|
+
* landed, so a throw does not by itself mean the load did not take effect — re-read the pointer rather than assuming.
|
|
678
|
+
* A collection, by name or by listing, that finds another writer has moved the row since the publish (a load, a
|
|
679
|
+
* rollback, a purge) stops and returns what it deleted in `collected`: a load that took effect does not throw for a
|
|
680
|
+
* race it won.
|
|
678
681
|
*
|
|
679
682
|
* **A `TransientError` from the registry write can leave the publish unsettled, and deletes nothing.** The
|
|
680
683
|
* generation's object is sent again after a throttle where the backend allows it (a write id tells a first send
|
|
@@ -840,6 +843,55 @@ export declare class CloudRoaring {
|
|
|
840
843
|
retireExpired(options?: Omit<RetireExpiredOptions, 'now'> & {
|
|
841
844
|
now?: number;
|
|
842
845
|
}): Promise<RetireExpiredResult>;
|
|
846
|
+
/**
|
|
847
|
+
* **Remove the registry rows that releases before 0.12.0 left as `deleted: true` envelopes.** Those releases never
|
|
848
|
+
* removed a row they deleted, and no later call removes it either: it is invisible to every read, and every full
|
|
849
|
+
* listing of the registry still reads it (one GET each). This is the one call that removes them. It is an admin call,
|
|
850
|
+
* not a sweep, and nothing schedules it: run it once after the last process on a release before 0.12.0 is gone.
|
|
851
|
+
*
|
|
852
|
+
* ```ts
|
|
853
|
+
* const preview = await store.reapRegistryTombstones({ dryRun: true });
|
|
854
|
+
* console.log(`would remove ${preview.wouldReap} of ${preview.examined} rows read`);
|
|
855
|
+
*
|
|
856
|
+
* const done = await store.reapRegistryTombstones({ confirmNoLegacyWriters: true });
|
|
857
|
+
* if (done.limited) runAgain(); // `limit` (default 1,000) stopped the run with rows unread
|
|
858
|
+
* ```
|
|
859
|
+
*
|
|
860
|
+
* **What it removes.** Only a row that is `deleted: true` and whose token carries no incarnation id, which is what a
|
|
861
|
+
* release before 0.12.0 left, or a 0.12.0 or later release left by deleting a row born before 0.12.0 (both are safe to
|
|
862
|
+
* remove), whatever its `status`. It never touches a live row, a `destroyed` row that is not `deleted` (a crypto-shred's
|
|
863
|
+
* or a `dropSegment`'s tombstone: the attestation of an erasure, which it cannot tell from another), or a deleted row
|
|
864
|
+
* with an incarnation id, and it never touches a generation. **So it does not clean a bucket completely**: the
|
|
865
|
+
* `destroyed` tombstones `dropSegment` leaves stay, so do live rows written before 0.12.0, and so does a tombstone a
|
|
866
|
+
* 0.12.0 or later release wrote while `conditionalDelete` was off (GCS by default, S3 on a custom endpoint): every full
|
|
867
|
+
* listing still reads it, and this call neither removes it nor runs on such a store.
|
|
868
|
+
* It also covers the library's own bookkeeping rows when `namespace` is not set, by the same rule. The default
|
|
869
|
+
* namespace cannot be scoped to: leave `namespace` out to cover it.
|
|
870
|
+
*
|
|
871
|
+
* **The two things it asks.** `confirmNoLegacyWriters: true` is required for a real run (without it, a
|
|
872
|
+
* `ValidationError`, and nothing is requested): your statement that no process on a release before 0.12.0 writes
|
|
873
|
+
* this registry, since one that did would, re-creating a removed name, issue the removed row's tokens again. And the
|
|
874
|
+
* registry must apply a delete under a precondition: where its `conditionalDelete` capability is off (GCS by default,
|
|
875
|
+
* S3 on a custom endpoint) it throws `CapabilityError` before any request, dry run included. A registry that keeps
|
|
876
|
+
* no such rows (the in-memory and local-filesystem ones) throws `UnsupportedError`.
|
|
877
|
+
*
|
|
878
|
+
* **Each removal is fenced** on the version it read, so a `create` that lands over the envelope first wins, the
|
|
879
|
+
* delete is refused, and the row is counted as `skipped.raced` (which also counts a row already gone). That holds where
|
|
880
|
+
* the backend applies the precondition, which auto-detection checks (S3 only on an AWS host). On an endpoint that
|
|
881
|
+
* ignores `If-Match` (MinIO, fake-gcs-server), a `conditionalDelete: true` you set makes this an unfenced delete: run
|
|
882
|
+
* it with every writer stopped. On a versioned bucket a delete leaves a delete marker and the earlier version, record
|
|
883
|
+
* included, so the name is not erased from the bucket. A `create` that had already read the envelope and
|
|
884
|
+
* whose write meets the removal throws `WriteConflictError`, which this library does not retry: retry the create, and
|
|
885
|
+
* it finds no row and writes one.
|
|
886
|
+
*
|
|
887
|
+
* **Cost** over R rows read and E removed: `ceil(R / 1000)` LIST requests, R GETs and E DELETEs (a dry run, no
|
|
888
|
+
* DELETEs). `limit` bounds the removals, and the run stops listing once it is spent. **A run is not resumable**: each
|
|
889
|
+
* lists and reads from the start (R GETs whatever the `limit`), a dry run with a `limit` reports the same first rows
|
|
890
|
+
* every time, so `wouldReap` is a total only when `limited` is `false`, and `limited` is `true` whenever the limit was
|
|
891
|
+
* spent with keys left, removable or not. An unreadable object stops the run with an error naming its key, and
|
|
892
|
+
* removes nothing more: it may be a newer release's row. Scope the run with `namespace` to get past one.
|
|
893
|
+
*/
|
|
894
|
+
reapRegistryTombstones(options?: ReapRegistryTombstonesOptions): Promise<ReapRegistryTombstonesResult>;
|
|
843
895
|
/**
|
|
844
896
|
* **Forget everything this store has cached about a segment.** Use it after destroying or retiring a segment
|
|
845
897
|
* through a path this store did not perform itself.
|
|
@@ -883,6 +935,8 @@ export declare class CloudRoaring {
|
|
|
883
935
|
* those of a live read that fetched across a publish (see {@link PinnedStorageChunkSource.currentVersion}).
|
|
884
936
|
*/
|
|
885
937
|
private pinSegment;
|
|
938
|
+
/** 16 hex digits from the store's `Rng`, which names the holder of one lease: unique, not secret. */
|
|
939
|
+
private mintHolder;
|
|
886
940
|
/**
|
|
887
941
|
* An engine reading every segment in `pins` at its pinned generation and everything else live, with the store's
|
|
888
942
|
* retries, so a transient fault on a pinned read is retried as it would be on a live one.
|
|
@@ -1004,8 +1058,9 @@ export interface BaseCombineOptions extends IdRange {
|
|
|
1004
1058
|
*/
|
|
1005
1059
|
export interface CombineOptions extends BaseCombineOptions {
|
|
1006
1060
|
/**
|
|
1007
|
-
* Segments whose ids are subtracted from the result. An **expired** handle here
|
|
1008
|
-
*
|
|
1061
|
+
* Segments whose ids are subtracted from the result. An **expired** handle here is refused, in every combine:
|
|
1062
|
+
* the stream throws {@link ValidationError} naming it, before any request is made, exactly as in
|
|
1063
|
+
* {@link Segment.andNot}.
|
|
1009
1064
|
*/
|
|
1010
1065
|
readonly exclude?: Segment[];
|
|
1011
1066
|
}
|
|
@@ -1053,7 +1108,7 @@ export interface MaterializeOptions extends CombineOptions {
|
|
|
1053
1108
|
* recovery story can depend on a materialisation collecting nothing: `rollbackSegment` refuses a target that
|
|
1054
1109
|
* has been collected. Pass a number to collect on the way through; `0` keeps only the
|
|
1055
1110
|
* generation this call publishes. It collects by listing the destination, so it clears every generation below the
|
|
1056
|
-
* new one beyond `keep`, however many earlier calls kept, where a `load()` deletes by name the
|
|
1111
|
+
* new one beyond `keep`, however many earlier calls kept, where a `load()` deletes by name the generations its
|
|
1057
1112
|
* publish pushes out of the window.
|
|
1058
1113
|
*/
|
|
1059
1114
|
readonly keep?: number;
|
|
@@ -1096,6 +1151,32 @@ export interface IdStream extends AsyncIterable<number> {
|
|
|
1096
1151
|
* solves the same problem the same way, by taking {@link BaseCombineOptions}.
|
|
1097
1152
|
*/
|
|
1098
1153
|
export type AndNotIntoOptions = Omit<MaterializeOptions, 'exclude'>;
|
|
1154
|
+
/** What {@link Segment.pin} and {@link Segment.pinAt} take besides the generation. */
|
|
1155
|
+
export interface PinOptions {
|
|
1156
|
+
/**
|
|
1157
|
+
* Hold the pinned generation until this instant: epoch-**milliseconds**, after now and at most
|
|
1158
|
+
* {@link MAX_LEASE_MS} (14 days) from it. A load's collection leaves a leased generation in the bucket until the lease
|
|
1159
|
+
* ends, and a read of the handle after it throws {@link LeaseExpiredError}, never empty.
|
|
1160
|
+
*/
|
|
1161
|
+
readonly leaseUntil?: number;
|
|
1162
|
+
}
|
|
1163
|
+
/** The lease a pinned handle holds, as {@link Segment.lease} reports it. */
|
|
1164
|
+
export interface Lease {
|
|
1165
|
+
/** 16 hex digits naming this lease in the segment's registry row. */
|
|
1166
|
+
readonly holder: string;
|
|
1167
|
+
/** Epoch-ms the lease runs to on this store's clock. */
|
|
1168
|
+
readonly until: number;
|
|
1169
|
+
}
|
|
1170
|
+
/**
|
|
1171
|
+
* The generation {@link Segment.pinAt} reopens: what an earlier pin recorded in {@link Segment.pinnedAt}. The
|
|
1172
|
+
* fingerprint is required, since a generation number is taken again after a purge and re-create.
|
|
1173
|
+
*/
|
|
1174
|
+
export interface PinAt {
|
|
1175
|
+
/** The generation number an earlier pin held. */
|
|
1176
|
+
readonly generation: number;
|
|
1177
|
+
/** The fingerprint of that generation's object, from the earlier pin's `pinnedAt.fingerprint`. */
|
|
1178
|
+
readonly fingerprint: string;
|
|
1179
|
+
}
|
|
1099
1180
|
/** What {@link Segment.stat} answers: the generation a handle reads, its id count and its metadata. */
|
|
1100
1181
|
export interface SegmentStat {
|
|
1101
1182
|
/** The generation read, or `null` when the segment has none. */
|
|
@@ -1130,6 +1211,13 @@ export declare class Segment {
|
|
|
1130
1211
|
* pinned handle passed as an **operand** is still read at its pin rather than live.
|
|
1131
1212
|
*/
|
|
1132
1213
|
readonly pinnedAt?: PinnedAt;
|
|
1214
|
+
/**
|
|
1215
|
+
* The lease this handle holds, when it came from `pin({ leaseUntil })`; `undefined` otherwise. It reports what was
|
|
1216
|
+
* taken and does not change when the lease ends: a read after `lease.until` (or after {@link Segment.release}) throws
|
|
1217
|
+
* {@link LeaseExpiredError}.
|
|
1218
|
+
*/
|
|
1219
|
+
readonly lease?: Lease;
|
|
1220
|
+
private readonly leaseHold?;
|
|
1133
1221
|
/**
|
|
1134
1222
|
* Not constructible: a handle comes from {@link CloudRoaring.segment}, which wires it to the store's engine,
|
|
1135
1223
|
* caches and write path. The constructor takes those internals, so it is not part of the surface.
|
|
@@ -1161,11 +1249,20 @@ export declare class Segment {
|
|
|
1161
1249
|
* 0, so a pin of the old segment never reads the new one: what it has already read still answers, as the
|
|
1162
1250
|
* instant it pinned, and anything it would have to fetch fails with `NotFoundError`, as a swept pin's does.
|
|
1163
1251
|
*
|
|
1164
|
-
* **It is a hold,
|
|
1165
|
-
* you: a pinned read deliberately does **not** heal forward, because silently
|
|
1166
|
-
* is the one thing a pin exists to prevent, so it fails instead. Size `keep` to cover
|
|
1167
|
-
* job — see [Generations and `keep`](https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/loading.md#generations-and-keep) —
|
|
1168
|
-
* you are not collecting.
|
|
1252
|
+
* **It is a hold, and `pin({ leaseUntil })` makes it a bounded one.** Without a lease nothing stops a collection
|
|
1253
|
+
* deleting the generation underneath you: a pinned read deliberately does **not** heal forward, because silently
|
|
1254
|
+
* serving a different generation is the one thing a pin exists to prevent, so it fails instead. Size `keep` to cover
|
|
1255
|
+
* your longest pinned job — see [Generations and `keep`](https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/loading.md#generations-and-keep) — take the pin on a segment
|
|
1256
|
+
* you are not collecting, or lease it.
|
|
1257
|
+
*
|
|
1258
|
+
* **A lease** (`leaseUntil`: epoch **milliseconds**, after now and at most 14 days out) keeps the generation out of
|
|
1259
|
+
* a load's collection until then, from any process, and the pin is taken with it: the lease is written to the segment's
|
|
1260
|
+
* row before the object is opened. A read of the handle after the lease, or after {@link Segment.release}, throws
|
|
1261
|
+
* {@link LeaseExpiredError} at every read site, including when the handle is an operand or an `exclude` of a combine
|
|
1262
|
+
* or the target of an `*Into`, and never reads empty. A stream checks the lease each time it reads a chunk. A segment
|
|
1263
|
+
* holds at most 64 live leases ({@link LeaseLimitError}). A collector holds a lease 60 seconds past its end, which covers
|
|
1264
|
+
* clocks that differ by that much. Erasure, shred, drop and retention expiry ignore a lease. See
|
|
1265
|
+
* [Hold a generation for a job](https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/reading.md#hold-a-generation-for-a-job-a-lease).
|
|
1169
1266
|
*
|
|
1170
1267
|
* A segment with no current generation pins nothing and reads empty, exactly as it would unpinned. A pinned
|
|
1171
1268
|
* segment whose row is later dropped or destroyed fails once it must open its object again, rather than go empty
|
|
@@ -1205,7 +1302,55 @@ export declare class Segment {
|
|
|
1205
1302
|
* Needs a store built on the `.crbm` storage source (the default when you pass a backend or a raw driver). Throws
|
|
1206
1303
|
* {@link UnsupportedError} on a store wired with a pre-built source that cannot pin.
|
|
1207
1304
|
*/
|
|
1208
|
-
pin(): Promise<Segment>;
|
|
1305
|
+
pin(options?: PinOptions): Promise<Segment>;
|
|
1306
|
+
/**
|
|
1307
|
+
* Hold what this pin names until `leaseUntil`, or `undefined` for a pin with no lease. Refuses an option it does not
|
|
1308
|
+
* know, a value that is not epoch-milliseconds in the future, one past the longest lease, and one past the handle's own
|
|
1309
|
+
* `expiresAt`, before any request is made.
|
|
1310
|
+
*/
|
|
1311
|
+
private leaseUntilOf;
|
|
1312
|
+
/**
|
|
1313
|
+
* End this handle's lease now, so a load's collection may take its generation. Idempotent: a handle with no lease, or
|
|
1314
|
+
* one already released, does nothing and makes no request. Every read of the handle after it throws
|
|
1315
|
+
* {@link LeaseExpiredError} (reason `'released'`), including one already under way at its next chunk. A lease that has
|
|
1316
|
+
* ended by its own time needs no release: this then makes no request either, and the registry row's entry is pruned
|
|
1317
|
+
* by a later write. Otherwise it is one registry read and one write; a failure of either is thrown as the registry's own
|
|
1318
|
+
* typed error, and the entry then holds until its time ends, `release()` being safe to call again.
|
|
1319
|
+
*/
|
|
1320
|
+
release(): Promise<void>;
|
|
1321
|
+
/**
|
|
1322
|
+
* The error a read of this handle must throw, or `undefined` when it holds no lease or the lease is live. One property
|
|
1323
|
+
* read on a handle with no lease. It is not {@link Segment.expired}: that rule reads empty, and a lease must never.
|
|
1324
|
+
*/
|
|
1325
|
+
private leaseError;
|
|
1326
|
+
/**
|
|
1327
|
+
* Throw the first lease error among `handles`: this handle and every operand and exclude of a call. It runs before
|
|
1328
|
+
* any `expiresAt` rule and before the engine, so no operand, exclude or cached reader can answer past a lease.
|
|
1329
|
+
*/
|
|
1330
|
+
private assertLeases;
|
|
1331
|
+
/**
|
|
1332
|
+
* Reopen a generation a pin named earlier, as a pinned handle like the one {@link Segment.pin} returns, held at
|
|
1333
|
+
* `at.generation` instead of the current one. Pass what the earlier pin recorded, `{ generation, fingerprint }`
|
|
1334
|
+
* from its {@link Segment.pinnedAt}. A generation number alone is not an identity, since a purged and re-created
|
|
1335
|
+
* name starts again at 0, so the fingerprint is required: a bare number, a malformed fingerprint or a key other
|
|
1336
|
+
* than `generation` and `fingerprint` is a {@link ValidationError}. Identify a pin by its `generation` and
|
|
1337
|
+
* `fingerprint`: the handle's `pinnedAt.version` can differ from the earlier pin's.
|
|
1338
|
+
*
|
|
1339
|
+
* Throws {@link NotFoundError} when the generation is gone (collected, purged, or on a crypto-shredded segment),
|
|
1340
|
+
* is above the row's pointer, or is another object than the fingerprint names, and never reads empty. A rollback
|
|
1341
|
+
* deletes nothing, so once a later load moves the pointer past a generation it rolled back from, that one can be
|
|
1342
|
+
* reopened while its object is stored. Nothing keeps a generation alive unless you lease it: a later load's
|
|
1343
|
+
* collection can delete it, which is sized by `keep` on every writer, or held by `options.leaseUntil` (see
|
|
1344
|
+
* {@link PinOptions} and {@link Segment.pin}). A leased `pinAt` writes the lease before it verifies the object, and
|
|
1345
|
+
* releases it if the verify fails; for a generation below the pointer, a collector that read the row before the lease
|
|
1346
|
+
* landed can still delete the generation inside its own round trip, and the handle then fails with `NotFoundError`
|
|
1347
|
+
* like any swept pin. A leased handle is checked at every read site as a leased `pin()` is. With a registry it costs one row read and one tail read,
|
|
1348
|
+
* and a leased `pinAt` one conditional write to the row besides; the handle
|
|
1349
|
+
* then reads as one from `pin()` does, including its failure on a chunk it has not cached once the generation is
|
|
1350
|
+
* swept. Works for an encrypted segment. The first argument is an object so that fields can join it later; the lease
|
|
1351
|
+
* is in `options`, the second.
|
|
1352
|
+
*/
|
|
1353
|
+
pinAt(at: PinAt, options?: PinOptions): Promise<Segment>;
|
|
1209
1354
|
/**
|
|
1210
1355
|
* An opaque string that names this handle's segment, namespace included: two handles of one segment have the
|
|
1211
1356
|
* same key, a live handle and its pins among them. Use it as a `Map` key or a log field when you track handles
|
|
@@ -1244,6 +1389,8 @@ export declare class Segment {
|
|
|
1244
1389
|
* awaiting.
|
|
1245
1390
|
*/
|
|
1246
1391
|
private refuseIfExpired;
|
|
1392
|
+
/** `result` as is, or, when a handle of the call holds a lease, checking that lease each time it reads a chunk. */
|
|
1393
|
+
private guarded;
|
|
1247
1394
|
/**
|
|
1248
1395
|
* Time an op with the injected clock and emit an `op` metric on completion (success or throw). Skipped
|
|
1249
1396
|
* entirely when no sink is wired, so the default path pays nothing.
|
|
@@ -1301,9 +1448,40 @@ export declare class Segment {
|
|
|
1301
1448
|
* without checking its options, as every read of one does.
|
|
1302
1449
|
*/
|
|
1303
1450
|
iterate(options?: IdRange): IdStream;
|
|
1451
|
+
/**
|
|
1452
|
+
* **Pinned handles only.** The ids at 1-based ranks `n`, `2n`, `3n` …, counted over the ids in `(after, through]`
|
|
1453
|
+
* when a range is given, ascending: `everyNth(1000)` yields the 1,000th id, then the 2,000th, and so on. A last
|
|
1454
|
+
* partial window yields nothing, so a caller cutting a send into windows of `n` appends its own final end.
|
|
1455
|
+
*
|
|
1456
|
+
* It places each boundary from the per-chunk counts the pin already holds and reads only the chunks that hold one,
|
|
1457
|
+
* each once however many boundaries it holds, so it reads at most one chunk per boundary and never more chunks
|
|
1458
|
+
* than the object has. A range with `after` inside a chunk reads that chunk too, to count the ids at or below
|
|
1459
|
+
* `after`, even when fewer than `n` ids remain above it; when the chunks in range hold fewer than `n` ids in all,
|
|
1460
|
+
* nothing is fetched. `through` stops the read and no chunk past it is read, though a `through` inside a chunk may read that last chunk and
|
|
1461
|
+
* yield nothing from it. The reads and the read-ahead window are
|
|
1462
|
+
* those of {@link Segment.iterate}, and the per-op budget is charged before the first fetch, with an upper bound on
|
|
1463
|
+
* the chunks the read can take. A pinned read after the generation is swept throws {@link NotFoundError}.
|
|
1464
|
+
*
|
|
1465
|
+
* The ranks come from the index's counts, which are trusted as {@link Segment.count} trusts them. A chunk that is
|
|
1466
|
+
* read must hold the number of ids its index says, or the read throws {@link IntegrityError}; a chunk that is not
|
|
1467
|
+
* read is not checked, so an index that is wrong about one, and, on an unencrypted object, still adds up to the footer's
|
|
1468
|
+
* total, moves the ids after it to the wrong ranks. An encrypted object's index is authenticated and bound to the
|
|
1469
|
+
* object, so a wrong count there needs the key.
|
|
1470
|
+
*
|
|
1471
|
+
* A live handle is refused with {@link UnsupportedError} at the first read, because its counts and its chunks could
|
|
1472
|
+
* come from two generations and name the wrong id. `n` that is not a positive integer throws
|
|
1473
|
+
* {@link ValidationError} at the first read, as does a bad bound. An expired handle reads empty.
|
|
1474
|
+
*
|
|
1475
|
+
* ```ts
|
|
1476
|
+
* const audience = await store.segment('audience').pin();
|
|
1477
|
+
* const ends: number[] = [];
|
|
1478
|
+
* for await (const id of audience.everyNth(1_000)) ends.push(id); // the end of each window of 1,000
|
|
1479
|
+
* ```
|
|
1480
|
+
*/
|
|
1481
|
+
everyNth(n: number, options?: IdRange): AsyncIterable<number>;
|
|
1304
1482
|
/**
|
|
1305
1483
|
* The options a combine hands the engine, read once, when it is called ({@link readOptions}), with the `exclude`
|
|
1306
|
-
* it was given
|
|
1484
|
+
* it was given ({@link excludesOf}) and mapped down to the plain refs
|
|
1307
1485
|
* `core` takes. A method rather than a module function because `ref` is class-private — the encapsulation is
|
|
1308
1486
|
* worth more than the free function.
|
|
1309
1487
|
*
|
|
@@ -1313,15 +1491,23 @@ export declare class Segment {
|
|
|
1313
1491
|
*/
|
|
1314
1492
|
private refsIn;
|
|
1315
1493
|
/**
|
|
1316
|
-
* The handles in `options.exclude
|
|
1317
|
-
*
|
|
1318
|
-
|
|
1319
|
-
|
|
1494
|
+
* The handles in `options.exclude`, read once so a getter sees one call. With none this is one property read and
|
|
1495
|
+
* a shared empty list; otherwise it is the caller's own array, so the common case allocates nothing.
|
|
1496
|
+
*/
|
|
1497
|
+
private excludesOf;
|
|
1498
|
+
/**
|
|
1499
|
+
* The refusal for a combine whose **exclusion** has expired, or `undefined` when none has. Unlike an expired
|
|
1500
|
+
* operand, which reads empty, an exclusion is a suppression or opt-out list: left out silently, the result
|
|
1501
|
+
* would include the very ids it was passed to remove. So it is refused with a {@link ValidationError} naming each
|
|
1502
|
+
* expired exclusion, as the `*Into` verbs refuse an expired handle.
|
|
1320
1503
|
*
|
|
1321
|
-
*
|
|
1322
|
-
*
|
|
1504
|
+
* It is judged against the injected clock when the combine is called, before the engine is asked for anything:
|
|
1505
|
+
* no request is made, and an expired exclusion that names a segment that does not exist is refused as expired,
|
|
1506
|
+
* not as an absent operand. It runs ahead of the expired-operand shortcuts, so a combine that would read empty
|
|
1507
|
+
* anyway still refuses a lapsed exclusion. A stream already being read is not re-checked when an exclusion
|
|
1508
|
+
* expires part-way, as an operand expiring part-way does not stop it either.
|
|
1323
1509
|
*/
|
|
1324
|
-
private
|
|
1510
|
+
private expiredExcludes;
|
|
1325
1511
|
/**
|
|
1326
1512
|
* Chunk-skipping intersection: stream the ids in **this** segment AND every segment in `others`, ascending.
|
|
1327
1513
|
* Fetches only the Storage chunks present in *all* operands (a key absent from any operand contributes nothing
|
|
@@ -1376,9 +1562,9 @@ export declare class Segment {
|
|
|
1376
1562
|
* suppression list: at most one read per surviving key of `this`, so subtracting a 61,000-chunk global
|
|
1377
1563
|
* opt-out list from a 40-chunk audience costs at most 40 reads, not 61,000.
|
|
1378
1564
|
*
|
|
1379
|
-
* An expired handle in `excludes`
|
|
1380
|
-
*
|
|
1381
|
-
* the same rule.
|
|
1565
|
+
* An expired handle in `excludes` is refused, not skipped: the stream throws {@link ValidationError} naming it,
|
|
1566
|
+
* before any request is made, because a lapsed suppression list would otherwise let through the ids it was
|
|
1567
|
+
* passed to remove. An expired `this` is empty. `exclude` on `intersect` and `union` follows the same rule.
|
|
1382
1568
|
*
|
|
1383
1569
|
* To filter the *result of an intersection*, do not chain — pass `exclude` to {@link intersect} instead, so
|
|
1384
1570
|
* the suppression folds into the same pass rather than materializing an intermediate segment first.
|
|
@@ -1399,5 +1585,6 @@ export declare class Segment {
|
|
|
1399
1585
|
workload?: Workload;
|
|
1400
1586
|
}): Promise<CostReport>;
|
|
1401
1587
|
}
|
|
1402
|
-
export {
|
|
1403
|
-
export
|
|
1588
|
+
export { deserializePortable };
|
|
1589
|
+
export { MemoryStorage, LocalFsStorage, CrbmStorageChunkSource, destroySegment, eraseNamespace, excludingReservedRows, readRetentionPolicy, MIN_EXPIRES_AT_MS, LEASE_SKEW_MS, MAX_LEASE_MS, MAX_LEASES_PER_SEGMENT, InProcessKeystore, NodeAead, aadFor, CloudRoaringError, ValidationError, WriteConflictError, IntegrityError, NotFoundError, UnsupportedError, CapabilityError, TransientError, KeyUnavailableError, BudgetExceededError, LeaseExpiredError, LeaseLimitError, isCloudRoaringError, isWriteConflictError, isTransientError, isNotFoundError, isIntegrityError, isValidationError, isLeaseExpiredError, isLeaseLimitError, CrbmReader, BufferReader, CountingMetricsSink, RecordingAuditSink, DEFAULT_RETRY_POLICY, AWS_US_EAST_1_ONDEMAND, ELASTICACHE_REDIS_US_EAST_1_ONDEMAND, ONE_REDIS_HA_CLUSTER, } from '@cloudbitmaps/core';
|
|
1590
|
+
export type { Aead, AeadSealed, AuditEvent, BlobReader, BlobSink, Budget, BudgetOption, ChunkRef, Clock, CodecBitmap, CodecInterface, ConsistencyErrorEntry, ConsistencyIssue, ConsistencyReport, CostReport, CrbmCrypto, CrbmReaderOptions, CrbmStorageChunkSourceOptions, DestroyResult, ClearRegistrySummary, DropResult, EncodedChunk, EraseDeps, EstimateInput, ExportFailure, ExportFormat, ExportManifest, ExportOptions, ExportSink, ExportWriter, ExportedSegment, GenKey, GenerationEntry, GenerationMetadata, GovernanceMeta, IAuditSink, IKeystore, IMetricsSink, IRegistryDriver, IStorageDriver, IdRange, InProcessKeystoreOptions, LoadGuard, LoadInput, LoadOptions, LoadRefusal, LoadResult, LocalFsStorageOptions, MemoryStorageOptions, MetricEvent, MetricOpName, MetricsSnapshot, NewRegistryRecord, PinnedAt, PinnedObject, PortableBitmap, PricingProfile, RedisNodeType, RedisSizing, RegCaps, RegistryPatch, RegistryRecord, RegistryStatus, RegistrySummary, ReapRegistryTombstonesOptions, ReapRegistryTombstonesResult, RegistryWriteOptions, LeaseEntry, RetentionPolicy, RetireEntry, RetireExpiredOptions, RetireExpiredResult, RetryPolicy, Rng, RollbackResult, SealedRegistrySummary, SegmentInfo, SegmentRef, SegmentSize, SegmentSizing, SetRetentionResult, StorageBackend, StorageCaps, StorageChunkSource, Token, Workload, WrappedDek, } from '@cloudbitmaps/core';
|