@cloudbitmaps/core 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -8
- package/dist/chunk-7GT632UF.js +641 -0
- package/dist/chunk-7GT632UF.js.map +7 -0
- package/dist/chunk-CGILDXGP.js +73 -0
- package/dist/chunk-CGILDXGP.js.map +7 -0
- package/dist/chunk-GKHERVVQ.js +121 -0
- package/dist/chunk-GKHERVVQ.js.map +7 -0
- package/dist/{chunk-IGLIZBR7.js → chunk-MVUD4R72.js} +253 -765
- package/dist/chunk-MVUD4R72.js.map +7 -0
- package/dist/core/audit.d.ts +53 -15
- package/dist/core/bit-route.d.ts +9 -0
- package/dist/core/blob.d.ts +4 -0
- package/dist/core/budget.d.ts +8 -3
- package/dist/core/check.d.ts +27 -0
- package/dist/core/codec.d.ts +1 -1
- package/dist/core/combine-expr.d.ts +9 -5
- package/dist/core/combine-feed.d.ts +11 -3
- package/dist/core/combine-many.d.ts +28 -13
- package/dist/core/consistency.d.ts +11 -2
- package/dist/core/cooperative.d.ts +7 -0
- package/dist/core/crbm-storage-source.d.ts +7 -4
- package/dist/core/crypto.d.ts +5 -0
- package/dist/core/determinism.d.ts +2 -1
- package/dist/core/discover.d.ts +5 -3
- package/dist/core/engine.d.ts +4 -4
- package/dist/core/erase-id.d.ts +52 -19
- package/dist/core/erasure.d.ts +62 -28
- package/dist/core/errors.d.ts +83 -9
- package/dist/core/leases.d.ts +2 -0
- package/dist/core/load-input.d.ts +48 -0
- package/dist/core/load.d.ts +71 -16
- package/dist/core/metrics.d.ts +22 -1
- package/dist/core/pinned-storage-source.d.ts +27 -8
- package/dist/core/pointer-id.d.ts +9 -0
- package/dist/core/ports.d.ts +107 -13
- package/dist/core/preflight.d.ts +80 -0
- package/dist/core/registry-scan.d.ts +0 -5
- package/dist/core/retention-sweep.d.ts +61 -16
- package/dist/core/retention.d.ts +18 -7
- package/dist/core/retry.d.ts +5 -2
- package/dist/core/rollback.d.ts +23 -12
- package/dist/core/validate.d.ts +33 -2
- package/dist/driver-kit.d.ts +9 -4
- package/dist/driver-kit.js +63 -27
- package/dist/driver-kit.js.map +2 -2
- package/dist/drivers/_shared/object-registry.d.ts +24 -1
- package/dist/drivers/_shared/options.d.ts +15 -7
- package/dist/drivers/backends.d.ts +19 -2
- package/dist/drivers/crypto.d.ts +18 -3
- package/dist/export/index.d.ts +25 -10
- package/dist/index.d.ts +19 -17
- package/dist/index.js +2457 -2283
- package/dist/index.js.map +4 -4
- package/dist/node.d.ts +14 -0
- package/dist/node.js +1064 -0
- package/dist/node.js.map +7 -0
- package/package.json +16 -2
- package/dist/chunk-IGLIZBR7.js.map +0 -7
- package/dist/core/absent-operands.d.ts +0 -8
- package/dist/core/bulk-load.d.ts +0 -148
- package/dist/core/cas-retry.d.ts +0 -27
- package/dist/core/chunk-checks.d.ts +0 -39
- package/dist/core/chunk-stream.d.ts +0 -22
- package/dist/core/chunk-window.d.ts +0 -37
- package/dist/core/combine-ledger.d.ts +0 -43
- package/dist/core/crbm/crc32c.d.ts +0 -10
- package/dist/core/crbm/fingerprint.d.ts +0 -23
- package/dist/core/crbm/format.d.ts +0 -150
- package/dist/core/crbm/frame.d.ts +0 -10
- package/dist/core/crbm/plan-reads.d.ts +0 -50
- package/dist/core/crbm/varint.d.ts +0 -14
- package/dist/core/crbm/writer.d.ts +0 -72
- package/dist/core/crbm-open.d.ts +0 -154
- package/dist/core/crbm-write.d.ts +0 -75
- package/dist/core/due-index.d.ts +0 -53
- package/dist/core/generation-gc.d.ts +0 -293
- package/dist/core/generation-list.d.ts +0 -11
- package/dist/core/id-window.d.ts +0 -46
- package/dist/core/invalidation-scope.d.ts +0 -77
- package/dist/core/item-pull.d.ts +0 -29
- package/dist/core/known-generations-shape.d.ts +0 -13
- package/dist/core/known-generations.d.ts +0 -147
- package/dist/core/ledger-seed.d.ts +0 -17
- package/dist/core/metadata.d.ts +0 -42
- package/dist/core/order.d.ts +0 -2
- package/dist/core/publish.d.ts +0 -207
- package/dist/core/reader-defaults.d.ts +0 -29
- package/dist/core/summary.d.ts +0 -110
- package/dist/core/token.d.ts +0 -41
- package/dist/drivers/_shared/entropy.d.ts +0 -5
- package/dist/drivers/_shared/registry.d.ts +0 -107
- package/dist/drivers/localfs/exact-case.d.ts +0 -34
- package/dist/drivers/localfs/fs-util.d.ts +0 -55
- package/dist/drivers/localfs/paths.d.ts +0 -40
- package/dist/drivers/localfs/registry.d.ts +0 -50
- package/dist/drivers/localfs/storage.d.ts +0 -28
- package/dist/drivers/memory.d.ts +0 -64
- package/dist/testing/fuzz-core.d.ts +0 -15
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Throws {@link ValidationError} naming every one of `candidates` whose segment does not exist, asking `exists` of
|
|
3
|
-
* each, at most `limit` at once, in their order. A streamed combine asks only after operands that hold no chunk at all;
|
|
4
|
-
* the batch combine also asks after every operand it did not index (one no output names, or every operand when the range
|
|
5
|
-
* is empty). `op` begins the message (`intersect`, `materializeMany`); `nameOf` is how the caller's operands are named
|
|
6
|
-
* in it, each in double quotes. The first `exists` to reject rejects the call at once; none is asked after that.
|
|
7
|
-
*/
|
|
8
|
-
export declare function refuseAbsent<T>(op: string, candidates: readonly T[], limit: number, exists: (candidate: T) => Promise<boolean>, nameOf: (candidate: T) => string): Promise<void>;
|
package/dist/core/bulk-load.d.ts
DELETED
|
@@ -1,148 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The loader `load()` is built on: ids, a decoded bitmap or a combine's chunks in, one generation written, and published
|
|
3
|
-
* when a registry is wired and the caller does not defer it, as `load()` does to publish after its guard.
|
|
4
|
-
*/
|
|
5
|
-
import { type IAuditSink } from './audit.js';
|
|
6
|
-
import { ValidationError } from './errors.js';
|
|
7
|
-
import type { Clock } from './determinism.js';
|
|
8
|
-
import type { Aead, IKeystore, WrappedDek } from './crypto.js';
|
|
9
|
-
import type { GenerationMetadata, GenKey, IStorageDriver, IRegistryDriver, RegistryRecord, RegistrySummary, SegmentRef } from './ports.js';
|
|
10
|
-
import type { CodecInterface } from './codec.js';
|
|
11
|
-
import { ChunkLoadInput, DecodedLoadInput } from './load-input.js';
|
|
12
|
-
/** What a bulk-load wrote — the driver's `{ size, sha256 }` plus a summary of the built generation. */
|
|
13
|
-
export interface BulkLoadResult {
|
|
14
|
-
/** Bytes written to the Storage object. */
|
|
15
|
-
readonly size: number;
|
|
16
|
-
/** The driver's content hash of the object. */
|
|
17
|
-
readonly sha256: string;
|
|
18
|
-
/** Distinct non-empty chunks written (≤ 65536). */
|
|
19
|
-
readonly chunkCount: number;
|
|
20
|
-
/**
|
|
21
|
-
* The DEK freshly minted for this generation, wrapped — present only when the segment is encrypted AND this
|
|
22
|
-
* call minted one (an existing segment reuses its key, which is already on the row).
|
|
23
|
-
*
|
|
24
|
-
* Returned so a caller that deliberately does **not** pass `registry` here — writing now and publishing later,
|
|
25
|
-
* which is how a load applies a guard while the old generation is still authoritative — can carry the key onto
|
|
26
|
-
* its own `publishGeneration`. Without it that publish stores no DEK and the generation it makes current is
|
|
27
|
-
* unreadable: the bytes are encrypted with a key nothing recorded.
|
|
28
|
-
*/
|
|
29
|
-
readonly wrappedDeks?: readonly WrappedDek[];
|
|
30
|
-
/** Whether the object was written encrypted: under the segment's key, or under one this call minted. */
|
|
31
|
-
readonly encrypted: boolean;
|
|
32
|
-
/** The written object's fingerprint, which tells it from an object stored later under the same key. */
|
|
33
|
-
readonly fingerprint: string;
|
|
34
|
-
/** Total distinct ids in the generation (post-dedup). */
|
|
35
|
-
readonly cardinality: number;
|
|
36
|
-
/**
|
|
37
|
-
* The summary of what was written, for the write that makes it current: its id count and metadata, sealed under the
|
|
38
|
-
* segment's key when the object was written with one. A caller that defers the publish (`publish: false`) passes it
|
|
39
|
-
* to {@link publishGeneration}.
|
|
40
|
-
*/
|
|
41
|
-
readonly summary: RegistrySummary;
|
|
42
|
-
/**
|
|
43
|
-
* Whether this generation is now the segment's **current** one — `undefined` when no `registry` was wired, so
|
|
44
|
-
* there is no pointer and no publish step to report on.
|
|
45
|
-
*
|
|
46
|
-
* `false` means the object is durable but a **concurrent writer published a higher generation first**, so this
|
|
47
|
-
* one is an orphan that no reader will ever resolve. The write succeeded and the *load* did not: whatever this
|
|
48
|
-
* call was asked to make the segment contain, the segment does not contain. That is not a detail a caller can
|
|
49
|
-
* be left to infer — a `*Into` materialisation that reported the generation it wrote as "the destination's new
|
|
50
|
-
* current generation" on this path would state a plain untruth — so the flag is on the result and the verbs that
|
|
51
|
-
* promise a published generation check it.
|
|
52
|
-
*/
|
|
53
|
-
readonly becameCurrent?: boolean;
|
|
54
|
-
}
|
|
55
|
-
/**
|
|
56
|
-
* **Bulk-load** a whole immutable generation from a flat id source — the batch "seed/rebuild" entry point.
|
|
57
|
-
* Unlike {@link writeCrbmGeneration} (which takes pre-grouped bitmaps), this
|
|
58
|
-
* consumes an arbitrary, **unsorted** stream of u32 ids (sync or async) and routes each into its chunk's
|
|
59
|
-
* Roaring bitmap as it goes. The input is consumed lazily and ids dedup on insert, so memory is bounded by
|
|
60
|
-
* the **built generation** — the in-memory Roaring representation of the *distinct* set (≤ one bitmap per
|
|
61
|
-
* non-empty chunk) — not by the input length: you can stream a billion duplicate-heavy ids and hold only the
|
|
62
|
-
* distinct result. Note this is `O(distinct set)`, **not** window-bounded like {@link SegmentEngine.intersect}:
|
|
63
|
-
* bulk-load holds the whole generation in RAM, which suits batch seed/rebuild jobs but not an
|
|
64
|
-
* unbounded-cardinality stream (a segment larger than RAM needs external-merge / pre-sorted input — later phase).
|
|
65
|
-
*
|
|
66
|
-
* Each id must be an integer in `[0, 2³²)` ({@link splitId} throws {@link ValidationError} otherwise) — the
|
|
67
|
-
* source is consumed lazily, so a bad id aborts mid-stream without writing a partial object (the driver
|
|
68
|
-
* commits only after the callback resolves). An empty source writes a valid empty generation.
|
|
69
|
-
*
|
|
70
|
-
* Writes a fresh full snapshot of a segment. The caller picks the generation number in `key` — `nextGeneration`
|
|
71
|
-
* computes one from the registry and the bucket, as `nextLoadGeneration` does for a load — and re-using an existing
|
|
72
|
-
* generation throws
|
|
73
|
-
* {@link WriteConflictError} (write-once). Without a registry a `StorageChunkSource` serves the **highest**
|
|
74
|
-
* generation present, so a too-high number silently shadows real data; with one, `publishGeneration` decides.
|
|
75
|
-
*/
|
|
76
|
-
export declare function bulkLoadCrbmGeneration(driver: IStorageDriver, key: GenKey, ids: Iterable<number> | AsyncIterable<number> | DecodedLoadInput | ChunkLoadInput, options?: {
|
|
77
|
-
registry?: IRegistryDriver;
|
|
78
|
-
keystore?: IKeystore;
|
|
79
|
-
requireEncryption?: boolean;
|
|
80
|
-
audit?: IAuditSink;
|
|
81
|
-
/** Bitmap codec. Optional in the type; a **flavor** package binds it ({@link requireCodec}). */
|
|
82
|
-
codec?: CodecInterface;
|
|
83
|
-
/**
|
|
84
|
-
* Injected clock. Supplying one makes bulk-load **cooperative**: it yields the event loop periodically so a
|
|
85
|
-
* long load does not stall everything else on the process. Without it the load still completes, just
|
|
86
|
-
* without yielding — which is the pre-existing behaviour, kept so this is purely additive.
|
|
87
|
-
*
|
|
88
|
-
* `@cloudbitmaps/roaring` supplies a real clock by default, so flavor users get cooperative behaviour with
|
|
89
|
-
* no wiring. `core/` cannot default it: it is timer-free by lint, which is exactly why waiting goes through
|
|
90
|
-
* this seam rather than `setTimeout`.
|
|
91
|
-
*/
|
|
92
|
-
clock?: Clock;
|
|
93
|
-
/**
|
|
94
|
-
* Write the object but do **not** advance the pointer (default: publish when a `registry` is wired).
|
|
95
|
-
*
|
|
96
|
-
* For a caller that has to inspect what it wrote before deciding whether it should become current — a
|
|
97
|
-
* guarded {@link loadSegment} is the one in-tree case — because the check is only meaningful while the old
|
|
98
|
-
* generation is still authoritative. The `registry` is still required for an encrypted segment and still
|
|
99
|
-
* consulted: it is where an existing segment's DEK lives, and reusing that key is not optional. The wrapped
|
|
100
|
-
* DEK comes back on the result so the deferred publish can store it.
|
|
101
|
-
*
|
|
102
|
-
* A caller that defers the publish owns what it wrote: an unpublished object sits ABOVE `currentGen`, where
|
|
103
|
-
* generation collection deliberately never looks, so nothing reclaims it until a later generation above it is
|
|
104
|
-
* current.
|
|
105
|
-
*/
|
|
106
|
-
publish?: boolean;
|
|
107
|
-
/**
|
|
108
|
-
* The generation's metadata (see {@link GenerationMetadata}), written into the object and carried in the summary.
|
|
109
|
-
* The caller has checked it against the metadata rules; the writer checks it again, and refuses a record that breaks
|
|
110
|
-
* one before a byte is written.
|
|
111
|
-
*/
|
|
112
|
-
metadata?: GenerationMetadata;
|
|
113
|
-
/**
|
|
114
|
-
* The segment's row as the caller already read it (`null`: it found none), for a caller that defers the
|
|
115
|
-
* publish (`publish: false`) and fences it on that same row. A present cleartext row is then used as read,
|
|
116
|
-
* rather than read again: a row that changes in between (a publish, a drop, a purge and re-create) makes the
|
|
117
|
-
* fenced publish lose, so the object is never published on the strength of the stale read. A row read as
|
|
118
|
-
* absent, or one carrying key material, is read again after the ids, as with no row passed: a first load must
|
|
119
|
-
* see a row another writer created meanwhile, and an encrypted segment's key is unwrapped only from a row read
|
|
120
|
-
* after the ids, so a segment shredded while they streamed is refused before its key is used.
|
|
121
|
-
*/
|
|
122
|
-
row?: RegistryRecord | null;
|
|
123
|
-
}): Promise<BulkLoadResult>;
|
|
124
|
-
/**
|
|
125
|
-
* What a load has already started when it calls {@link bulkLoadAhead}: its number, and the segment's data key, each
|
|
126
|
-
* still in flight so their round trips overlap the encoding. Both are joined where the write needs them, and neither
|
|
127
|
-
* may be left to reject unobserved.
|
|
128
|
-
*/
|
|
129
|
-
interface LoadAhead {
|
|
130
|
-
/** The number the generation takes (see {@link nextLoadGeneration}); joined just before the object is written. */
|
|
131
|
-
readonly generation: PromiseLike<number>;
|
|
132
|
-
/**
|
|
133
|
-
* The key a caller already asked the keystore to unwrap, with the wrappings it asked about. Used only when the
|
|
134
|
-
* row the write reads after the ids carries exactly these wrappings; otherwise the write unwraps for itself.
|
|
135
|
-
*/
|
|
136
|
-
readonly unwrapped?: {
|
|
137
|
-
readonly wrapped: readonly WrappedDek[];
|
|
138
|
-
readonly aead: PromiseLike<Aead>;
|
|
139
|
-
};
|
|
140
|
-
}
|
|
141
|
-
/** What a load with `requireEncryption` meets on a segment that already has a cleartext generation. */
|
|
142
|
-
export declare function cleartextUnderRequiredEncryption(ref: SegmentRef, currentGen: number): ValidationError;
|
|
143
|
-
/**
|
|
144
|
-
* {@link bulkLoadCrbmGeneration} for a load whose number, and perhaps its key, are still being fetched: the same
|
|
145
|
-
* options, with the same meaning, and the same writes. The number is joined only where the object is written.
|
|
146
|
-
*/
|
|
147
|
-
export declare function bulkLoadAhead(driver: IStorageDriver, ref: SegmentRef, ahead: LoadAhead, ids: Parameters<typeof bulkLoadCrbmGeneration>[2], options?: NonNullable<Parameters<typeof bulkLoadCrbmGeneration>[3]>): Promise<BulkLoadResult>;
|
|
148
|
-
export {};
|
package/dist/core/cas-retry.d.ts
DELETED
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
import type { WriteConflictError } from './errors.js';
|
|
2
|
-
import { type ChurnDeps } from './leases.js';
|
|
3
|
-
import type { IRegistryDriver, RegistryRecord, SegmentRef } from './ports.js';
|
|
4
|
-
/** What `casRetry` needs: the registry, and the clock and rng that spread a wait out. */
|
|
5
|
-
type CasRetryDeps = ChurnDeps & {
|
|
6
|
-
readonly registry: IRegistryDriver;
|
|
7
|
-
};
|
|
8
|
-
/** How many attempts a caller gets, and the error it throws once they are spent. */
|
|
9
|
-
interface CasRetryPlan {
|
|
10
|
-
/** Attempts that count; a wait out of lease churn does not use one. */
|
|
11
|
-
readonly attempts: number;
|
|
12
|
-
/** The error thrown when every attempt lost. */
|
|
13
|
-
readonly contention: () => WriteConflictError;
|
|
14
|
-
}
|
|
15
|
-
/**
|
|
16
|
-
* Read a segment's row, run `step` on it, and run it again on a fresh read when `step` throws a write conflict.
|
|
17
|
-
*
|
|
18
|
-
* Every lost race uses one of `plan.attempts`, except one lost by a run that followed a wait. When the row read after a
|
|
19
|
-
* lost race differs from the one `step` was given only in its leases, the loop waits the churn out (up to the churn
|
|
20
|
-
* bound), reads the row again and runs `step` against that without using an attempt. A race lost on the last attempt
|
|
21
|
-
* is not waited out. `step` returns the caller's result, or throws: a write conflict retries, anything else leaves
|
|
22
|
-
* `casRetry` untouched.
|
|
23
|
-
*
|
|
24
|
-
* Exhausting `attempts` throws `plan.contention()`.
|
|
25
|
-
*/
|
|
26
|
-
export declare function casRetry<T>(deps: CasRetryDeps, ref: SegmentRef, plan: CasRetryPlan, step: (row: RegistryRecord | null) => Promise<T>): Promise<T>;
|
|
27
|
-
export {};
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
import type { CodecBitmap, CodecInterface } from './codec.js';
|
|
2
|
-
/**
|
|
3
|
-
* Tier-derived keys are untrusted (invariant 5) — fail fast on a corrupt/out-of-range key rather than
|
|
4
|
-
* letting it flow into the id-routing step and produce a bogus id.
|
|
5
|
-
*/
|
|
6
|
-
export declare function assertChunkKeyInRange(k: number): void;
|
|
7
|
-
/** A chunk holds at least one id and at most {@link CHUNK_COUNT}; a count outside that came from a corrupt tier. */
|
|
8
|
-
export declare function assertChunkCardinalityInRange(c: number): void;
|
|
9
|
-
/**
|
|
10
|
-
* A segment's chunk keys, ascending — a shape read off the index, no payload. Keys are untrusted (invariant 5): each is
|
|
11
|
-
* an integer in `[0, CHUNK_COUNT)` and none repeats. One pass range-checks every key and notes whether the list is
|
|
12
|
-
* already strictly ascending, which an index read off a `.crbm` always is; only a list that is not is sorted and scanned
|
|
13
|
-
* for a repeat. The caller gets a list of its own.
|
|
14
|
-
*/
|
|
15
|
-
export declare function checkedChunkKeys(listed: Iterable<number>): number[];
|
|
16
|
-
/**
|
|
17
|
-
* The other half of invariant 5: a chunk payload holds **remainders** — 16-bit offsets within one chunk — so
|
|
18
|
-
* every value must be `<= 0xffff`.
|
|
19
|
-
*
|
|
20
|
-
* Nothing upstream establishes that. The byte cap bounds *size*, and CRC/AEAD prove the bytes are the bytes
|
|
21
|
-
* that were written — which anyone able to write the bucket satisfies trivially. A value `>= 65536` would then
|
|
22
|
-
* reach the id-routing step, which masks it (`remainder & 0xffff`) and emits a **fabricated id belonging to a different
|
|
23
|
-
* chunk's id space**: indistinguishable from real data, inflating `count()` and creating spurious `intersect`
|
|
24
|
-
* matches.
|
|
25
|
-
*
|
|
26
|
-
* Costs one `maximum()` per chunk, not per id — `maximum` is optional on the codec seam precisely so a codec
|
|
27
|
-
* that cannot answer in O(1) opts out instead of making the read path walk every value.
|
|
28
|
-
*
|
|
29
|
-
* One value stands for all of them only because the decode is structurally checked. Roaring answers `maximum()`
|
|
30
|
-
* from its last container, so a payload listing its containers out of order would report the wrong one's
|
|
31
|
-
* largest value and pass here while holding values above 65,535. The codec's `safeDeserialize` refuses that
|
|
32
|
-
* payload, and every other shape that would make this answer wrong, before it reaches this check.
|
|
33
|
-
*/
|
|
34
|
-
export declare function assertChunkPayloadInRange(bitmap: CodecBitmap, chunkKey: number): void;
|
|
35
|
-
/**
|
|
36
|
-
* A chunk's stored bytes decoded under the size cap (invariant 5) and range-checked: the one place a chunk becomes a
|
|
37
|
-
* bitmap, whether its bytes came from a request of its own or from a stream of coalesced ranges.
|
|
38
|
-
*/
|
|
39
|
-
export declare function decodeChunkBytes(codec: CodecInterface, bytes: Uint8Array, chunkKey: number, maxBitmapBytes: number): CodecBitmap;
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import type { ChunkRead } from './ports.js';
|
|
2
|
-
/**
|
|
3
|
-
* One operand's chunks as the source streams them, taken a key at a time. The engine asks for keys in ascending order
|
|
4
|
-
* and the stream yields them in that order, so each `take` is the next item; `take` calls the iterator at once, before
|
|
5
|
-
* anything is awaited, which is what keeps concurrent takes lined up with the keys they were made for (an async
|
|
6
|
-
* generator answers overlapping `next()` calls in the order they were made).
|
|
7
|
-
*
|
|
8
|
-
* A stream that fails fails every take after the one that met it with the same error, and a stream that ends before the
|
|
9
|
-
* key asked for, or answers another key, is an {@link IntegrityError}: a source that misaligns its answers must not
|
|
10
|
-
* hand one chunk's bytes to another key.
|
|
11
|
-
*
|
|
12
|
-
* Internal to core: not exported from any entry point.
|
|
13
|
-
*/
|
|
14
|
-
export declare class ChunkStream {
|
|
15
|
-
private readonly iterator;
|
|
16
|
-
private failure;
|
|
17
|
-
constructor(stream: AsyncIterable<ChunkRead>);
|
|
18
|
-
/** The chunk for `key`, which must be the next key the stream was opened over. */
|
|
19
|
-
take(key: number): Promise<ChunkRead>;
|
|
20
|
-
/** Stop the stream: no further range is started, and ranges already in flight finish and are dropped. */
|
|
21
|
-
close(): void;
|
|
22
|
-
}
|
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* An ordered window of reads: up to `max` reads open ahead of the one being consumed, handed back in key order.
|
|
3
|
-
* With `ramp`, the window opens 1, 2, 4 … wide (or at `rampStart`, then doubling) up to `max` instead of `max` at once, so a consumer that stops
|
|
4
|
-
* after a few chunks has fetched a handful, not a full window.
|
|
5
|
-
*
|
|
6
|
-
* `T` is whatever one read yields (a decoded chunk, or the raw bytes of one), and `K` what names a read to launch: a
|
|
7
|
-
* chunk key, or a planned range. The reads to launch are a list, or an iterator pulled from only as the window opens
|
|
8
|
-
* a read, so a long plan is never made ahead of what the window can hold. Each read is wrapped to resolve and
|
|
9
|
-
* never reject, so a read nobody consumes (the consumer stopped, or an earlier chunk failed) cannot raise an
|
|
10
|
-
* unhandled rejection; its error surfaces from {@link take} only if the consumer reaches that chunk. Memory is
|
|
11
|
-
* bounded by the window: at most `max` results are held ahead.
|
|
12
|
-
*
|
|
13
|
-
* Internal to core: not exported from any entry point.
|
|
14
|
-
*/
|
|
15
|
-
export declare class ChunkWindow<T, K = number> {
|
|
16
|
-
private readonly fetch;
|
|
17
|
-
private readonly max;
|
|
18
|
-
private readonly ramp;
|
|
19
|
-
/** With `ramp`, the width the window opens at (default 1). */
|
|
20
|
-
private readonly rampStart;
|
|
21
|
-
private readonly open;
|
|
22
|
-
private launched;
|
|
23
|
-
private taken;
|
|
24
|
-
private readonly list;
|
|
25
|
-
private readonly source;
|
|
26
|
-
/** What the source threw, so a later {@link take} throws it again rather than reading the spent source as done. */
|
|
27
|
-
private failure;
|
|
28
|
-
constructor(keys: readonly K[] | Iterator<K>, fetch: (key: K) => Promise<T>, max: number, ramp: boolean,
|
|
29
|
-
/** With `ramp`, the width the window opens at (default 1). */
|
|
30
|
-
rampStart?: number);
|
|
31
|
-
/** The next read to launch, or {@link DONE} when there is none. */
|
|
32
|
-
private nextKey;
|
|
33
|
-
/** Resolves once every read launched and not yet taken has settled; it never rejects, as the reads never do. */
|
|
34
|
-
settle(): Promise<void>;
|
|
35
|
-
/** The next read's result in key order, or the error that read raised. */
|
|
36
|
-
take(): Promise<T>;
|
|
37
|
-
}
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The resident-bytes ledger of a batch combine. Pure: no I/O, time or randomness.
|
|
3
|
-
*
|
|
4
|
-
* It counts what is **resident**, not what a bitmap serializes to: a native bitmap costs a fixed amount over its
|
|
5
|
-
* serialized bytes, so a million small chunks hold far more than their serialized size says. Measured on the shipped
|
|
6
|
-
* codec with one bitmap per chunk, the cost over serialized size was about 440 bytes for a chunk of up to 50 ids,
|
|
7
|
-
* rising to about 1,700 bytes around 3,000 ids (an array container's spare capacity) and falling to about 300 bytes for
|
|
8
|
-
* a bitset. {@link residentBytes} is a model that stays above every measured point.
|
|
9
|
-
*/
|
|
10
|
-
/**
|
|
11
|
-
* The fixed resident cost of one native bitmap beyond its serialized bytes. Measured on the shipped codec, over 50,000
|
|
12
|
-
* bitmaps of one chunk each, it was 430 to 450 bytes for a chunk of up to 50 ids on macOS (arm64) and 357 to 373 on Linux
|
|
13
|
-
* (arm64, glibc 2.36), and 279 to 433 for a bitset; the constant sits above the highest of those.
|
|
14
|
-
*/
|
|
15
|
-
export declare const RESIDENT_BITMAP_OVERHEAD = 512;
|
|
16
|
-
/** An upper bound on the serialized size of a chunk of `cardinality` ids: an array container, capped at a bitset. */
|
|
17
|
-
export declare function serializedBound(cardinality: number): number;
|
|
18
|
-
/** The resident bytes of one bitmap whose serialized size is `serialized`: above every measured point. */
|
|
19
|
-
export declare function residentBytes(serialized: number): number;
|
|
20
|
-
/** The resident bytes of one chunk of at most `cardinality` ids. */
|
|
21
|
-
export declare function residentBound(cardinality: number): number;
|
|
22
|
-
/**
|
|
23
|
-
* A byte budget with a high-water mark. Everything the pass keeps resident is charged to it and released when it is
|
|
24
|
-
* let go, so `used` is the pass's own count of what it holds, and `maxBufferedBytes` is a bound on it.
|
|
25
|
-
*/
|
|
26
|
-
export declare class ResidentLedger {
|
|
27
|
-
readonly limit: number;
|
|
28
|
-
private held;
|
|
29
|
-
private peak;
|
|
30
|
-
constructor(limit: number);
|
|
31
|
-
get used(): number;
|
|
32
|
-
get highWater(): number;
|
|
33
|
-
/** Bytes still free. */
|
|
34
|
-
get room(): number;
|
|
35
|
-
/** Charge `bytes` if they fit. */
|
|
36
|
-
tryCharge(bytes: number): boolean;
|
|
37
|
-
/**
|
|
38
|
-
* Charge `bytes`, asking `makeRoom` to free some while they do not fit. `makeRoom` releases what it frees (through
|
|
39
|
-
* {@link release}) and answers whether it freed anything; the charge is refused when it cannot.
|
|
40
|
-
*/
|
|
41
|
-
reserve(bytes: number, makeRoom: () => boolean): boolean;
|
|
42
|
-
release(bytes: number): void;
|
|
43
|
-
}
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* CRC32C (Castagnoli, polynomial 0x1EDC6F41) — the pre-deserialize integrity screen for `.crbm`
|
|
3
|
-
* payloads, index, and footer.
|
|
4
|
-
*
|
|
5
|
-
* Pure TypeScript, table-driven: `core/` may not import `crypto`/`node:crypto` (the determinism lint
|
|
6
|
-
* enforces it), and a from-scratch implementation is what the cross-language golden corpus will be pinned
|
|
7
|
-
* against. CRC32C (not CRC32) is the locked choice — it matches the hardware-accelerated checksum used
|
|
8
|
-
* by SSE/iSCSI/Btrfs, and it is the algorithm the `.crbm` format locks in.
|
|
9
|
-
*/
|
|
10
|
-
export declare function crc32c(bytes: Uint8Array, seed?: number): number;
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
/** What names an object: its size, then its footer's CRC. One spelling, for an open reader and a footer read alone. */
|
|
2
|
-
export declare const sizePart: (size: number) => string;
|
|
3
|
-
/** An object's fingerprint, from its size and its footer's CRC (see `CrbmReader.fingerprint`). */
|
|
4
|
-
export declare const fingerprintFor: (size: number, footerCrc: number) => string;
|
|
5
|
-
/**
|
|
6
|
-
* A fingerprint taken apart, or `undefined` for text that is not one an object can have: a size that is a safe integer
|
|
7
|
-
* no smaller than a `.crbm` preamble and footer, and a footer CRC that fits in 32 bits. For a row's summary, which
|
|
8
|
-
* names its object by it and reports the size from it.
|
|
9
|
-
*/
|
|
10
|
-
export declare function fingerprintParts(fingerprint: unknown): {
|
|
11
|
-
readonly size: number;
|
|
12
|
-
readonly footerCrc: number;
|
|
13
|
-
} | undefined;
|
|
14
|
-
/**
|
|
15
|
-
* The footer CRC of a fingerprint a writer or a reader produced, which a row's ledger records beside the generation.
|
|
16
|
-
* Throws {@link IntegrityError} for text that is not a fingerprint, which neither produces.
|
|
17
|
-
*/
|
|
18
|
-
export declare function footerCrcOf(fingerprint: string): number;
|
|
19
|
-
/**
|
|
20
|
-
* A fingerprint spelled short, for a key a read builds on every chunk: its size and its footer's CRC in base 36,
|
|
21
|
-
* `<size>.<crc>`. Two compact spellings are equal exactly when the fingerprints are.
|
|
22
|
-
*/
|
|
23
|
-
export declare function compactFingerprint(fingerprint: string): string;
|
|
@@ -1,150 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Frozen `.crbm` v1 layout constants (format 1.0, with its optional extension block).
|
|
3
|
-
*
|
|
4
|
-
* These byte widths/offsets are pinned by the golden corpus and must never change for v1 —
|
|
5
|
-
* a new layout is a new format version. All multi-byte integers are little-endian (v1 fixes LE).
|
|
6
|
-
*/
|
|
7
|
-
import { FOOTER_BYTES, PREAMBLE_BYTES } from './frame.js';
|
|
8
|
-
/** 4-byte magic at both file ends: "CRBM". */
|
|
9
|
-
export declare const MAGIC: Uint8Array<ArrayBuffer>;
|
|
10
|
-
export declare const VERSION_MAJOR = 1;
|
|
11
|
-
export declare const VERSION_MINOR = 0;
|
|
12
|
-
export { FOOTER_BYTES, PREAMBLE_BYTES };
|
|
13
|
-
/** Payloads begin immediately after the preamble. */
|
|
14
|
-
export declare const PAYLOAD_START = 8;
|
|
15
|
-
/** Byte offsets of each field within the 104-byte footer. */
|
|
16
|
-
export declare const FOOTER: {
|
|
17
|
-
readonly indexOffset: 0;
|
|
18
|
-
readonly indexLength: 8;
|
|
19
|
-
readonly indexCrc32c: 16;
|
|
20
|
-
readonly flags: 20;
|
|
21
|
-
readonly payloadCodecId: 24;
|
|
22
|
-
readonly elementWidth: 26;
|
|
23
|
-
readonly containerCodec: 27;
|
|
24
|
-
readonly versionMajor: 28;
|
|
25
|
-
readonly versionMinor: 29;
|
|
26
|
-
readonly reserved2: 30;
|
|
27
|
-
readonly generation: 32;
|
|
28
|
-
readonly indexNonce: 40;
|
|
29
|
-
readonly indexTag: 52;
|
|
30
|
-
readonly keyId: 68;
|
|
31
|
-
readonly chunkCount: 84;
|
|
32
|
-
readonly totalCardinality: 88;
|
|
33
|
-
readonly footerCrc32c: 96;
|
|
34
|
-
readonly endMagic: 100;
|
|
35
|
-
};
|
|
36
|
-
/** Footer bytes covered by `footer_crc32c` (everything before the CRC field). */
|
|
37
|
-
export declare const FOOTER_CRC_COVERAGE: 96;
|
|
38
|
-
/** `flags` bit positions. */
|
|
39
|
-
export declare const FLAG_ENCRYPTED: number;
|
|
40
|
-
export declare const FLAG_LITTLE_ENDIAN: number;
|
|
41
|
-
/**
|
|
42
|
-
* The object carries an extension block just before its index (a generation's metadata). Set only when there is
|
|
43
|
-
* something to put in it, so an object without metadata is the same bytes as before the block existed; a reader that
|
|
44
|
-
* does not know the bit refuses the object rather than reading past what it cannot see.
|
|
45
|
-
*/
|
|
46
|
-
export declare const FLAG_EXTENSION: number;
|
|
47
|
-
/** Flag bits a v1 reader understands; any bit outside this mask is an unsupported feature. */
|
|
48
|
-
export declare const KNOWN_FLAGS: number;
|
|
49
|
-
/**
|
|
50
|
-
* Default element width: 32-bit ids (the u32 member space). `64` is the *reserved* escape above the
|
|
51
|
-
* ~4.29 B-ids/segment u32 ceiling; a v1 (32-bit) reader must **reject** a 64-bit generation rather than feed
|
|
52
|
-
* it to the 32-bit deserializer (which would mis-count fleet-wide). A future 64-bit generation is therefore a
|
|
53
|
-
* **major**-version bump (auto-rejected by old readers), never a silent minor one — see
|
|
54
|
-
* {@link CrbmReader.open} validates this.
|
|
55
|
-
*/
|
|
56
|
-
export declare const ELEMENT_WIDTH_32 = 32;
|
|
57
|
-
/**
|
|
58
|
-
* **Which codec produced the chunk payloads.** Recorded in the footer, validated on read.
|
|
59
|
-
*
|
|
60
|
-
* `.crbm` is a shared container: the index, the CRC32Cs, the AEAD framing and the generation model are all
|
|
61
|
-
* codec-independent, and **only the chunk payload bytes belong to a flavor** (hence *Chunked Remote BitMap* —
|
|
62
|
-
* see the `.crbm` format section of the API reference). This field is what lets one container hold either, the same way ZIP tags each member
|
|
63
|
-
* with a compression method.
|
|
64
|
-
*
|
|
65
|
-
* **A codec id, not a roaring-specific one.** A second codec is genuinely expected — `soaring` is a planned
|
|
66
|
-
* Roaring *variant*, so its serialized bytes are unlikely to be roaring-portable, and it lands *after* the `1.0`
|
|
67
|
-
* format freeze. A field frozen under a codec-specific meaning could not be reinterpreted later without a major
|
|
68
|
-
* format version.
|
|
69
|
-
*
|
|
70
|
-
* **Ids are permanent once published.** Add to {@link KNOWN_PAYLOAD_CODEC_IDS} when a codec ships; never
|
|
71
|
-
* reuse or renumber. Ids are deliberately *not* pre-allocated for codecs that do not exist — a reserved number
|
|
72
|
-
* for an unbuilt codec is a guess about a format nobody has designed.
|
|
73
|
-
*/
|
|
74
|
-
export declare const PAYLOAD_CODEC_ROARING_PORTABLE = 1;
|
|
75
|
-
/**
|
|
76
|
-
* Every payload codec id this reader can decode.
|
|
77
|
-
*
|
|
78
|
-
* The reader validates membership rather than equality with a single constant. That is the whole point of an
|
|
79
|
-
* id registry: an unknown id is rejected with a typed error naming it, so an old reader meeting a
|
|
80
|
-
* future-codec generation **fails closed** — the correct direction, and the reason a store built on one codec
|
|
81
|
-
* can never silently misread another's bytes as its own. (The homogeneity contract means one store is one
|
|
82
|
-
* codec, so meeting a foreign generation implies misconfiguration, and a loud rejection is exactly what you
|
|
83
|
-
* want there.)
|
|
84
|
-
*/
|
|
85
|
-
export declare const KNOWN_PAYLOAD_CODEC_IDS: ReadonlySet<number>;
|
|
86
|
-
/**
|
|
87
|
-
* The only container codec v1 defines: `0` = none (payloads are stored as the portable roaring serialization
|
|
88
|
-
* with no extra container-level transform). A non-zero codec is a future format feature; a v1 reader rejects
|
|
89
|
-
* it ({@link UnsupportedError}) rather than mis-decode. Stamped by the writer, validated by the reader.
|
|
90
|
-
*/
|
|
91
|
-
export declare const CONTAINER_CODEC_NONE = 0;
|
|
92
|
-
/**
|
|
93
|
-
* Default speculative-tail-read size. The spec's worst-case index (a full 16-bit span) is ~400–650 KB,
|
|
94
|
-
* so 256 KB collapses the read to a single GET for the vast majority of real (sparse/medium) segments;
|
|
95
|
-
* only near-full-span segments take the documented second GET.
|
|
96
|
-
*/
|
|
97
|
-
export declare const DEFAULT_TAIL_BYTES: number;
|
|
98
|
-
/**
|
|
99
|
-
* Hard cap on the index region a reader will fetch/parse from one (untrusted) object. The index is read
|
|
100
|
-
* whole on the cheap `open()`/`count()` path, so it must be bounded before allocation — generously
|
|
101
|
-
* above the ~650 KB full-span worst case, well below a denial-of-wallet allocation.
|
|
102
|
-
*/
|
|
103
|
-
export declare const DEFAULT_MAX_INDEX_BYTES: number;
|
|
104
|
-
/**
|
|
105
|
-
* Largest serialized bitmap the engine decodes (1 MiB, per chunk). A well-formed 16-bit chunk serializes to at most
|
|
106
|
-
* about 8.2 KiB, so no legitimate chunk is near it; both the read engine and the erase path decode under this one cap.
|
|
107
|
-
*/
|
|
108
|
-
export declare const DEFAULT_MAX_BITMAP_BYTES: number;
|
|
109
|
-
/** A chunk's cardinality is in `[1, 65536]` (empty chunks are never written). */
|
|
110
|
-
export declare const MAX_CHUNK_CARDINALITY = 65536;
|
|
111
|
-
/**
|
|
112
|
-
* The extension block, present when the footer sets {@link FLAG_EXTENSION}: `section* ‖ u32 sectionsLength ‖ u32
|
|
113
|
-
* crc32c ‖ "CRBX"`, between the last payload and the index, so the index starts right after it. A reader finds it from
|
|
114
|
-
* `indexOffset`: the trailer is the {@link EXT_TRAILER_BYTES} bytes before the index, and the CRC32C covers the sections
|
|
115
|
-
* and the length field. Each section is `u8 type ‖ u32 length ‖ bytes`, in strictly ascending type order; a reader
|
|
116
|
-
* skips a type it does not know, so a section must be safe to ignore.
|
|
117
|
-
*/
|
|
118
|
-
export declare const EXT_MAGIC: Uint8Array<ArrayBuffer>;
|
|
119
|
-
/** The trailer: the sections' length (u32), their CRC32C (u32) and {@link EXT_MAGIC}. */
|
|
120
|
-
export declare const EXT_TRAILER_BYTES = 12;
|
|
121
|
-
/** A section's header: its type (u8) and its length (u32). */
|
|
122
|
-
export declare const EXT_SECTION_HEADER_BYTES = 5;
|
|
123
|
-
/**
|
|
124
|
-
* Cap on the sections of one block, in bytes. A reader refuses a larger block from its trailer alone, before it reads
|
|
125
|
-
* any sections beyond what it already holds (the read that brings the index may already hold up to this much before
|
|
126
|
-
* it), and a writer of any 1.x minor keeps within it, so a later section type stays readable by this reader.
|
|
127
|
-
*/
|
|
128
|
-
export declare const MAX_EXT_BYTES: number;
|
|
129
|
-
/**
|
|
130
|
-
* Section type 1: the generation's metadata, its canonical JSON in UTF-8. On an encrypted object the section is
|
|
131
|
-
* `nonce ‖ ciphertext ‖ tag`, sealed under the segment's key with the metadata scope of the associated data.
|
|
132
|
-
* Type 0 is not a section type: a reader refuses it.
|
|
133
|
-
*/
|
|
134
|
-
export declare const EXT_SECTION_METADATA = 1;
|
|
135
|
-
/** Fixed width of a per-chunk CRC32C field in the index (high-entropy → not varint). */
|
|
136
|
-
export declare const CRC32C_BYTES = 4;
|
|
137
|
-
/**
|
|
138
|
-
* v1 AEAD framing sizes (AES-256-GCM), fixed by the format exactly like the payload codec id — a
|
|
139
|
-
* different cipher is a new format version. An encrypted chunk payload is stored as `nonce ‖ ciphertext ‖ tag`;
|
|
140
|
-
* the encrypted **index**'s nonce/tag live in the footer's reserved `indexNonce`/`indexTag` slots. Matches
|
|
141
|
-
* {@link FOOTER.indexNonce} (12 B) and {@link FOOTER.indexTag} (16 B).
|
|
142
|
-
*/
|
|
143
|
-
export declare const AEAD_NONCE_BYTES = 12;
|
|
144
|
-
export declare const AEAD_TAG_BYTES = 16;
|
|
145
|
-
/**
|
|
146
|
-
* Hard cap on one encrypted chunk's stored payload (a cleartext one is capped at the decode cap): the decode cap plus the AEAD framing, so an
|
|
147
|
-
* entry the engine would refuse to decode is refused when the object is opened, before any payload is read. No writer
|
|
148
|
-
* exceeds it: a stored payload is a serialized 16-bit chunk (at most about 8.2 KiB) plus, when encrypted, that framing.
|
|
149
|
-
*/
|
|
150
|
-
export declare const DEFAULT_MAX_PAYLOAD_BYTES: number;
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The two widths every `.crbm` v1 object is framed by: its front preamble and its footer. Frozen with the rest of the
|
|
3
|
-
* layout in `format.ts`, which re-exports them. They live in a module of their own, with no imports, so that what needs
|
|
4
|
-
* only the smallest size an object can have (the registry's check of a summary's fingerprint) does not bring the
|
|
5
|
-
* format's other constants, or the reader, with it.
|
|
6
|
-
*/
|
|
7
|
-
/** Front preamble: magic(4) + version_major(1) + version_minor(1) + reserved(2). */
|
|
8
|
-
export declare const PREAMBLE_BYTES = 8;
|
|
9
|
-
/** Fixed footer size (v1.0). */
|
|
10
|
-
export declare const FOOTER_BYTES = 104;
|
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The most unneeded bytes one read may carry between two needed chunks: reading 256 KiB that nothing wants costs
|
|
3
|
-
* a few milliseconds of transfer, while the request it saves costs a round trip.
|
|
4
|
-
*/
|
|
5
|
-
export declare const MAX_COALESCE_GAP_BYTES: number;
|
|
6
|
-
/**
|
|
7
|
-
* The most bytes one read may span, unless it holds a single chunk. It equals the decode cap, so a window of
|
|
8
|
-
* coalesced reads holds the window's width times the per-chunk cap that bounds a window of single-chunk reads.
|
|
9
|
-
*/
|
|
10
|
-
export declare const MAX_COALESCED_READ_BYTES: number;
|
|
11
|
-
/**
|
|
12
|
-
* The default number of range reads a chunk stream holds ahead of its consumer, in flight or landed and not yet taken;
|
|
13
|
-
* the rest of the plan follows as the consumer takes them. A range is at most {@link MAX_COALESCED_READ_BYTES} unless
|
|
14
|
-
* it is one chunk, so a stream holds at most this many times the per-chunk decode cap, whatever the key list and the
|
|
15
|
-
* object: the memory bound a window of single-chunk reads has, and no more connections than it opens.
|
|
16
|
-
*/
|
|
17
|
-
export declare const MAX_RANGES_IN_FLIGHT = 32;
|
|
18
|
-
/** One needed chunk, as its index entry places it in the object. */
|
|
19
|
-
export interface ChunkExtent {
|
|
20
|
-
readonly key: number;
|
|
21
|
-
readonly offset: number;
|
|
22
|
-
readonly length: number;
|
|
23
|
-
}
|
|
24
|
-
/** One byte range to read, and the needed chunks inside it, in object order. */
|
|
25
|
-
export interface PlannedRead {
|
|
26
|
-
readonly offset: number;
|
|
27
|
-
readonly length: number;
|
|
28
|
-
readonly chunks: readonly ChunkExtent[];
|
|
29
|
-
}
|
|
30
|
-
/** The part of an object that holds chunk payloads: `[start, end)`. */
|
|
31
|
-
export interface ChunkRegion {
|
|
32
|
-
readonly start: number;
|
|
33
|
-
readonly end: number;
|
|
34
|
-
}
|
|
35
|
-
/**
|
|
36
|
-
* Plan the reads for `extents`: the needed chunks of one object, ascending by offset (the index lists chunks in key
|
|
37
|
-
* order, which is object order), each at least a byte long, none overlapping another. The reads come out ascending
|
|
38
|
-
* and disjoint, each covering one or more of the chunks, every chunk in exactly one read.
|
|
39
|
-
*
|
|
40
|
-
* Lazy: the plan is a walk, so a consumer that takes the first reads pulls only the extents those reads cover (and
|
|
41
|
-
* the one after, which closes the last of them), and a plan over a very wide object costs nothing until it is read.
|
|
42
|
-
* Each extent is checked as the walk reaches it, before the read that would carry it is handed out, so no read ever
|
|
43
|
-
* covers an extent that was not checked; a plan taken to its end has checked them all. The reads are exactly the ones
|
|
44
|
-
* a plan made all at once would hold, in the same order.
|
|
45
|
-
*
|
|
46
|
-
* Throws {@link IntegrityError} for an extent that is not a pair of safe integers, or is outside `region`, or out of
|
|
47
|
-
* order, or overlapping the one before it:
|
|
48
|
-
* the index a plan is made from is untrusted bytes, and a read is never planned outside the chunk region.
|
|
49
|
-
*/
|
|
50
|
-
export declare function planChunkReads(extents: Iterable<ChunkExtent>, region: ChunkRegion): Generator<PlannedRead, void, undefined>;
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
/** Append the LEB128 encoding of a u32 `value` to `out`. */
|
|
2
|
-
export declare function writeVarint(out: number[], value: number): void;
|
|
3
|
-
interface VarintRead {
|
|
4
|
-
readonly value: number;
|
|
5
|
-
/** Offset just past the bytes consumed. */
|
|
6
|
-
readonly next: number;
|
|
7
|
-
}
|
|
8
|
-
/**
|
|
9
|
-
* Decode a LEB128 varint from `bytes` starting at `offset`. Throws `IntegrityError` on a truncated or
|
|
10
|
-
* over-long (> 5-byte / > u32) encoding — the bytes are untrusted, so a malformed index must fail
|
|
11
|
-
* cleanly, never loop or overflow.
|
|
12
|
-
*/
|
|
13
|
-
export declare function readVarint(bytes: Uint8Array, offset: number): VarintRead;
|
|
14
|
-
export {};
|