@cloudbitmaps/core 0.19.1 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/README.md +85 -0
  2. package/dist/{chunk-RSLVTRJY.js → chunk-IGLIZBR7.js} +982 -281
  3. package/dist/chunk-IGLIZBR7.js.map +7 -0
  4. package/dist/core/absent-operands.d.ts +8 -0
  5. package/dist/core/audit.d.ts +67 -4
  6. package/dist/core/bit-route.d.ts +2 -1
  7. package/dist/core/blob.d.ts +12 -1
  8. package/dist/core/bulk-load.d.ts +148 -0
  9. package/dist/core/cas-retry.d.ts +27 -0
  10. package/dist/core/check.d.ts +22 -0
  11. package/dist/core/chunk-checks.d.ts +6 -1
  12. package/dist/core/chunk-window.d.ts +11 -4
  13. package/dist/core/combine-expr.d.ts +1 -3
  14. package/dist/core/combine-feed.d.ts +2 -16
  15. package/dist/core/combine-many.d.ts +21 -15
  16. package/dist/core/concurrency.d.ts +8 -0
  17. package/dist/core/consistency.d.ts +76 -18
  18. package/dist/core/crbm/fingerprint.d.ts +23 -0
  19. package/dist/core/crbm/format.d.ts +2 -5
  20. package/dist/core/crbm/frame.d.ts +10 -0
  21. package/dist/core/crbm/plan-reads.d.ts +9 -3
  22. package/dist/core/crbm/reader.d.ts +51 -12
  23. package/dist/core/crbm/varint.d.ts +2 -1
  24. package/dist/core/crbm/writer.d.ts +5 -1
  25. package/dist/core/crbm-open.d.ts +154 -0
  26. package/dist/core/crbm-storage-source.d.ts +243 -490
  27. package/dist/core/crbm-write.d.ts +75 -0
  28. package/dist/core/crypto.d.ts +6 -1
  29. package/dist/core/engine.d.ts +239 -47
  30. package/dist/core/erase-id.d.ts +97 -27
  31. package/dist/core/erasure.d.ts +4 -3
  32. package/dist/core/errors.d.ts +8 -2
  33. package/dist/core/generation-gc.d.ts +131 -16
  34. package/dist/core/generation-list.d.ts +11 -0
  35. package/dist/core/id-window.d.ts +46 -0
  36. package/dist/core/invalidation-scope.d.ts +77 -0
  37. package/dist/core/known-generations-shape.d.ts +13 -0
  38. package/dist/core/known-generations.d.ts +147 -0
  39. package/dist/core/leases.d.ts +9 -1
  40. package/dist/core/ledger-seed.d.ts +17 -0
  41. package/dist/core/load.d.ts +17 -36
  42. package/dist/core/lru.d.ts +2 -15
  43. package/dist/core/metrics.d.ts +9 -7
  44. package/dist/core/order.d.ts +2 -0
  45. package/dist/core/pinned-storage-source.d.ts +7 -7
  46. package/dist/core/pointer-id.d.ts +19 -0
  47. package/dist/core/ports.d.ts +210 -45
  48. package/dist/core/publish.d.ts +207 -0
  49. package/dist/core/reader-defaults.d.ts +14 -3
  50. package/dist/core/registry-scan.d.ts +0 -2
  51. package/dist/core/retention-sweep.d.ts +69 -65
  52. package/dist/core/retention.d.ts +1 -1
  53. package/dist/core/retry.d.ts +12 -19
  54. package/dist/core/rollback.d.ts +62 -32
  55. package/dist/core/summary.d.ts +74 -25
  56. package/dist/core/token.d.ts +13 -8
  57. package/dist/core/validate.d.ts +2 -0
  58. package/dist/driver-kit.d.ts +6 -1
  59. package/dist/driver-kit.js +90 -10
  60. package/dist/driver-kit.js.map +3 -3
  61. package/dist/drivers/_shared/bytes.d.ts +6 -0
  62. package/dist/drivers/_shared/faults.d.ts +15 -0
  63. package/dist/drivers/_shared/object-keys.d.ts +18 -0
  64. package/dist/drivers/_shared/object-registry-keys.d.ts +0 -2
  65. package/dist/drivers/_shared/object-registry.d.ts +3 -1
  66. package/dist/drivers/_shared/options.d.ts +19 -0
  67. package/dist/drivers/_shared/registry.d.ts +31 -19
  68. package/dist/drivers/localfs/exact-case.d.ts +0 -3
  69. package/dist/drivers/localfs/fs-util.d.ts +22 -0
  70. package/dist/drivers/localfs/registry.d.ts +10 -4
  71. package/dist/drivers/localfs/storage.d.ts +16 -8
  72. package/dist/drivers/memory.d.ts +14 -9
  73. package/dist/drivers/retry/retrying-chunk-source.d.ts +1 -5
  74. package/dist/export/index.d.ts +16 -0
  75. package/dist/index.d.ts +2 -5
  76. package/dist/index.js +4303 -2938
  77. package/dist/index.js.map +4 -4
  78. package/package.json +1 -1
  79. package/dist/chunk-RSLVTRJY.js.map +0 -7
  80. package/dist/core/cost.d.ts +0 -351
  81. package/dist/core/kept-generations.d.ts +0 -40
@@ -3,18 +3,23 @@
3
3
  * generations can be restored **independently**, so a failover can recover the registry *ahead of* the storage
4
4
  * objects — leaving `currentGen` pointing at a generation whose `.crbm` isn't present yet. That is likelier
5
5
  * than it sounds even with both in one bucket: a restore scoped to a prefix, or replayed
6
- * per-object from a version history, recovers the two prefixes at different points. That's a torn restore: reads of the affected segment then throw. This scan
6
+ * per-object from a version history, recovers the two prefixes at different points. That's a torn restore: reads of
7
+ * the affected segment then throw. This scan
7
8
  * detects it up front (run it at startup after a restore) instead of discovering it on the first read.
8
9
  *
9
10
  * Read-only; bounded fan-out. `destroyed` (crypto-shredded) segments are skipped — their Storage is intentionally
10
11
  * gone/unreadable, not a torn restore. A segment whose Storage/registry can't be read this pass is recorded in
11
- * `errored` (never aborts the scan). Each segment is checked against its **authoritative live pointer** — one
12
+ * `errored` (never aborts the scan). One segment can add several entries to `inconsistent` (its current object, and with
13
+ * `generations` one per known generation whose object is not the one recorded) and can also be in `errored`, when a
14
+ * read failed after those were found; with `generations` a segment costs up to one read per entry of its ledger.
15
+ * Each segment is checked against its **authoritative live pointer** — one
12
16
  * strong `registry.get` per segment — never the enumeration snapshot from `registry.list`, which a driver may
13
17
  * serve eventually-consistently and so lag a recent in-place pointer advance: trusting it would both
14
18
  * miss a torn *live* generation and cry torn on a generation the pointer has already advanced past (GC'd during
15
- * the scan). Residual: a load's publish plus a GC landing in the tiny per-segment get→list gap can still yield a
16
- * transient false positive — run the scan against a quiesced fleet (the documented restore procedure), or re-run
17
- * to confirm a reported tear.
19
+ * the scan). Residual: a load's publish plus a GC landing in the per-segment gap between the pointer read and the
20
+ * object reads (one read, or with `generations` as many as the ledger names) can still yield a transient false
21
+ * positive — run the scan against a quiesced fleet (the documented restore procedure), or re-run to confirm a
22
+ * reported tear.
18
23
  */
19
24
  import type { IStorageDriver, IRegistryDriver } from './ports.js';
20
25
  import type { IKeystore } from './crypto.js';
@@ -33,15 +38,35 @@ export { DEFAULT_MAX_SCAN_SEGMENTS } from './registry-scan.js';
33
38
  export interface ConsistencyIssue {
34
39
  readonly segment: string;
35
40
  readonly namespace?: string;
36
- /** The registry's `currentGen` for the segment — the generation whose `.crbm` is missing from Storage. */
41
+ /**
42
+ * The registry's `currentGen` for the segment. For `missing-storage-generation` it is the generation whose `.crbm` is
43
+ * missing from Storage; for the other issues it is the segment's current generation, whichever generation the issue
44
+ * is about (see `generation`).
45
+ */
37
46
  readonly currentGen: number;
38
47
  /**
39
48
  * `missing-storage-generation`: `currentGen` references a Storage generation that is not present (torn restore).
40
- * `summary-mismatch` (only when asked for): the generation is there, but its row's summary says a different id
41
- * count or metadata than the object holds (a row restored from another point than its bucket, or a number re-taken
42
- * since). A count answers from that summary until a read opens the object, so it is what a restore leaves wrong.
49
+ * `summary-mismatch` (only when asked for): the generation is there, but its row's summary names another object
50
+ * than the one under its number, says a different id count or metadata than the object holds (a row restored from
51
+ * another point than its bucket, or an object put back from outside the library), or records a fingerprint that
52
+ * names no object. A summary that names another object is what a count and a `stat()` answer from while every read
53
+ * that opens the object refuses it; one whose count or metadata disagrees is answered from until a read opens the
54
+ * object, which stops that store using it; one whose fingerprint names no object is not used, so every read opens the
55
+ * object unchecked.
56
+ * `known-generation-mismatch` (only when asked for): the object under the number `generation` is not the one the row's
57
+ * ledger of known generations recorded for it: its footer checksum differs, or its footer fails its own checks (it is
58
+ * not a `.crbm` object, or is cut short). It says nothing of what a rollback does with that generation.
59
+ * What it covers: each object's footer is held to the recorded checksum, so another object, or bytes that are not a
60
+ * `.crbm`, under a known number is found. It does not find damage inside an object whose footer is intact (a cut or a
61
+ * flipped byte in the body: a read of that generation reports it as `IntegrityError`), nor a metadata change that keeps
62
+ * the footer's bytes.
43
63
  */
44
- readonly issue: 'missing-storage-generation' | 'summary-mismatch';
64
+ readonly issue: 'missing-storage-generation' | 'summary-mismatch' | 'known-generation-mismatch';
65
+ /**
66
+ * With `known-generation-mismatch` only: the generation whose object is not the one the row recorded. It is
67
+ * `currentGen` when the current object is the one that differs. Absent on every other issue.
68
+ */
69
+ readonly generation?: number;
45
70
  }
46
71
  /** A segment that could not be checked this pass (Storage/registry read fault) — not proof of a torn restore. */
47
72
  export interface ConsistencyErrorEntry {
@@ -53,12 +78,17 @@ export interface ConsistencyErrorEntry {
53
78
  export interface ConsistencyReport {
54
79
  /** Registered segments scanned (a `destroyed` segment counts as scanned but is never an issue). */
55
80
  readonly checked: number;
56
- /** Segments whose `currentGen` `.crbm` is absent — recover the object store (or restore to a coherent point). */
81
+ /**
82
+ * The issues found, one entry each: a segment whose `currentGen` `.crbm` is absent (recover the object store, or
83
+ * restore to a coherent point), and with the options asked for the other kinds. One segment can have several entries:
84
+ * at most one for its current object plus one per entry of its ledger, so at most 1 plus the ledger's length (384 at
85
+ * most). The ledger's length is bounded by the registry, and the report is not capped.
86
+ */
57
87
  readonly inconsistent: ConsistencyIssue[];
58
88
  /**
59
89
  * Segments that couldn't be read this pass (a transient/partial object store during a restore is exactly when
60
- * this runs) — **triage these too**: an unread segment is neither proven-coherent nor proven-torn. Empty on a
61
- * clean pass.
90
+ * this runs) — **triage these too**: an unread segment is neither proven-coherent nor proven-torn. A segment can be in
91
+ * this list and in `inconsistent` too: what was found before the read that failed is kept. Empty on a clean pass.
62
92
  */
63
93
  readonly errored: ConsistencyErrorEntry[];
64
94
  /**
@@ -67,17 +97,44 @@ export interface ConsistencyReport {
67
97
  * only when summaries were asked for.
68
98
  */
69
99
  readonly summariesUnchecked?: number;
100
+ /**
101
+ * With `generations: true`, the rows with a pointer whose ledger of known generations could not be used (absent, or
102
+ * failing the validation every reader of it applies, or not shaped as a ledger a registry of this library stores):
103
+ * none of their known generations was held against its object. A count, not an issue. Present only when generations
104
+ * were asked for.
105
+ */
106
+ readonly generationsUnchecked?: number;
70
107
  }
71
108
  /**
72
109
  * Verify every registered segment's `currentGen` `.crbm` actually exists in Storage. Enumerates the registry
73
- * (optionally one namespace) and, for each non-`destroyed` segment, checks the object store lists that
74
- * generation. Returns the torn segments in `inconsistent` (empty ⇒ coherent) and any unreadable segments in
110
+ * (optionally one namespace) and, for each non-`destroyed` segment, asks the object store for that generation with a
111
+ * zero-length tail read. Returns the torn segments in `inconsistent` (empty ⇒ coherent) and any unreadable segments in
75
112
  * `errored`.
76
113
  *
114
+ * When that read fails, in any way, the segment's listing decides and its own error is the segment's, so a missing
115
+ * object, a missing bucket and a failing store are reported as when every segment was listed. One difference: a store
116
+ * that answers reads but refuses listings (an identity without list permission, or a listing-only outage) reports each
117
+ * object that is there as present, where every segment was reported errored.
118
+ *
77
119
  * With `summaries: true` it also opens each segment's current object (one tail read, and a second for an index longer
78
- * than it) and holds the row's summary against it, reporting `summary-mismatch` where they disagree. A sealed summary
79
- * needs `deps.keystore`; without it the segment is counted in `summariesUnchecked`. Off by default, since the
80
- * default check only lists.
120
+ * than it) and holds the row's summary against it: the object it names by its fingerprint, and the count and metadata
121
+ * it says, reporting `summary-mismatch` where they disagree, and for a clear summary whose fingerprint names no
122
+ * object, which no read uses. For a row with a clear summary that open is the only request of the object; if it fails
123
+ * before its first read answered, the listing decides, and once that read has answered a failure is an error of the
124
+ * segment. A sealed summary needs `deps.keystore`; without it the segment is counted in `summariesUnchecked`. Off by
125
+ * default, since the default check only looks for the object.
126
+ *
127
+ * With `generations: true` it also holds every generation a row's ledger of known generations names, the current one
128
+ * included, against its object: one footer read each (104 bytes, no key, not retried), reporting
129
+ * `known-generation-mismatch` with the `generation` where the object's footer checksum is not the one the ledger records
130
+ * or its footer fails its own checks. That finds another object, or bytes that are not a `.crbm`, under a known number;
131
+ * it does not find damage inside an object whose footer is intact (a read of it reports that as `IntegrityError`), nor a
132
+ * metadata change that keeps the footer's bytes. A segment adds at most 1 plus its ledger's length issues. A named
133
+ * generation that is not in the bucket is not reported. A row with a pointer whose ledger cannot be used (absent, failing
134
+ * the validation every reader of it applies, or not shaped as a ledger a registry of this library stores) is counted in
135
+ * `generationsUnchecked` and none of its objects is read. The listing is not used: a read of a missing object
136
+ * answers `NotFoundError`, which is how an absent one is told. A segment whose current object is missing is reported as
137
+ * ever and its older generations are still checked. Off by default.
81
138
  */
82
139
  export declare function runConsistencyCheck(deps: {
83
140
  readonly storage: IStorageDriver;
@@ -88,4 +145,5 @@ export declare function runConsistencyCheck(deps: {
88
145
  concurrency?: number;
89
146
  maxScanSegments?: number;
90
147
  summaries?: boolean;
148
+ generations?: boolean;
91
149
  }): Promise<ConsistencyReport>;
@@ -0,0 +1,23 @@
1
+ /** What names an object: its size, then its footer's CRC. One spelling, for an open reader and a footer read alone. */
2
+ export declare const sizePart: (size: number) => string;
3
+ /** An object's fingerprint, from its size and its footer's CRC (see `CrbmReader.fingerprint`). */
4
+ export declare const fingerprintFor: (size: number, footerCrc: number) => string;
5
+ /**
6
+ * A fingerprint taken apart, or `undefined` for text that is not one an object can have: a size that is a safe integer
7
+ * no smaller than a `.crbm` preamble and footer, and a footer CRC that fits in 32 bits. For a row's summary, which
8
+ * names its object by it and reports the size from it.
9
+ */
10
+ export declare function fingerprintParts(fingerprint: unknown): {
11
+ readonly size: number;
12
+ readonly footerCrc: number;
13
+ } | undefined;
14
+ /**
15
+ * The footer CRC of a fingerprint a writer or a reader produced, which a row's ledger records beside the generation.
16
+ * Throws {@link IntegrityError} for text that is not a fingerprint, which neither produces.
17
+ */
18
+ export declare function footerCrcOf(fingerprint: string): number;
19
+ /**
20
+ * A fingerprint spelled short, for a key a read builds on every chunk: its size and its footer's CRC in base 36,
21
+ * `<size>.<crc>`. Two compact spellings are equal exactly when the fingerprints are.
22
+ */
23
+ export declare function compactFingerprint(fingerprint: string): string;
@@ -4,16 +4,14 @@
4
4
  * These byte widths/offsets are pinned by the golden corpus and must never change for v1 —
5
5
  * a new layout is a new format version. All multi-byte integers are little-endian (v1 fixes LE).
6
6
  */
7
+ import { FOOTER_BYTES, PREAMBLE_BYTES } from './frame.js';
7
8
  /** 4-byte magic at both file ends: "CRBM". */
8
9
  export declare const MAGIC: Uint8Array<ArrayBuffer>;
9
10
  export declare const VERSION_MAJOR = 1;
10
11
  export declare const VERSION_MINOR = 0;
11
- /** Front preamble: magic(4) + version_major(1) + version_minor(1) + reserved(2). */
12
- export declare const PREAMBLE_BYTES = 8;
12
+ export { FOOTER_BYTES, PREAMBLE_BYTES };
13
13
  /** Payloads begin immediately after the preamble. */
14
14
  export declare const PAYLOAD_START = 8;
15
- /** Fixed footer size (v1.0). */
16
- export declare const FOOTER_BYTES = 104;
17
15
  /** Byte offsets of each field within the 104-byte footer. */
18
16
  export declare const FOOTER: {
19
17
  readonly indexOffset: 0;
@@ -39,7 +37,6 @@ export declare const FOOTER: {
39
37
  export declare const FOOTER_CRC_COVERAGE: 96;
40
38
  /** `flags` bit positions. */
41
39
  export declare const FLAG_ENCRYPTED: number;
42
- export declare const FLAG_INDEX_COMPRESSED: number;
43
40
  export declare const FLAG_LITTLE_ENDIAN: number;
44
41
  /**
45
42
  * The object carries an extension block just before its index (a generation's metadata). Set only when there is
@@ -0,0 +1,10 @@
1
+ /**
2
+ * The two widths every `.crbm` v1 object is framed by: its front preamble and its footer. Frozen with the rest of the
3
+ * layout in `format.ts`, which re-exports them. They live in a module of their own, with no imports, so that what needs
4
+ * only the smallest size an object can have (the registry's check of a summary's fingerprint) does not bring the
5
+ * format's other constants, or the reader, with it.
6
+ */
7
+ /** Front preamble: magic(4) + version_major(1) + version_minor(1) + reserved(2). */
8
+ export declare const PREAMBLE_BYTES = 8;
9
+ /** Fixed footer size (v1.0). */
10
+ export declare const FOOTER_BYTES = 104;
@@ -34,11 +34,17 @@ export interface ChunkRegion {
34
34
  }
35
35
  /**
36
36
  * Plan the reads for `extents`: the needed chunks of one object, ascending by offset (the index lists chunks in key
37
- * order, which is object order), each at least a byte long, none overlapping another. The reads come back
38
- * ascending and disjoint, each covering one or more of the chunks, every chunk in exactly one read.
37
+ * order, which is object order), each at least a byte long, none overlapping another. The reads come out ascending
38
+ * and disjoint, each covering one or more of the chunks, every chunk in exactly one read.
39
+ *
40
+ * Lazy: the plan is a walk, so a consumer that takes the first reads pulls only the extents those reads cover (and
41
+ * the one after, which closes the last of them), and a plan over a very wide object costs nothing until it is read.
42
+ * Each extent is checked as the walk reaches it, before the read that would carry it is handed out, so no read ever
43
+ * covers an extent that was not checked; a plan taken to its end has checked them all. The reads are exactly the ones
44
+ * a plan made all at once would hold, in the same order.
39
45
  *
40
46
  * Throws {@link IntegrityError} for an extent that is not a pair of safe integers, or is outside `region`, or out of
41
47
  * order, or overlapping the one before it:
42
48
  * the index a plan is made from is untrusted bytes, and a read is never planned outside the chunk region.
43
49
  */
44
- export declare function planChunkReads(extents: readonly ChunkExtent[], region: ChunkRegion): readonly PlannedRead[];
50
+ export declare function planChunkReads(extents: Iterable<ChunkExtent>, region: ChunkRegion): Generator<PlannedRead, void, undefined>;
@@ -7,7 +7,7 @@ import type { ChunkRead, GenerationMetadata, ReadChunksOptions } from '../ports.
7
7
  * `cardinalityMinusOne` stores `cardinality - 1` because a chunk's cardinality runs `[1, 65536]` and 65,536 does
8
8
  * not fit a `u16`; an offset is a `f64` because an object past 4 GiB puts a payload beyond a `u32`.
9
9
  */
10
- export interface ParsedIndex {
10
+ interface ParsedIndex {
11
11
  readonly keys: Uint16Array;
12
12
  readonly cardinalityMinusOne: Uint16Array;
13
13
  readonly lengths: Uint32Array;
@@ -29,7 +29,12 @@ export declare const RETAINED_BYTES_PER_INDEX_ENTRY = 20;
29
29
  */
30
30
  export declare function indexCapacity(indexLength: number): number;
31
31
  export interface CrbmReaderOptions {
32
- /** Speculative tail size in bytes (default 256 KB; clamped up to at least the footer size). */
32
+ /**
33
+ * How many bytes the first tail read asks for, a whole number (default 256 KiB; clamped up to at least the footer
34
+ * size). Any value of at least an object's `tailBytes` opens it in that one read. Where an open keeps a small object's
35
+ * chunks (a `CrbmStorageChunkSource`'s does), it keeps them only when this read covers the whole object, so a value
36
+ * below the default can leave them to reads of their own.
37
+ */
33
38
  readonly tailBytes?: number;
34
39
  /** Hard cap on a single chunk payload length (default: the 1 MiB decode cap, plus the 28-byte nonce and tag on an encrypted object). The engine decodes under `maxBitmapBytes`; raise this with it. */
35
40
  readonly maxPayloadBytes?: number;
@@ -49,13 +54,20 @@ export interface CrbmReaderOptions {
49
54
  */
50
55
  readonly lineage?: unknown;
51
56
  }
52
- /** An object's fingerprint, from its size and its footer's CRC (see {@link CrbmReader.fingerprint}). */
53
- export declare const fingerprintFor: (size: number, footerCrc: number) => string;
54
57
  /**
55
- * A fingerprint spelled short, for a key a read builds on every chunk: its size and its footer's CRC in base 36,
56
- * `<size>.<crc>`. Two compact spellings are equal exactly when the fingerprints are.
58
+ * Refuses a `size` that is not a whole byte count its own tail fits in. The size is the tier's word too: one that
59
+ * is not would turn off every bounds check an open makes against it.
60
+ */
61
+ export declare function checkSize(tail: Uint8Array, size: number): void;
62
+ /**
63
+ * What the footer of the object behind `blob` says with no key and no index, from one tail read of a footer's worth:
64
+ * whether it is encrypted, and its CRC (the one in {@link CrbmReader.fingerprint}). A footer that fails its own checks
65
+ * throws, as an open would.
57
66
  */
58
- export declare function compactFingerprint(fingerprint: string): string;
67
+ export declare function readFooter(blob: BlobReader): Promise<{
68
+ readonly encrypted: boolean;
69
+ readonly footerCrc: number;
70
+ }>;
59
71
  /**
60
72
  * Whether the object behind `blob` says it is encrypted, from one tail read of a footer's worth, with no key and no
61
73
  * index; a footer that fails its own checks throws, as an open would. For a caller that must not open an object with
@@ -68,6 +80,14 @@ export declare function footerSaysEncrypted(blob: BlobReader): Promise<boolean>;
68
80
  * object and that region is at most `keepChunkBytesUpTo` bytes. Not exported from a package entry.
69
81
  */
70
82
  export declare function openCrbmReaderKeeping(blob: BlobReader, options: CrbmReaderOptions, keepChunkBytesUpTo: number | undefined): Promise<CrbmReader>;
83
+ /**
84
+ * How many bytes the first tail read of an open asks for: the caller's `tailBytes` or the default, and a footer's worth
85
+ * at least. Not exported from a package entry: {@link CrbmReader.open} and a caller that sends that read ahead of it
86
+ * ask for the same size.
87
+ */
88
+ export declare function openTailBytes(options: {
89
+ readonly tailBytes?: number;
90
+ }): number;
71
91
  export declare class CrbmReader {
72
92
  private readonly blob;
73
93
  private readonly objectSize;
@@ -107,8 +127,13 @@ export declare class CrbmReader {
107
127
  * the object it reopens is the one it pinned, since a purged and reloaded name reuses the generation number.
108
128
  */
109
129
  get fingerprint(): string;
110
- /** Total object bytes (from the one-GET tail read) — for grounded storage cost. */
130
+ /** Total object bytes (from the one-GET tail read): the `sizeBytes` a segment's `stat()` reports. */
111
131
  get sizeBytes(): number;
132
+ /**
133
+ * The bytes of the object after its chunk payloads: any extension block, the index and the footer. A tail read of
134
+ * this many bytes covers all of them.
135
+ */
136
+ get tailBytes(): number;
112
137
  /**
113
138
  * Retained JS heap of this reader's parsed index — the weight the storage reader cache bounds on (a wide
114
139
  * segment's parsed index, not its payloads, dominates the reader's footprint): the exact byte length of the
@@ -141,6 +166,16 @@ export declare class CrbmReader {
141
166
  count(): number;
142
167
  /** The slot holding `chunkKey` (binary search over the ascending keys), or -1 when this generation has none. */
143
168
  private slotOf;
169
+ /**
170
+ * {@link slotOf} for a key at or after the one that held slot `at`, which a walk over ascending keys has: the slot
171
+ * itself or the next are tried first, so a read of nearly every chunk costs no search.
172
+ */
173
+ private slotFrom;
174
+ /**
175
+ * The extents of the chunks `chunkKeys` name that this generation holds, in key order, each once (a key given twice
176
+ * is read once). A walk, so nothing is looked up until the plan pulls it.
177
+ */
178
+ private neededExtents;
144
179
  has(chunkKey: number): boolean;
145
180
  /**
146
181
  * Range-GET one chunk's payload, verifying its CRC32C before returning. Returns `null` if the chunk is
@@ -175,10 +210,13 @@ export declare class CrbmReader {
175
210
  * as {@link getChunk} answers for an absent key. A key given twice is read once and answered at both positions with
176
211
  * the same view.
177
212
  *
178
- * The whole key list is planned once, up front, against this one generation's index. The ranges then run through a
179
- * window of `options.concurrency` reads ({@link MAX_RANGES_IN_FLIGHT} by default) that counts a range until the
180
- * consumer has taken it, so a stream never holds more than that many ranges, in flight or landed, however many keys
181
- * it was given or how slowly it is read; with `options.ramp` the window opens 1, 2, 4 … wide. A range is at most 1 MiB
213
+ * The key list is planned against this one generation's index ({@link planChunkReads}), as a walk the window pulls
214
+ * from only as it opens a range, so a stream that stops early has planned little: the same ranges, in the same
215
+ * order, as a plan of the whole list would make, and each chunk's extent is checked before the range that carries
216
+ * it is requested. The ranges run through a window of `options.concurrency` reads ({@link MAX_RANGES_IN_FLIGHT} by
217
+ * default) that counts a range until the consumer has taken it, so a stream never holds more than that many ranges,
218
+ * in flight or landed, however many keys it was given or how slowly it is read; with `options.ramp` the window opens
219
+ * 1, 2, 4 … wide. A range is at most 1 MiB
182
220
  * unless it is one chunk, which is at most the payload cap, so the bound is that many times the per-chunk cap
183
221
  * (1 MiB, and 28 bytes more for an encrypted chunk). Nothing is read until the first item is asked for, and a
184
222
  * consumer that stops launches no more: ranges already in flight finish and are dropped, and a failure of one of
@@ -230,3 +268,4 @@ export type ChunkStream = AsyncGenerator<Omit<ChunkRead, 'version'>> & {
230
268
  */
231
269
  export declare function parseExtension(sections: Uint8Array, crypto: CrbmCrypto | undefined): GenerationMetadata | undefined;
232
270
  export declare function parseIndex(indexBytes: Uint8Array, payloadEnd: number, maxPayloadBytes: number, minPayloadBytes?: number): ParsedIndex;
271
+ export {};
@@ -1,6 +1,6 @@
1
1
  /** Append the LEB128 encoding of a u32 `value` to `out`. */
2
2
  export declare function writeVarint(out: number[], value: number): void;
3
- export interface VarintRead {
3
+ interface VarintRead {
4
4
  readonly value: number;
5
5
  /** Offset just past the bytes consumed. */
6
6
  readonly next: number;
@@ -11,3 +11,4 @@ export interface VarintRead {
11
11
  * cleanly, never loop or overflow.
12
12
  */
13
13
  export declare function readVarint(bytes: Uint8Array, offset: number): VarintRead;
14
+ export {};
@@ -35,6 +35,8 @@ export declare class CrbmWriter {
35
35
  private readonly metadata;
36
36
  /** Bytes written so far, the footer included once {@link finish} has run. */
37
37
  private written;
38
+ /** Where the chunk payloads end: the extension block's start when there is metadata, else the index's. */
39
+ private payloadEnd;
38
40
  private footerCrc32c;
39
41
  constructor(sink: BlobSink, options: CrbmWriterOptions);
40
42
  /**
@@ -50,11 +52,13 @@ export declare class CrbmWriter {
50
52
  /**
51
53
  * The finished object's size and its footer's CRC: what names it, as a reader's fingerprint does. A writer that
52
54
  * has to tell its own object from another stored later under the same key compares these with the object's
53
- * footer. Throws before {@link finish}.
55
+ * footer. `tailBytes` is the bytes after the chunk payloads (any extension block, the index and the footer), which
56
+ * is what {@link CrbmReader.tailBytes} reports for the object. Throws before {@link finish}.
54
57
  */
55
58
  get identity(): {
56
59
  readonly size: number;
57
60
  readonly footerCrc: number;
61
+ readonly tailBytes: number;
58
62
  };
59
63
  private ensurePreamble;
60
64
  /**
@@ -0,0 +1,154 @@
1
+ import type { BlobReader } from './blob.js';
2
+ import type { CrbmCrypto } from './crypto.js';
3
+ import type { GenerationMetadata, GenKey, IStorageDriver, TailRead } from './ports.js';
4
+ import { CrbmReader } from './crbm/reader.js';
5
+ import type { CrbmReaderOptions } from './crbm/reader.js';
6
+ import type { SummaryDescription } from './summary.js';
7
+ /**
8
+ * Adapt one `(driver, key)` pair to the codec's `BlobReader` seam. With `noteTail`, each tail read the reader makes is
9
+ * shown to it with the bytes, `size` and `version` the driver answered, after the copy below when one is made.
10
+ *
11
+ * A tail's bytes reach the reader as a plain `Uint8Array`: the reader keeps a small object's chunk region by slicing
12
+ * the tail, and a subclass's `slice` need not copy. A Node `Buffer`'s is a view, so a driver that answered with one
13
+ * would leave the reader holding the driver's memory, all of it, as the cache weighs only the region. Such an answer
14
+ * is copied here, once ({@link ownedTail}); a plain `Uint8Array` of this realm, which every shipped driver returns, is
15
+ * handed through as it is.
16
+ */
17
+ export declare function storageBlobReader(driver: IStorageDriver, key: GenKey, noteTail?: (tail: TailRead) => void): BlobReader;
18
+ /**
19
+ * `tail` with its bytes in a plain `Uint8Array` of this realm. Any other `Uint8Array`, a subclass (a Node `Buffer`) or
20
+ * one made in another realm (a `vm` context), is copied once, and the copy is built from the answer's fields by name,
21
+ * so a `size` or `version` the driver serves from a getter on its own class carries over. The brand is read, not
22
+ * `instanceof`, which is false for another realm's arrays. A plain `Uint8Array` of this realm is handed through as it is.
23
+ */
24
+ export declare function ownedTail(tail: TailRead): TailRead;
25
+ /**
26
+ * The object under `key`, when one read of its footer matches `fingerprint` ({@link CrbmReader.fingerprint}), with the
27
+ * version the driver reported on that read (`undefined` from a driver that reports none). `null` when it is another
28
+ * object, when it is gone, and when the read fails in any way. A fingerprint names the object's bytes, not the writer:
29
+ * an object another writer stored under the number with the same bytes matches too. A caller that deletes what this
30
+ * matches passes the version as the delete's `ifVersion`, so a number taken again between the read and the delete keeps
31
+ * the object put under it since.
32
+ */
33
+ export declare function provenObject(storage: IStorageDriver, key: GenKey, fingerprint: string): Promise<{
34
+ readonly version: string | undefined;
35
+ } | null>;
36
+ /**
37
+ * Whether the object under `key` is still the one `fingerprint` names ({@link CrbmReader.fingerprint}), from one read
38
+ * of its footer. Unlike {@link provenObject}, a fault is not an answer: `NotFoundError` when the key holds no object,
39
+ * and any other failure of the read, thrown as it is.
40
+ */
41
+ export declare function sameObjectUnder(storage: IStorageDriver, key: GenKey, fingerprint: string): Promise<boolean>;
42
+ /**
43
+ * Whether the object under `key` says it is encrypted, from one read of its footer, with no key. `noteVersion` is shown
44
+ * the version the driver reported on that read, when it reported one: a caller that deletes the object for what its
45
+ * footer says passes it as the delete's `ifVersion`.
46
+ */
47
+ export declare function objectIsEncrypted(storage: IStorageDriver, key: GenKey, noteVersion?: (version: string) => void): Promise<boolean>;
48
+ /**
49
+ * What one tail read of a rollback target says about it with no key: whether the footer says it is encrypted and its
50
+ * CRC, and the remembered tail an open from it ({@link openRollbackTargetFrom}) is answered from.
51
+ */
52
+ interface RollbackTargetFooter {
53
+ readonly encrypted: boolean;
54
+ readonly footerCrc: number;
55
+ readonly blob: BlobReader;
56
+ }
57
+ /**
58
+ * The footer step of a rollback: one tail read of the object under `key`, which is remembered for the open that may
59
+ * follow. A footer that fails its own checks is an {@link IntegrityError}; a missing object is {@link NotFoundError}.
60
+ */
61
+ export declare function readRollbackTargetFooter(storage: IStorageDriver, key: GenKey): Promise<RollbackTargetFooter>;
62
+ /**
63
+ * The reader {@link RollbackTargetFooter}'s object opens as, from its remembered tail, when the object and the key
64
+ * agree (an encrypted object opened with `crypto`, a cleartext one with none); `undefined` when they disagree, which
65
+ * the caller refuses as a target the row cannot use. An object that is another generation's, and an index or metadata
66
+ * that do not open, are an {@link IntegrityError}.
67
+ */
68
+ export declare function openRollbackTargetFrom(footer: RollbackTargetFooter, key: GenKey, crypto: CrbmCrypto | undefined): Promise<CrbmReader | undefined>;
69
+ /**
70
+ * The footer CRC of the object under `key` ({@link CrbmReader.fingerprint} carries it), from one read of a footer's
71
+ * worth, so it reads an encrypted object's CRC as it does a cleartext one's, with no key. {@link NotFoundError} when
72
+ * the key holds no object; a footer that fails its own checks is an {@link IntegrityError}.
73
+ */
74
+ export declare function footerCrcUnder(storage: IStorageDriver, key: GenKey): Promise<number>;
75
+ /**
76
+ * Whether the object under `key` is the one `fingerprint` names, from one read of its footer: false when it is another
77
+ * object or gone. Unlike {@link provenObject}, a read that cannot tell throws, for a caller that must not take "could
78
+ * not look" for "not this object".
79
+ */
80
+ export declare function provesOwnObject(storage: IStorageDriver, key: GenKey, fingerprint: string): Promise<boolean>;
81
+ /**
82
+ * Open a {@link CrbmReader} on one generation over the storage driver's range/tail reads (decrypting if `crypto`).
83
+ * The reader the write paths use to re-read what they wrote, and the erasure rewrite uses to stream the old
84
+ * generation; the engine's read path goes through {@link CrbmStorageChunkSource} instead, which caches these. With
85
+ * `named`, the fingerprint the row's summary records, the object must be that one: another is {@link NotFoundError}.
86
+ * The version of the object it opened, from the tail read the open made, is what {@link objectVersionOf} answers.
87
+ */
88
+ export declare function openGenerationReader(storage: IStorageDriver, key: GenKey, crypto: CrbmCrypto | undefined, options?: Omit<CrbmReaderOptions, 'crypto'>, named?: string): Promise<CrbmReader>;
89
+ /**
90
+ * {@link openGenerationReader}, with `noteVersion` shown the version {@link objectVersionOf} would answer as soon as the
91
+ * tail read that carries it returns, before the open goes on: a caller whose open then fails, on an object sealed under
92
+ * another key, say, still knows which object it met, and can condition a delete of it on that read.
93
+ */
94
+ export declare function openGenerationReaderNoting(storage: IStorageDriver, key: GenKey, crypto: CrbmCrypto | undefined, noteVersion: (version: string) => void): Promise<CrbmReader>;
95
+ /**
96
+ * The version of the object `reader` was opened on, as the driver reported it on the tail read the open made, for a
97
+ * delete conditioned on that object ({@link StorageDeleteOptions.ifVersion}); `undefined` for a reader
98
+ * {@link openGenerationReader} did not open, or a driver that reports none. Held beside the reader rather than on it, so
99
+ * nothing is added to the reader's public surface, and no read the engine makes pays for it.
100
+ */
101
+ export declare function objectVersionOf(reader: CrbmReader): string | undefined;
102
+ /**
103
+ * {@link openGenerationReader}, of the object `summary.fingerprint` names: `undefined` when the object under the key is
104
+ * another one, found from its footer before its index is opened, so an object sealed under another key is told apart as
105
+ * well. The tail read is sized from the summary's `tailBytes` ({@link sizedTailBytes}), so a wide generation's index
106
+ * comes in that read.
107
+ */
108
+ export declare function openNamedGeneration(storage: IStorageDriver, key: GenKey, crypto: CrbmCrypto | undefined, summary: Pick<SummaryDescription, 'fingerprint' | 'tailBytes'>, noteTail?: (tail: TailRead) => void): Promise<CrbmReader | undefined>;
109
+ /**
110
+ * Open a reader on `key`, refusing an object whose footer names another generation. Every writer stamps the key's
111
+ * number, and the chunk cache, the load guard and the erasure rewrite all trust it, so an object that disagrees
112
+ * was written under another key or altered. Every open, the live read's, a pin's and the write paths', comes here.
113
+ * With `named`, the object must also be the one that fingerprint names: another one is {@link NotFoundError}, a move.
114
+ * With `early`, the tail read {@link startTailRead} began, the open takes that read as its own instead of making one.
115
+ */
116
+ export declare function openChecked(storage: IStorageDriver, key: GenKey, options: CrbmReaderOptions, keepChunkBytesUpTo?: number, named?: string, noteTail?: (tail: TailRead) => void, early?: Promise<TailRead>): Promise<CrbmReader>;
117
+ /**
118
+ * The size of the tail read an open makes when its row's summary says how many bytes of the object's end the open
119
+ * needs (`tailBytes`: any extension block, the index and the footer), so that one read reaches the index of a wide
120
+ * generation and no range read follows it. With no summary it is the source's own size ({@link openTailBytes}).
121
+ *
122
+ * The number came from a registry the library may not have written, so it only ever sizes a read: never below the
123
+ * source's own tail, at most the object less its preamble (the size its fingerprint names), and at most the source's
124
+ * `maxIndexBytes` (or the default) plus an extension block, its trailer and the footer, the most any open reads past
125
+ * its own tail. A hint that is not a whole number of bytes above zero is ignored, and so is a result that is not one. A
126
+ * wrong hint is a longer read or one more range read, never a different answer: the reader checks every byte it takes.
127
+ */
128
+ export declare function sizedTailBytes(summary: Pick<SummaryDescription, 'fingerprint' | 'tailBytes'> | undefined, options: {
129
+ readonly tailBytes?: number;
130
+ readonly maxIndexBytes?: number;
131
+ }): number;
132
+ /**
133
+ * Send the tail read an open of `key` makes, now, and hand back its answer for {@link openChecked}'s `early`: the open
134
+ * then waits on the one request instead of sending its own, so work the open waits on first, such as a keystore call,
135
+ * runs beside the request and not before it. Nothing is parsed here: the bytes are untrusted until the open reads them.
136
+ * The returned promise already has a handler attached, so a caller that never reaches the open (its key failed) leaves
137
+ * no unhandled rejection; the open that does reach it still sees the rejection.
138
+ */
139
+ export declare function startTailRead(storage: IStorageDriver, key: GenKey, options: CrbmReaderOptions): Promise<TailRead>;
140
+ /**
141
+ * Re-open a freshly written generation and assert it round-trips exactly what was streamed into it: the same
142
+ * per-chunk key set *and* the same total cardinality (on top of the codec's own per-chunk CRC + footer checks), and the
143
+ * metadata it was given. `expected` is the streaming writer's tally (the stream is consumed, so it can't be
144
+ * re-iterated) — the key-set comparison catches a dropped/extra chunk that a cardinality-only check could miss when
145
+ * two errors cancel out. The count and the metadata are what the generation's row summary is about to claim, so the
146
+ * summary is held to the object it describes before it is published.
147
+ * Throws {@link IntegrityError}: the object is on disk but must not be published.
148
+ */
149
+ export declare function verifyGeneration(storage: IStorageDriver, key: GenKey, expected: {
150
+ readonly chunkKeys: readonly number[];
151
+ readonly cardinality: number;
152
+ readonly metadata?: GenerationMetadata | undefined;
153
+ }, crypto: CrbmCrypto | undefined): Promise<void>;
154
+ export {};