@cloudbitmaps/core 0.21.0 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/README.md +23 -8
  2. package/dist/chunk-7GT632UF.js +641 -0
  3. package/dist/chunk-7GT632UF.js.map +7 -0
  4. package/dist/chunk-CGILDXGP.js +73 -0
  5. package/dist/chunk-CGILDXGP.js.map +7 -0
  6. package/dist/chunk-GKHERVVQ.js +121 -0
  7. package/dist/chunk-GKHERVVQ.js.map +7 -0
  8. package/dist/{chunk-IGLIZBR7.js → chunk-MVUD4R72.js} +253 -765
  9. package/dist/chunk-MVUD4R72.js.map +7 -0
  10. package/dist/core/audit.d.ts +53 -15
  11. package/dist/core/bit-route.d.ts +9 -0
  12. package/dist/core/blob.d.ts +4 -0
  13. package/dist/core/budget.d.ts +8 -3
  14. package/dist/core/check.d.ts +27 -0
  15. package/dist/core/codec.d.ts +1 -1
  16. package/dist/core/combine-expr.d.ts +9 -5
  17. package/dist/core/combine-feed.d.ts +11 -3
  18. package/dist/core/combine-many.d.ts +28 -13
  19. package/dist/core/consistency.d.ts +11 -2
  20. package/dist/core/cooperative.d.ts +7 -0
  21. package/dist/core/crbm-storage-source.d.ts +7 -4
  22. package/dist/core/crypto.d.ts +5 -0
  23. package/dist/core/determinism.d.ts +2 -1
  24. package/dist/core/discover.d.ts +5 -3
  25. package/dist/core/engine.d.ts +4 -4
  26. package/dist/core/erase-id.d.ts +52 -19
  27. package/dist/core/erasure.d.ts +62 -28
  28. package/dist/core/errors.d.ts +83 -9
  29. package/dist/core/leases.d.ts +2 -0
  30. package/dist/core/load-input.d.ts +48 -0
  31. package/dist/core/load.d.ts +71 -16
  32. package/dist/core/metrics.d.ts +22 -1
  33. package/dist/core/pinned-storage-source.d.ts +27 -8
  34. package/dist/core/pointer-id.d.ts +9 -0
  35. package/dist/core/ports.d.ts +107 -13
  36. package/dist/core/preflight.d.ts +80 -0
  37. package/dist/core/registry-scan.d.ts +0 -5
  38. package/dist/core/retention-sweep.d.ts +61 -16
  39. package/dist/core/retention.d.ts +18 -7
  40. package/dist/core/retry.d.ts +5 -2
  41. package/dist/core/rollback.d.ts +23 -12
  42. package/dist/core/validate.d.ts +33 -2
  43. package/dist/driver-kit.d.ts +9 -4
  44. package/dist/driver-kit.js +63 -27
  45. package/dist/driver-kit.js.map +2 -2
  46. package/dist/drivers/_shared/object-registry.d.ts +24 -1
  47. package/dist/drivers/_shared/options.d.ts +15 -7
  48. package/dist/drivers/backends.d.ts +19 -2
  49. package/dist/drivers/crypto.d.ts +18 -3
  50. package/dist/export/index.d.ts +25 -10
  51. package/dist/index.d.ts +19 -17
  52. package/dist/index.js +2457 -2283
  53. package/dist/index.js.map +4 -4
  54. package/dist/node.d.ts +14 -0
  55. package/dist/node.js +1064 -0
  56. package/dist/node.js.map +7 -0
  57. package/package.json +16 -2
  58. package/dist/chunk-IGLIZBR7.js.map +0 -7
  59. package/dist/core/absent-operands.d.ts +0 -8
  60. package/dist/core/bulk-load.d.ts +0 -148
  61. package/dist/core/cas-retry.d.ts +0 -27
  62. package/dist/core/chunk-checks.d.ts +0 -39
  63. package/dist/core/chunk-stream.d.ts +0 -22
  64. package/dist/core/chunk-window.d.ts +0 -37
  65. package/dist/core/combine-ledger.d.ts +0 -43
  66. package/dist/core/crbm/crc32c.d.ts +0 -10
  67. package/dist/core/crbm/fingerprint.d.ts +0 -23
  68. package/dist/core/crbm/format.d.ts +0 -150
  69. package/dist/core/crbm/frame.d.ts +0 -10
  70. package/dist/core/crbm/plan-reads.d.ts +0 -50
  71. package/dist/core/crbm/varint.d.ts +0 -14
  72. package/dist/core/crbm/writer.d.ts +0 -72
  73. package/dist/core/crbm-open.d.ts +0 -154
  74. package/dist/core/crbm-write.d.ts +0 -75
  75. package/dist/core/due-index.d.ts +0 -53
  76. package/dist/core/generation-gc.d.ts +0 -293
  77. package/dist/core/generation-list.d.ts +0 -11
  78. package/dist/core/id-window.d.ts +0 -46
  79. package/dist/core/invalidation-scope.d.ts +0 -77
  80. package/dist/core/item-pull.d.ts +0 -29
  81. package/dist/core/known-generations-shape.d.ts +0 -13
  82. package/dist/core/known-generations.d.ts +0 -147
  83. package/dist/core/ledger-seed.d.ts +0 -17
  84. package/dist/core/metadata.d.ts +0 -42
  85. package/dist/core/order.d.ts +0 -2
  86. package/dist/core/publish.d.ts +0 -207
  87. package/dist/core/reader-defaults.d.ts +0 -29
  88. package/dist/core/summary.d.ts +0 -110
  89. package/dist/core/token.d.ts +0 -41
  90. package/dist/drivers/_shared/entropy.d.ts +0 -5
  91. package/dist/drivers/_shared/registry.d.ts +0 -107
  92. package/dist/drivers/localfs/exact-case.d.ts +0 -34
  93. package/dist/drivers/localfs/fs-util.d.ts +0 -55
  94. package/dist/drivers/localfs/paths.d.ts +0 -40
  95. package/dist/drivers/localfs/registry.d.ts +0 -50
  96. package/dist/drivers/localfs/storage.d.ts +0 -28
  97. package/dist/drivers/memory.d.ts +0 -64
  98. package/dist/testing/fuzz-core.d.ts +0 -15
@@ -73,10 +73,13 @@ export type AuditEvent = {
73
73
  * segment's registry row changed while the load was writing). `cardinality` is what the refused generation
74
74
  * would have contained, and `0` for a load that lost its generation number and wrote nothing.
75
75
  *
76
- * `unanswered` is present, and `true`, on a `'superseded'` refusal whose registry write ended without an answer
77
- * before the row was found to have moved on: that write may have landed, so the generation may have been the
78
- * segment's current one for a while before another writer replaced it. The refusal says only that it is not
79
- * current now. Absent when every write the load made was answered.
76
+ * `unanswered` is present, and `true`, on a `'superseded'` refusal after a registry write whose answer did not
77
+ * say whether it landed: one that ended without an answer, or a conflict its registry marked as one that may
78
+ * follow the write's own landing (Azure Blob marks every conflict of a row write, since its client may have sent
79
+ * the write twice). That write may have landed, so the generation may have been the segment's current one for a
80
+ * while before another write replaced it, or, where a write renewed the row's `pointerId` over it, may be current
81
+ * still. Absent when every write was answered and every conflict was one its registry vouches the write did not
82
+ * land.
80
83
  */
81
84
  readonly kind: 'segment.load-refused';
82
85
  readonly namespace?: string;
@@ -115,11 +118,12 @@ export type AuditEvent = {
115
118
  * not hold, or an encrypted one under a cleartext row (a first load's that lost the race, or crashed). The one
116
119
  * emitter is `eraseIdFromSegment`, once a listing of the bucket shows nothing left holding the id, so the record
117
120
  * exists only for a finished erasure. A call that ends otherwise throws or reports `erased: false` with another
118
- * reason and emits nothing, even when it had deleted a holder; its re-run emits when it finishes, and reports
119
- * `'not-member'` with no event when another collection deleted the last holder meanwhile, as its ledger entry
120
- * does. One finished erasure emits it and then throws `NotFoundError`: one that found, after its deletes, the
121
- * segment's pointer on a generation it had deleted (a rollback moved it there meanwhile), so the deletion is on
122
- * the record although the call cannot answer.
121
+ * reason and emits nothing, even when it had deleted a holder; its re-run emits when it finishes, and, when
122
+ * another collection deleted the last holder meanwhile, emits nothing and answers `'not-member'`, which
123
+ * `eraseSubject`'s ledger shows by listing nothing for the segment. One finished erasure emits it and then
124
+ * throws `NotFoundError`: one that found, after its deletes, the segment's pointer on a generation it had
125
+ * deleted (a rollback moved it there meanwhile), so the deletion is on the record although the call cannot
126
+ * answer.
123
127
  *
124
128
  * `fromGeneration` is the newest generation the call found holding the id, and is absent when it found the id in
125
129
  * none and deleted only objects it could not search: then the ledger lists no entry for the segment, and this
@@ -137,24 +141,52 @@ export type AuditEvent = {
137
141
  readonly collected: readonly number[];
138
142
  } | {
139
143
  /**
140
- * A segment was **crypto-shredded** — its wrapped DEK(s) are gone, so its at-rest Storage bytes are now
141
- * permanently unreadable. Emitted only for a genuine key shred, never for a cleartext tombstone (which
142
- * leaves the Storage bytes readable) or an idempotent re-run.
144
+ * A segment was **crypto-shredded**: its wrapped DEK(s) are gone from the segment's **current** registry row,
145
+ * so nothing that opens the segment through that row can decrypt its Storage bytes, in the bucket or in any
146
+ * backup, replica or WORM copy of its objects. It destroys no KEK: a noncurrent version, a backup or a
147
+ * point-in-time copy of the row still holds the wrapped DEK(s), and a registry restore to a point before the
148
+ * shred makes the segment readable again wherever its objects survive. The destruction is complete once no
149
+ * retained copy of the row holds them, or once every KEK that wrapped them is destroyed. A process that had
150
+ * already opened the segment keeps its unwrapped key in memory until told (`store.invalidate(ref)`). Emitted
151
+ * only for a genuine key shred, never for a cleartext tombstone (which leaves the Storage bytes readable) or
152
+ * an idempotent re-run.
143
153
  */
144
154
  readonly kind: 'segment.erase';
145
155
  readonly namespace?: string;
146
156
  readonly segment: string;
147
157
  /** The incarnation of the row the shred turned into a tombstone. */
148
158
  readonly incarnation?: string;
159
+ } | {
160
+ /**
161
+ * A segment was tombstoned **without a key shred**: `destroySegment` or `eraseNamespace` with `allowCleartext`
162
+ * turned the row of a segment that holds no wrapped key into a `destroyed` tombstone. No read resolves the
163
+ * segment any more, but nothing was deleted and no key was destroyed, so its objects stay readable in the
164
+ * bucket, in any backup and through any copy of the row from before the tombstone.
165
+ *
166
+ * Deliberately not {@link AuditEvent} `segment.erase`, which attests a key shred, nor `segment.dispose`, which
167
+ * attests that a drop swept the storage: this one is the record that the row was tombstoned, which is what a
168
+ * registry restore undoes and what the restore procedure looks for. `objectsReadable` is always `true`; it
169
+ * is a field so a consumer that forwards events cannot read this one as an erasure receipt. Emitted once per
170
+ * tombstone this call wrote, not for the idempotent re-run, and not for a segment that was encrypted
171
+ * (`segment.erase`), or dropped (`segment.dispose`).
172
+ */
173
+ readonly kind: 'segment.tombstone';
174
+ readonly namespace?: string;
175
+ readonly segment: string;
176
+ /** The incarnation of the row the call turned into a tombstone. */
177
+ readonly incarnation?: string;
178
+ /** Always `true`: the segment's objects were not deleted and no key was destroyed. */
179
+ readonly objectsReadable: true;
149
180
  } | {
150
181
  /**
151
182
  * A segment was **disposed of** — tombstoned and its storage reclaimed (its Storage generations deleted) by
152
183
  * `dropSegment`, *without* a key shred.
153
184
  *
154
185
  * Deliberately a separate kind from {@link AuditEvent} `segment.erase`, and the distinction is the point.
155
- * `segment.erase` attests that bytes are unreadable **everywhere, backups included** — the only claim that
156
- * survives WORM. Deleting an object is strictly weaker: a noncurrent version, a cross-region replica or a
157
- * PITR snapshot can still hold the cleartext. Emitting one kind for both would make a compliance dashboard
186
+ * `segment.erase` attests that the current row no longer holds the key, which is the one erasure claim that
187
+ * reaches WORM copies of the objects, once no retained copy of the row holds it. Deleting an object is
188
+ * strictly weaker: a noncurrent version, a cross-region replica or a PITR snapshot can still hold the
189
+ * cleartext. Emitting one kind for both would make a compliance dashboard
158
190
  * over-attest, so a cleartext drop gets this instead.
159
191
  *
160
192
  * An **encrypted** segment dropped via `dropSegment` emits **both** — `segment.erase` for the key shred and
@@ -178,6 +210,10 @@ export type AuditEvent = {
178
210
  };
179
211
  /** Sink for {@link AuditEvent}s. Injected via the lifecycle options; omit it and nothing is recorded. */
180
212
  export interface IAuditSink {
213
+ /**
214
+ * Receives one audit event, called synchronously by the operation that caused it. What it throws, or rejects with, is
215
+ * swallowed and never breaks that operation.
216
+ */
181
217
  onEvent(event: AuditEvent): void;
182
218
  }
183
219
  /** The default sink: records nothing. */
@@ -200,8 +236,10 @@ export declare function ignoreRejection(returned: unknown): void;
200
236
  /** A ready-made sink that records events into an in-memory list — handy for tests + simple audit trails. */
201
237
  export declare class RecordingAuditSink implements IAuditSink {
202
238
  private readonly recorded;
239
+ /** Records the event, in emission order. */
203
240
  onEvent(event: AuditEvent): void;
204
241
  /** An independent copy of the recorded events, in emission order. */
205
242
  snapshot(): AuditEvent[];
243
+ /** Forgets every recorded event. */
206
244
  reset(): void;
207
245
  }
@@ -1,3 +1,8 @@
1
+ /**
2
+ * BitRouteSplitter — maps a 32-bit id to its chunk (top 16 bits) and remainder
3
+ * (bottom 16 bits), per the data model.
4
+ */
5
+ import { ValidationError } from './errors.js';
1
6
  export declare const U32_MAX = 4294967295;
2
7
  export declare const CHUNK_COUNT = 65536;
3
8
  interface BitRoute {
@@ -6,6 +11,10 @@ interface BitRoute {
6
11
  /** Bottom 16 bits — the value within the chunk (0..65535). */
7
12
  readonly remainder: number;
8
13
  }
14
+ /** Was `err` thrown by {@link splitId} for an id outside the u32 range? */
15
+ export declare function isBadIdError(err: unknown): err is ValidationError;
16
+ /** The refusal {@link splitId} throws for a value that is not a u32 id, for a caller that checks the range itself. */
17
+ export declare function refusalOfId(id: unknown): ValidationError;
9
18
  /** Split a u32 id into `{ chunkKey, remainder }`. Throws `ValidationError` for non-u32 input. */
10
19
  export declare function splitId(id: number): BitRoute;
11
20
  /**
@@ -3,6 +3,10 @@
3
3
  * the write resolves, so a sink that holds bytes past that copies them.
4
4
  */
5
5
  export interface BlobSink {
6
+ /**
7
+ * Appends `bytes` to the object. Once it resolves the caller may reuse the buffer, so a sink that keeps the bytes
8
+ * copies them.
9
+ */
6
10
  write(bytes: Uint8Array): Promise<void>;
7
11
  }
8
12
  /**
@@ -15,6 +15,11 @@
15
15
  * search generation by generation is also charged one unit for each generation it opens beyond the current one,
16
16
  * since that is O(generations) work the segment count alone does not show.
17
17
  */
18
+ /**
19
+ * The cap on one operation's fan-out: its `maxRequests` units, counted in storage chunk fetches for a read and in
20
+ * segments scanned for an admin scan. `eraseSubject` also charges one unit for each generation beyond the current one
21
+ * that it opens, in a segment it has to search generation by generation.
22
+ */
18
23
  export interface Budget {
19
24
  /** Max units of fan-out (Storage chunk fetches, or segments scanned) one op may issue before it is refused. */
20
25
  readonly maxRequests: number;
@@ -30,13 +35,13 @@ export type BudgetOption = Partial<Budget> | false;
30
35
  /**
31
36
  * Resolve a budget option against a fallback to a concrete `Budget`, or `null` when disabled. Validates: anything but
32
37
  * `false`, an absent value or an object of `{ maxRequests }` is refused, since a cap of the wrong shape (`5`,
33
- * `{ maxRequest: 5 }`) would otherwise read as no override and leave the fallback in force. `null` and a key whose value
34
- * is `undefined` read as absent.
38
+ * `{ maxRequest: 5 }`) would otherwise read as no override and leave the fallback in force. A key whose value is `undefined` reads
39
+ * as absent; a `null` budget and a `maxRequests` of `null` are refused.
35
40
  */
36
41
  export declare function resolveBudget(opt: BudgetOption | undefined, fallback: Budget): Budget | null;
37
42
  /**
38
43
  * Resolve a **per-op** budget override against the store's own resolved budget (never the raw global default):
39
- * `undefined` or `null` inherits the store budget as-is (including a disabled `null`); a partial `{}` / omitted
44
+ * `undefined` inherits the store budget as-is (including a disabled `null`); a partial `{}` / omitted
40
45
  * `maxRequests` inherits the store's *tightening* (so a per-op passthrough of an absent config value can't
41
46
  * silently lift a tight tenant ceiling back to the generous default); `{ maxRequests }` replaces it; `false`
42
47
  * disables. Falls back to {@link DEFAULT_BUDGET} only when the store itself has no budget (disabled).
@@ -1,3 +1,8 @@
1
+ /**
2
+ * A value as a message shows it: a number or boolean as itself, a bigint with an `n` (so `5n` is not mistaken for
3
+ * `5`), a string quoted (so `'200'` is not mistaken for `200`), and anything else by its type.
4
+ */
5
+ export declare function shown(value: unknown): string;
1
6
  /**
2
7
  * `value` if it is a safe integer from `min` to `max` (inclusive; `max` defaults to the largest safe integer), else a
3
8
  * `ValidationError` naming `what`: `<what> must be an integer from <min> to <max>; got <value>`.
@@ -8,6 +13,11 @@ export declare function requireInteger(value: unknown, bounds: {
8
13
  readonly what: string;
9
14
  readonly note?: string;
10
15
  }): number;
16
+ /**
17
+ * Refuse a `requireEncryption` that is present and not exactly a boolean. Every site that acts on the flag compares it
18
+ * with `=== true`, so a truthy string would be taken for `true` by one and for `false` by another.
19
+ */
20
+ export declare function checkRequireEncryption(value: unknown): void;
11
21
  /** `value` if it is a safe integer of 1 or more, else a `ValidationError` naming `what`. */
12
22
  export declare function requirePositiveInteger(value: unknown, what: string): number;
13
23
  /**
@@ -20,3 +30,20 @@ export declare function requireFiniteNumber(value: unknown, bounds: {
20
30
  readonly what: string;
21
31
  readonly note?: string;
22
32
  }): number;
33
+ /**
34
+ * Refuse an options bag that is not an object, or that holds a key not in `keys`, naming each such key and the keys it
35
+ * takes. `hint`, when given, ends the message, to point at where a likely mistake belongs (`an S3 client goes in
36
+ * \`client\``). Only the bag's own enumerable string keys are read.
37
+ */
38
+ export declare function refuseUnknownOptions(name: string, options: unknown, keys: readonly string[], hint?: string): void;
39
+ /**
40
+ * The option `name`'s `value`, refused with `ValidationError` unless it is absent or a boolean: a truthy string or a
41
+ * `1` is not taken for `true`, and `null` is not taken for absent. An absent value comes back `undefined`, for the
42
+ * caller's default.
43
+ */
44
+ export declare function requireBool(name: string, value: unknown): boolean | undefined;
45
+ /**
46
+ * The option `name`'s `value`, refused with `ValidationError` unless it is absent or a string with something in it
47
+ * (one that is empty or only whitespace is refused, as is `null`). An absent value comes back `undefined`.
48
+ */
49
+ export declare function requireNonEmptyString(name: string, value: unknown): string | undefined;
@@ -103,7 +103,7 @@ export interface CodecBitmap {
103
103
  /**
104
104
  * This set as storage chunks: ascending by chunk key, empty chunks left out, each payload exactly the bytes that
105
105
  * {@link CodecInterface.fromValues} of that chunk's low 16 bits, then {@link optimize}, then {@link serialize}
106
- * would give. So a generation written from these is byte for byte the one the same ids write. It may re-encode
106
+ * would give. So a generation written from these holds chunk for chunk what the same ids write. It may re-encode
107
107
  * this bitmap in place first, as `optimize()` does; membership is untouched.
108
108
  *
109
109
  * **Flavor-author surface, and optional.** A load from a bitmap or from serialized bytes writes these, never
@@ -1,10 +1,14 @@
1
- /** A string names an operand; a node holds exactly one operator over a non-empty list of expressions. */
2
- export type CombineExpr = string | {
3
- and: CombineExpr[];
1
+ /**
2
+ * An expression over the operands of a batch combine (a store's `materializeMany`): a string names an operand; a node
3
+ * holds exactly one of `and`, `or` or `andNot` over a non-empty list of expressions. `andNot` subtracts every entry after
4
+ * the first from the first, so it takes at least two. Plain JSON, so it can be logged and sent between tasks.
5
+ */
6
+ export type Expr = string | {
7
+ and: Expr[];
4
8
  } | {
5
- or: CombineExpr[];
9
+ or: Expr[];
6
10
  } | {
7
- andNot: CombineExpr[];
11
+ andNot: Expr[];
8
12
  };
9
13
  /** The most operator nodes one expression may nest. A literal that contains itself is refused by it. */
10
14
  export declare const MAX_EXPR_DEPTH = 64;
@@ -1,6 +1,14 @@
1
1
  import type { CodecBitmap, CodecInterface } from './codec.js';
2
- /** One record of a feed: the ids each named operand holds at one chunk key. */
3
- export interface CombineManyFeedRecord {
2
+ /**
3
+ * One record of the feed of a batch combine (a store's `materializeMany`): the ids some of the fed operands hold at one
4
+ * chunk key. `key` is `id >>> 16`, an integer from 0 to 65,535, never below the previous record's; a key may arrive as
5
+ * several consecutive records, each carrying some of the operands, and one record per key is the usual shape. `operands`
6
+ * maps a declared fed name to its ids at the key: a real `Uint32Array` (not an `Int32Array`, `Buffer`, array or
7
+ * look-alike), strictly ascending, every id inside the key. A name is given at most once per key, and an empty array is
8
+ * the same as leaving the name out. The record is copied as it arrives, so the caller may drop or reuse its arrays once
9
+ * it has yielded it.
10
+ */
11
+ export interface MaterializeManyFeedRecord {
4
12
  /** The chunk key, `id >>> 16`, from 0 to 65,535. Non-decreasing over the feed; a key may arrive as several records. */
5
13
  readonly key: number;
6
14
  /** Ids by operand name: ascending, unique, every one inside `key`. An operand is named at most once per key. */
@@ -11,7 +19,7 @@ export interface CombineManyFeed {
11
19
  /** The fed operand names, declared up front. */
12
20
  readonly names: readonly string[];
13
21
  /** The records, read once, in key order. */
14
- readonly records: AsyncIterable<CombineManyFeedRecord>;
22
+ readonly records: AsyncIterable<MaterializeManyFeedRecord>;
15
23
  /**
16
24
  * The ids each fed operand holds over the whole feed, keyed by exactly `names`: an object, or a function called once
17
25
  * after the last record. A count that differs from what the feed held refuses every fed output.
@@ -1,14 +1,14 @@
1
1
  import type { CodecBitmap, CodecInterface } from './codec.js';
2
2
  import type { IMetricsSink } from './metrics.js';
3
3
  import type { CombineManyFeed } from './combine-feed.js';
4
- import type { CombineExpr } from './combine-expr.js';
4
+ import type { Expr } from './combine-expr.js';
5
5
  import type { Clock } from './determinism.js';
6
6
  import type { Budget } from './budget.js';
7
7
  import type { LoadGuard } from './load.js';
8
8
  import type { GenerationMetadata, SegmentRef, StorageChunkSource } from './ports.js';
9
- export type { CombineExpr } from './combine-expr.js';
9
+ export type { Expr } from './combine-expr.js';
10
10
  export { MAX_EXPR_DEPTH, MAX_EXPR_NODES } from './combine-expr.js';
11
- export type { CombineManyFeed, CombineManyFeedRecord } from './combine-feed.js';
11
+ export type { CombineManyFeed, MaterializeManyFeedRecord } from './combine-feed.js';
12
12
  /** The range requests a pass keeps in flight at once across all its operands, inside a client's socket pool. */
13
13
  export declare const MAX_RANGES_IN_FLIGHT = 64;
14
14
  /** One named stored operand, as the pass reads it. */
@@ -48,9 +48,9 @@ export interface CombineManyOperand {
48
48
  }
49
49
  /** What one output asks for. */
50
50
  export interface CombineManyOutput<R> {
51
- readonly expr: CombineExpr;
51
+ readonly expr: Expr;
52
52
  /** Subtracted from the result: a root `andNot`. */
53
- readonly exclude?: readonly CombineExpr[];
53
+ readonly exclude?: readonly Expr[];
54
54
  readonly allowEmpty?: boolean;
55
55
  readonly guard?: LoadGuard;
56
56
  readonly metadata?: GenerationMetadata;
@@ -126,8 +126,8 @@ export type CombineManyOutcome<R> = {
126
126
  readonly ok: false;
127
127
  readonly error: unknown;
128
128
  };
129
- /** What the call did to one operand. */
130
- export interface CombineManyOperandStats {
129
+ /** What a batch combine (a store's `materializeMany`) did to one operand: see {@link MaterializeManyStats.operands}. */
130
+ export interface MaterializeManyOperandStats {
131
131
  /** Whether an output of the call read it. An operand no output names is not read. */
132
132
  readonly read: boolean;
133
133
  /** Whether the operand was read at a pinned generation. */
@@ -155,14 +155,18 @@ export interface CombineManyOperandStats {
155
155
  /** Chunk reads made of it, over all groups. */
156
156
  readonly chunkReads: number;
157
157
  }
158
- /** What the call did to one output. */
159
- export interface CombineManyOutputStats {
158
+ /** What a batch combine (a store's `materializeMany`) did to one output: see {@link MaterializeManyStats.outputs}. */
159
+ export interface MaterializeManyOutputStats {
160
160
  /** The names of the operands its expression reads: the stored ones in call order, then the fed ones in `feed.names` order. */
161
161
  readonly operands: readonly string[];
162
162
  /** The group that computed it (from 0), or `null` when it never ran. */
163
163
  readonly group: number | null;
164
164
  /** Clock time the group's pass began and the output's publish settled (or it failed), when it ran. */
165
165
  readonly startedAt: number | null;
166
+ /**
167
+ * Clock time (ms) when its publish settled or failed, or `null` when the output never ran. Read from the same clock
168
+ * as `startedAt`.
169
+ */
166
170
  readonly endedAt: number | null;
167
171
  /** Chunks the output holds. */
168
172
  readonly chunks: number;
@@ -171,9 +175,14 @@ export interface CombineManyOutputStats {
171
175
  /** The most its buffer held resident, in the ledger's own measure; compare with `boundBytes` to see how tight a plan was. */
172
176
  readonly peakBytes: number;
173
177
  }
174
- export interface CombineManyStats {
178
+ /**
179
+ * What a batch combine (a store's `materializeMany`) did: the groups it ran, its requests by class, the chunk-read budget
180
+ * it used, the most it held resident, and, for each operand, the generation it was pinned at, began at and ended at.
181
+ */
182
+ export interface MaterializeManyStats {
175
183
  /** Groups run, the re-runs of outputs a lying index made too big included. */
176
184
  readonly groups: number;
185
+ /** The requests the call made, by class. */
177
186
  readonly requests: {
178
187
  /** Range requests sent for operand chunks (retries of one request count once). */
179
188
  readonly rangeReads: number;
@@ -205,6 +214,7 @@ export interface CombineManyStats {
205
214
  readonly put: number;
206
215
  };
207
216
  };
217
+ /** The chunk-read budget: the limit in force, the reads the plan needed and the reads made. */
208
218
  readonly budget: {
209
219
  /** The limit used: `null` when lifted. */
210
220
  readonly maxRequests: number | null;
@@ -213,6 +223,9 @@ export interface CombineManyStats {
213
223
  /** Chunk reads made. */
214
224
  readonly used: number;
215
225
  };
226
+ /**
227
+ * The memory the call held: its limit, the most the ledger held at once and what the plan holds for the whole call.
228
+ */
216
229
  readonly memory: {
217
230
  readonly maxBufferedBytes: number;
218
231
  /** The most the ledger held at once. */
@@ -220,6 +233,7 @@ export interface CombineManyStats {
220
233
  /** What the plan holds for the whole call, charged to the ledger: operand indexes, and each output's keys and expression. */
221
234
  readonly planBytes: number;
222
235
  };
236
+ /** How many chunk keys the call skipped reading. */
223
237
  readonly chunks: {
224
238
  /** Index keys not read because no output needed them there, summed over groups. */
225
239
  readonly pruned: number;
@@ -233,12 +247,13 @@ export interface CombineManyStats {
233
247
  readonly ids: number;
234
248
  };
235
249
  /** By operand name, an object with no prototype, so any name is a key. */
236
- readonly operands: Readonly<Record<string, CombineManyOperandStats>>;
237
- readonly outputs: readonly CombineManyOutputStats[];
250
+ readonly operands: Readonly<Record<string, MaterializeManyOperandStats>>;
251
+ /** One entry per output of the call, in the order the call gave them. */
252
+ readonly outputs: readonly MaterializeManyOutputStats[];
238
253
  }
239
254
  export interface CombineManyRun<R> {
240
255
  readonly outputs: readonly CombineManyOutcome<R>[];
241
- readonly stats: CombineManyStats;
256
+ readonly stats: MaterializeManyStats;
242
257
  }
243
258
  /** A request that passed every check that needs no I/O: what {@link runCombineMany} runs. */
244
259
  export interface CompiledCombineMany<R> {
@@ -7,8 +7,8 @@
7
7
  * the affected segment then throw. This scan
8
8
  * detects it up front (run it at startup after a restore) instead of discovering it on the first read.
9
9
  *
10
- * Read-only; bounded fan-out. `destroyed` (crypto-shredded) segments are skipped — their Storage is intentionally
11
- * gone/unreadable, not a torn restore. A segment whose Storage/registry can't be read this pass is recorded in
10
+ * Read-only; bounded fan-out. `destroyed` segments are skipped: no generation resolves through one, so a missing
11
+ * object under it is not a torn restore. A segment whose Storage/registry can't be read this pass is recorded in
12
12
  * `errored` (never aborts the scan). One segment can add several entries to `inconsistent` (its current object, and with
13
13
  * `generations` one per known generation whose object is not the one recorded) and can also be in `errored`, when a
14
14
  * read failed after those were found; with `generations` a segment costs up to one read per entry of its ledger.
@@ -35,8 +35,11 @@ import type { IKeystore } from './crypto.js';
35
35
  * something a modest operator box survives. Raisable, because a ceiling you cannot lift is a landmine.
36
36
  */
37
37
  export { DEFAULT_MAX_SCAN_SEGMENTS } from './registry-scan.js';
38
+ /** One problem the consistency check found with one segment; `issue` says which. */
38
39
  export interface ConsistencyIssue {
40
+ /** The segment's name. */
39
41
  readonly segment: string;
42
+ /** The segment's namespace, or `undefined` for the default one. */
40
43
  readonly namespace?: string;
41
44
  /**
42
45
  * The registry's `currentGen` for the segment. For `missing-storage-generation` it is the generation whose `.crbm` is
@@ -70,11 +73,17 @@ export interface ConsistencyIssue {
70
73
  }
71
74
  /** A segment that could not be checked this pass (Storage/registry read fault) — not proof of a torn restore. */
72
75
  export interface ConsistencyErrorEntry {
76
+ /** The name of the segment that could not be checked. */
73
77
  readonly segment: string;
78
+ /** That segment's namespace, or `undefined` for the default one. */
74
79
  readonly namespace?: string;
75
80
  /** The read error's message (never the raw error, so the report stays serializable/loggable). */
76
81
  readonly error: string;
77
82
  }
83
+ /**
84
+ * What a consistency check found: how many segments it scanned, the issues, the segments it could not read, and, when
85
+ * asked for, the checks it could not make.
86
+ */
78
87
  export interface ConsistencyReport {
79
88
  /** Registered segments scanned (a `destroyed` segment counts as scanned but is never an issue). */
80
89
  readonly checked: number;
@@ -51,6 +51,13 @@ export interface Yielder {
51
51
  * (~60 yields across a 1M-id load) while still capping any single blocking stretch in the low tens of ms.
52
52
  */
53
53
  export declare const YIELD_EVERY = 1024;
54
+ /**
55
+ * Ids between a load's ingest-loop yields, sorted or not. 16x coarser than the per-chunk cadence because the per-id
56
+ * work is ~40 ns (a `splitId` and an array push) against ~1.3 µs per chunk — matching cadences would put the yield cost
57
+ * on the wrong side of the work it interrupts. Sized against the *slowest* ingest, an in-memory async generator at
58
+ * ~220 ns/id, so the resulting stretch stays a few ms there and well under 1 ms on a plain array.
59
+ */
60
+ export declare const YIELD_EVERY_IDS: number;
54
61
  /**
55
62
  * Build a periodic yield.
56
63
  *
@@ -377,7 +377,8 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
377
377
  * the one read path with no resilience.
378
378
  *
379
379
  * **`status` is re-checked here**, on every open rather than once at pin time. A pin taken before a
380
- * crypto-shred must not keep unwrapping a DEK the shred destroyed; a destroyed segment resolves no generation
380
+ * crypto-shred must not keep unwrapping a DEK whose wrapping the shred removed; a destroyed segment resolves no
381
+ * generation
381
382
  * for anyone, and a pin's read of one fails. That holds from the pin's next open: a reader already open keeps
382
383
  * the key it unwrapped until it is evicted or invalidated.
383
384
  *
@@ -419,7 +420,8 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
419
420
  /**
420
421
  * Whether the segment is a registered name at all. With a registry: a row that is not a `destroyed` tombstone,
421
422
  * including one with no generation yet (minted by `setRetention` before the first load), which somebody deliberately
422
- * created. A tombstone holds nothing, so it answers `false`. Without a registry there is no registration to
423
+ * created. No generation resolves through a tombstone, so it answers `false`. Without a registry there is no
424
+ * registration to
423
425
  * consult, so the honest answer is whether the bucket holds any generation of it.
424
426
  *
425
427
  * Deliberately NOT served from the snapshot memo: the memo resolves to `null` for both states this exists to
@@ -569,8 +571,9 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
569
571
  *
570
572
  * A *second* NotFound is pathological (GC outrunning resolution) and propagates rather than fabricating an
571
573
  * absent answer — never return a wrong result. `ifGone` is returned when the segment has no committed
572
- * generation to serve: storage is empty, or the row was dropped or crypto-shredded, in which case reading empty
573
- * is the documented outcome rather than a failure.
574
+ * generation to serve: storage is empty, or the row is destroyed (crypto-shredded, destroyed in the clear or
575
+ * dropped), in which case
576
+ * reading empty is the documented outcome rather than a failure.
574
577
  *
575
578
  * **Bounded to exactly two resolve-and-open round trips.** That is a cost contract, not a detail: the retry
576
579
  * re-reads the *registry*, so an unbounded one would hammer the shared, throttle-prone resource in a
@@ -8,8 +8,11 @@
8
8
  */
9
9
  /** One AEAD result: a fresh per-message nonce, the ciphertext, and the authentication tag. */
10
10
  export interface AeadSealed {
11
+ /** The fresh per-message nonce. */
11
12
  readonly nonce: Uint8Array;
13
+ /** The encrypted bytes. */
12
14
  readonly ciphertext: Uint8Array;
15
+ /** The authentication tag. */
13
16
  readonly tag: Uint8Array;
14
17
  }
15
18
  /**
@@ -36,7 +39,9 @@ export interface Aead {
36
39
  * plaintext key material — safe to store in the registry next to the (encrypted) data.
37
40
  */
38
41
  export interface WrappedDek {
42
+ /** The id of the key-encryption key (KEK) this wrapping is under. */
39
43
  readonly keyId: string;
44
+ /** The wrapped data key, an opaque base64 string. */
40
45
  readonly wrapped: string;
41
46
  }
42
47
  /**
@@ -42,7 +42,8 @@ export interface Rng {
42
42
  }
43
43
  /**
44
44
  * Injected entropy: `length` cryptographically strong random bytes, for an identity that must never repeat (a
45
- * registry row's incarnation id, and the random part every registry write adds to its token). Deliberately not an
45
+ * registry row's incarnation id, the random part every registry write adds to its token, and a generation's write
46
+ * identity, the 12 bytes a cleartext object's footer carries). Deliberately not an
46
47
  * {@link Rng}: an `Rng` is seedable for simulation, and a seeded source hands every process the same "random"
47
48
  * identity. Production wiring uses the platform's Web Crypto; a test injects a deterministic source to replay a run.
48
49
  */
@@ -20,9 +20,11 @@ export interface SegmentInfo extends SegmentRef {
20
20
  */
21
21
  readonly currentGen: number | null;
22
22
  /**
23
- * `destroyed` is a tombstone: the row stays so sweeps can still find it, and the generations are gone — on an
24
- * encrypted segment the DEK wrappings went with them, which is what makes the bytes unrecoverable rather than
25
- * merely deleted.
23
+ * `destroyed` is a tombstone: the row stays so sweeps can still find it, and no generation resolves through it. A
24
+ * `dropSegment` deletes the generations (check its `generationsRemaining`); a crypto-shred leaves them in the bucket
25
+ * and removes the DEK wrappings from the row, which makes every copy of the bytes unreadable once no retained copy of
26
+ * the row still holds them, or every KEK that wrapped them is destroyed. A cleartext segment destroyed with
27
+ * `allowCleartext` has no key to remove: its objects stay in the bucket and stay readable.
26
28
  */
27
29
  readonly status: RegistryRecord['status'];
28
30
  }
@@ -346,11 +346,11 @@ export declare class SegmentEngine {
346
346
  * for the range's edge, never a chunk the cache holds or another operand's. The reads, the budget, the metric and the
347
347
  * errors are the id reads' own.
348
348
  */
349
- intersectChunks(segs: readonly SegmentRef[], options?: CombineOptions): AsyncGenerator<CombinedChunk>;
349
+ intersectChunks(segs: readonly SegmentRef[], options?: CombineOptions, label?: string): AsyncGenerator<CombinedChunk>;
350
350
  /** {@link union}, as chunks: see {@link intersectChunks}. */
351
- unionChunks(segs: readonly SegmentRef[], options?: CombineOptions): AsyncGenerator<CombinedChunk>;
351
+ unionChunks(segs: readonly SegmentRef[], options?: CombineOptions, label?: string): AsyncGenerator<CombinedChunk>;
352
352
  /** {@link andNot}, as chunks: see {@link intersectChunks}. */
353
- andNotChunks(seg: SegmentRef, excludes: readonly SegmentRef[], options?: Omit<CombineOptions, 'exclude'>): AsyncGenerator<CombinedChunk>;
353
+ andNotChunks(seg: SegmentRef, excludes: readonly SegmentRef[], options?: Omit<CombineOptions, 'exclude'>, label?: string): AsyncGenerator<CombinedChunk>;
354
354
  /**
355
355
  * Everything a combine decides before it reads a chunk: validation, each operand's keys and generation, the
356
356
  * absent-operand refusal, key alignment, the budget and the metric. `null` when the range is empty. The per-id
@@ -546,7 +546,7 @@ export declare class SegmentEngine {
546
546
  * the read listed the segment's keys: the read then lists them again, and caches as a read begun after the call.
547
547
  *
548
548
  * The decoded-chunk cache is keyed by generation, which handles a *publish* (the new generation misses) but
549
- * not a *destruction*: a segment that was erased from, dropped, shredded or retired has no newer generation
549
+ * not a *destruction*: a segment that was erased from, dropped, destroyed or retired has no newer generation
550
550
  * whose key would miss, so its decoded chunks stay resident and keep answering. That is why the erasing
551
551
  * process itself could report `erased: true` and then still answer `true` for the same id, out of RAM, with
552
552
  * no storage read to intercept.