@cloudbitmaps/core 0.21.0 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/README.md +23 -8
  2. package/dist/chunk-7GT632UF.js +641 -0
  3. package/dist/chunk-7GT632UF.js.map +7 -0
  4. package/dist/chunk-CGILDXGP.js +73 -0
  5. package/dist/chunk-CGILDXGP.js.map +7 -0
  6. package/dist/chunk-GKHERVVQ.js +121 -0
  7. package/dist/chunk-GKHERVVQ.js.map +7 -0
  8. package/dist/{chunk-IGLIZBR7.js → chunk-MVUD4R72.js} +253 -765
  9. package/dist/chunk-MVUD4R72.js.map +7 -0
  10. package/dist/core/audit.d.ts +53 -15
  11. package/dist/core/bit-route.d.ts +9 -0
  12. package/dist/core/blob.d.ts +4 -0
  13. package/dist/core/budget.d.ts +8 -3
  14. package/dist/core/check.d.ts +27 -0
  15. package/dist/core/codec.d.ts +1 -1
  16. package/dist/core/combine-expr.d.ts +9 -5
  17. package/dist/core/combine-feed.d.ts +11 -3
  18. package/dist/core/combine-many.d.ts +28 -13
  19. package/dist/core/consistency.d.ts +11 -2
  20. package/dist/core/cooperative.d.ts +7 -0
  21. package/dist/core/crbm-storage-source.d.ts +7 -4
  22. package/dist/core/crypto.d.ts +5 -0
  23. package/dist/core/determinism.d.ts +2 -1
  24. package/dist/core/discover.d.ts +5 -3
  25. package/dist/core/engine.d.ts +4 -4
  26. package/dist/core/erase-id.d.ts +52 -19
  27. package/dist/core/erasure.d.ts +62 -28
  28. package/dist/core/errors.d.ts +83 -9
  29. package/dist/core/leases.d.ts +2 -0
  30. package/dist/core/load-input.d.ts +48 -0
  31. package/dist/core/load.d.ts +71 -16
  32. package/dist/core/metrics.d.ts +22 -1
  33. package/dist/core/pinned-storage-source.d.ts +27 -8
  34. package/dist/core/pointer-id.d.ts +9 -0
  35. package/dist/core/ports.d.ts +107 -13
  36. package/dist/core/preflight.d.ts +80 -0
  37. package/dist/core/registry-scan.d.ts +0 -5
  38. package/dist/core/retention-sweep.d.ts +61 -16
  39. package/dist/core/retention.d.ts +18 -7
  40. package/dist/core/retry.d.ts +5 -2
  41. package/dist/core/rollback.d.ts +23 -12
  42. package/dist/core/validate.d.ts +33 -2
  43. package/dist/driver-kit.d.ts +9 -4
  44. package/dist/driver-kit.js +63 -27
  45. package/dist/driver-kit.js.map +2 -2
  46. package/dist/drivers/_shared/object-registry.d.ts +24 -1
  47. package/dist/drivers/_shared/options.d.ts +15 -7
  48. package/dist/drivers/backends.d.ts +19 -2
  49. package/dist/drivers/crypto.d.ts +18 -3
  50. package/dist/export/index.d.ts +25 -10
  51. package/dist/index.d.ts +19 -17
  52. package/dist/index.js +2457 -2283
  53. package/dist/index.js.map +4 -4
  54. package/dist/node.d.ts +14 -0
  55. package/dist/node.js +1064 -0
  56. package/dist/node.js.map +7 -0
  57. package/package.json +16 -2
  58. package/dist/chunk-IGLIZBR7.js.map +0 -7
  59. package/dist/core/absent-operands.d.ts +0 -8
  60. package/dist/core/bulk-load.d.ts +0 -148
  61. package/dist/core/cas-retry.d.ts +0 -27
  62. package/dist/core/chunk-checks.d.ts +0 -39
  63. package/dist/core/chunk-stream.d.ts +0 -22
  64. package/dist/core/chunk-window.d.ts +0 -37
  65. package/dist/core/combine-ledger.d.ts +0 -43
  66. package/dist/core/crbm/crc32c.d.ts +0 -10
  67. package/dist/core/crbm/fingerprint.d.ts +0 -23
  68. package/dist/core/crbm/format.d.ts +0 -150
  69. package/dist/core/crbm/frame.d.ts +0 -10
  70. package/dist/core/crbm/plan-reads.d.ts +0 -50
  71. package/dist/core/crbm/varint.d.ts +0 -14
  72. package/dist/core/crbm/writer.d.ts +0 -72
  73. package/dist/core/crbm-open.d.ts +0 -154
  74. package/dist/core/crbm-write.d.ts +0 -75
  75. package/dist/core/due-index.d.ts +0 -53
  76. package/dist/core/generation-gc.d.ts +0 -293
  77. package/dist/core/generation-list.d.ts +0 -11
  78. package/dist/core/id-window.d.ts +0 -46
  79. package/dist/core/invalidation-scope.d.ts +0 -77
  80. package/dist/core/item-pull.d.ts +0 -29
  81. package/dist/core/known-generations-shape.d.ts +0 -13
  82. package/dist/core/known-generations.d.ts +0 -147
  83. package/dist/core/ledger-seed.d.ts +0 -17
  84. package/dist/core/metadata.d.ts +0 -42
  85. package/dist/core/order.d.ts +0 -2
  86. package/dist/core/publish.d.ts +0 -207
  87. package/dist/core/reader-defaults.d.ts +0 -29
  88. package/dist/core/summary.d.ts +0 -110
  89. package/dist/core/token.d.ts +0 -41
  90. package/dist/drivers/_shared/entropy.d.ts +0 -5
  91. package/dist/drivers/_shared/registry.d.ts +0 -107
  92. package/dist/drivers/localfs/exact-case.d.ts +0 -34
  93. package/dist/drivers/localfs/fs-util.d.ts +0 -55
  94. package/dist/drivers/localfs/paths.d.ts +0 -40
  95. package/dist/drivers/localfs/registry.d.ts +0 -50
  96. package/dist/drivers/localfs/storage.d.ts +0 -28
  97. package/dist/drivers/memory.d.ts +0 -64
  98. package/dist/testing/fuzz-core.d.ts +0 -15
@@ -1,6 +1,6 @@
1
1
  import { type IAuditSink } from './audit.js';
2
2
  import { type CodecBitmap, type CodecInterface } from './codec.js';
3
- import type { Clock, Rng } from './determinism.js';
3
+ import type { Clock, Entropy, Rng } from './determinism.js';
4
4
  import type { IKeystore } from './crypto.js';
5
5
  import { type LoadInput } from './load-input.js';
6
6
  import { type ReadRetry } from './retry.js';
@@ -18,6 +18,14 @@ export interface LoadDeps {
18
18
  * bound). Absent, the read retry's source is used if it has one, and otherwise each wait is its bound.
19
19
  */
20
20
  readonly rng?: Rng;
21
+ /**
22
+ * The source of the write's identity: 12 random bytes the new generation's footer carries (for a cleartext generation;
23
+ * an encrypted one carries its index's AEAD nonce), so two loads of the same ids at the same number are two objects
24
+ * and every check that a footer is "mine" names this write. Required: a load without one (a JavaScript caller can
25
+ * leave it out) is refused with {@link ValidationError} before any read, keystore call or write.
26
+ * `@cloudbitmaps/roaring` supplies Web Crypto's by default.
27
+ */
28
+ readonly entropy: Entropy;
21
29
  /**
22
30
  * The store's read retry, for the read the guard makes of the current generation: a transient fault there is run
23
31
  * again under it rather than failing the load. Absent, the read is made once. It does not govern the write: a
@@ -75,12 +83,20 @@ export interface LoadGuard {
75
83
  * `'outputs[3].'`).
76
84
  */
77
85
  export declare function checkedGuard(guard: LoadGuard | undefined, where: string): LoadGuard | undefined;
86
+ /**
87
+ * Options of `store.load` and of core's `loadSegment`: what a load may publish (`allowEmpty`, `guard`), what it keeps
88
+ * (`keep`) and what it records (`metadata`, `audit`).
89
+ */
78
90
  export interface LoadOptions {
79
91
  /**
80
92
  * Publish an empty generation over a non-empty one. Off by default: an empty result is far more often an
81
93
  * upstream failure than an intent, and the failure is indistinguishable from success once it lands.
82
94
  */
83
95
  readonly allowEmpty?: boolean;
96
+ /**
97
+ * Bounds a plausible result must meet, judged before the pointer moves; a load outside them is refused
98
+ * (`published: false`), not thrown. See {@link LoadGuard}.
99
+ */
84
100
  readonly guard?: LoadGuard;
85
101
  /**
86
102
  * Generations to keep below the new pointer as a grace window (default 1). A non-negative integer: anything
@@ -97,14 +113,17 @@ export interface LoadOptions {
97
113
  * load superseded comes first. After a rollback that is the one the rollback restored, and it stays in the window of
98
114
  * every load after it, ahead of the generations the rollback moved away from, for as many loads as `keep` says.
99
115
  *
100
- * The segment's row records the generations a load keeps, up to 64, so a load whose number one existence check found
101
- * free collects without listing: it deletes the generations its publish pushed out of the window, and lists the
102
- * segment's objects on every sixteenth generation to take whatever that pass leaves, such as an object a crashed load
103
- * left below the pointer. A `keep` above 64 collects by listing on every load, and so does the first load of a row with
104
- * no usable ledger, which starts one from the bucket; so does a load whose check found the current generation's object
105
- * gone: the guard's read of it, or, when the guard took the size from the row's summary, one zero-byte read made
106
- * before a `keep` of 1 or more takes a name. A `keep` at least the generation published collects nothing and asks
107
- * for nothing.
116
+ * The segment's row records the generations a load keeps, up to 127.
117
+ *
118
+ * With a `keep` of at most 64, a load whose number one existence check found free deletes by name the generations
119
+ * its publish pushed out of the window, and lists the segment's objects on every sixteenth generation to take
120
+ * whatever that pass leaves, such as an object a crashed load left below the pointer.
121
+ *
122
+ * A load lists instead on every load when `keep` is above 64, when the row has a pointer and no usable record (the
123
+ * load starts one from the bucket), when the row's record names more than 64 generations below the pointer that no
124
+ * lease holds (as after a `keep` above 64, or `materializeMany`), or when the guard's read, or the zero-byte read
125
+ * before a delete by name, finds the current generation's object missing. A publish that a read of the row settles
126
+ * after its last attempt lists too. A `keep` at least the generation published collects nothing and asks for nothing.
108
127
  */
109
128
  readonly keep?: number;
110
129
  /**
@@ -119,7 +138,28 @@ export interface LoadOptions {
119
138
  * over as it is. Sealed under the segment's key when the segment is encrypted.
120
139
  */
121
140
  readonly metadata?: GenerationMetadata;
141
+ /** Audit sink for this load: it receives `segment.publish`, or `segment.load-refused` when the load is refused. */
122
142
  readonly audit?: IAuditSink;
143
+ /**
144
+ * The ids ascend: write them as they are read, one chunk at a time, while the object uploads, so the load holds one
145
+ * chunk and the writer's index of the chunks written (up to about 7 MiB, about 8 MiB for an encrypted segment,
146
+ * derived, at the 65,536 chunks a segment can have) rather than the whole generation, whatever the segment's size.
147
+ * Repeats are fine (an id equal to the one before is skipped); an id lower than the one before throws
148
+ * `ValidationError`, naming its position and never the id. The object is the same, byte for byte, as an unsorted load
149
+ * of the same ids writes under the same write identity.
150
+ *
151
+ * For ids only (an iterable or an async iterable): with `{ serialized }`, `{ bitmap }` or a `RoaringBitmap32` of the
152
+ * flavor's own `roaring` (one from another copy loads as its ids, and takes `sorted`), and on core's
153
+ * `loadSegmentChunks`, it throws `ValidationError`, as does a value that is not a boolean, before any request.
154
+ *
155
+ * The ids are read after the segment's row is read and judged, and after its number and key are settled, so a load
156
+ * that refuses a destroyed or keyless segment reads none of them. They are read while the object uploads: an error
157
+ * they raise, or a refused id, fails the upload, and no object is committed, but the parts uploaded before it may
158
+ * already have been billed (see the production guide). That error reaches the caller as the very object raised,
159
+ * and the storage driver never sees it. The ids can be read once: a storage driver that runs its write again gets
160
+ * `IntegrityError`, and nothing is written.
161
+ */
162
+ readonly sorted?: boolean;
123
163
  }
124
164
  /** The bounds a guard can refuse a generation for: every {@link LoadRefusal} but a lost race. */
125
165
  export type GuardRefusal = Exclude<LoadRefusal, 'superseded'>;
@@ -141,7 +181,7 @@ export interface LoadJudgement {
141
181
  * would, and an encrypted one with no keystore the `KeyUnavailableError`. What it says holds for now only: a load
142
182
  * made later is judged against the segment as it is then.
143
183
  */
144
- export declare function judgeLoad(ref: SegmentRef, cardinality: number, deps: LoadDeps, options?: Pick<LoadOptions, 'allowEmpty' | 'guard'>): Promise<LoadJudgement>;
184
+ export declare function judgeLoad(ref: SegmentRef, cardinality: number, deps: Omit<LoadDeps, 'entropy'>, options?: Pick<LoadOptions, 'allowEmpty' | 'guard'>): Promise<LoadJudgement>;
145
185
  /** Why a load did not become current. */
146
186
  export type LoadRefusal = 'empty' | 'min-cardinality' | 'min-retained' | 'max-growth'
147
187
  /**
@@ -150,12 +190,17 @@ export type LoadRefusal = 'empty' | 'min-cardinality' | 'min-retained' | 'max-gr
150
190
  * writing: another load published, a retention change, a rollback or an erasure wrote the row, the row was
151
191
  * deleted, or a row appeared where the load found none. Once another write has changed the row, the object stays in
152
192
  * the bucket, still billed, until a later load's collection deletes it or the segment is dropped, because its
153
- * generation number may by then name another writer's object; until it goes, a rollback onto it needs `allowUnknown`, as onto any generation the row does not know. Once the row is
154
- * `destroyed` (dropped or crypto-shredded), the object is deleted; once the row is gone, so is an object whose footer
155
- * matches what this load wrote (a re-created segment's load of the same cleartext ids writes a match), and so is a
156
- * cleartext one under a row with key material.
193
+ * generation number may by then name another writer's object; until it goes, a rollback onto it needs
194
+ * `allowUnknown`, as onto any generation the row does not know. Once the row is `destroyed` (crypto-shredded,
195
+ * destroyed in the clear or dropped), the object is deleted; once the row is gone, so is an object whose footer
196
+ * matches what this load wrote (another load's object of the same ids is not a match: the footer carries each write's
197
+ * own identity), and so is a cleartext one under a row with key material.
157
198
  */
158
199
  | 'superseded';
200
+ /**
201
+ * What a load did: the generation it wrote, whether that is now current (`published`, and `reason` when not), the
202
+ * object it wrote and what it collected.
203
+ */
159
204
  export interface LoadResult {
160
205
  /**
161
206
  * The generation written. Present even when refused: a refusal deletes that object while the segment's row is
@@ -169,10 +214,19 @@ export interface LoadResult {
169
214
  readonly published: boolean;
170
215
  /** Set only when `published` is false. */
171
216
  readonly reason?: LoadRefusal;
217
+ /**
218
+ * Bytes of the generation's object as stored, or 0 when another load took the generation number first and
219
+ * nothing was written.
220
+ */
172
221
  readonly size: number;
222
+ /**
223
+ * SHA-256 of the object as stored, or `''` when another load took the generation number first. The library neither
224
+ * keeps nor checks it; reads are checked with CRC32C.
225
+ */
173
226
  readonly sha256: string;
227
+ /** Non-empty chunks in the generation written (at most 65,536), or 0 when nothing was written. */
174
228
  readonly chunkCount: number;
175
- /** Distinct ids in the generation that was written, post-dedup. */
229
+ /** Distinct ids in the generation that was written, post-dedup, or 0 when nothing was written. */
176
230
  readonly cardinality: number;
177
231
  /**
178
232
  * What the segment held when the guard judged it — `null` when there was no current generation, or when no
@@ -197,7 +251,8 @@ export interface LoadResult {
197
251
  *
198
252
  * `input` is ids, or a whole bitmap as `{ serialized }` portable Roaring bytes or a `{ bitmap }` with
199
253
  * `serialize('portable')` ({@link LoadInput}). A bitmap input is checked, decoded and serialized before the first
200
- * request, and is written from its own chunks; the generation is byte for byte the one its ids would write.
254
+ * request, and is written from its own chunks; the generation holds chunk for chunk what its ids would write, and carries its own write
255
+ * identity in the footer like any other.
201
256
  *
202
257
  * Returns what was written and whether it became current. A refusal is **not** an exception: `published: false`
203
258
  * with a `reason` is a normal outcome a caller branches on, in the same shape as a successful load, because the
@@ -133,9 +133,13 @@ export type MetricEvent = {
133
133
  };
134
134
  /**
135
135
  * The sink you plug in. One method, synchronous, fire-and-forget. It must never throw back into the
136
- * caller — `CloudRoaring` wraps a user sink with {@link safeMetrics} so a buggy sink can't break I/O.
136
+ * caller — `CloudRoaring` wraps a user sink so that a buggy sink can't break I/O.
137
137
  */
138
138
  export interface IMetricsSink {
139
+ /**
140
+ * Receives one event, synchronously. What it throws, or an async one's rejection, is swallowed and never breaks the
141
+ * call it observes.
142
+ */
139
143
  onEvent(event: MetricEvent): void;
140
144
  }
141
145
  /** The default sink: discards everything, zero overhead. Used whenever no sink is wired. */
@@ -147,23 +151,37 @@ export declare const NOOP_METRICS: IMetricsSink;
147
151
  export declare function safeMetrics(sink: IMetricsSink): IMetricsSink;
148
152
  /** Accumulated totals — the shape returned by {@link CountingMetricsSink.snapshot}. */
149
153
  export interface MetricsSnapshot {
154
+ /**
155
+ * Chunk requests to storage: how many, the bytes they returned, and the time they took in ms, transient-retry backoff
156
+ * included.
157
+ */
150
158
  readonly storage: {
151
159
  readonly gets: number;
152
160
  readonly bytes: number;
153
161
  readonly totalMs: number;
154
162
  };
163
+ /**
164
+ * Decoded-chunk cache hits and misses. A chunk counted without a lookup is a miss; `materializeMany` reads no cache, so
165
+ * it counts none.
166
+ */
155
167
  readonly cache: {
156
168
  readonly hits: number;
157
169
  readonly misses: number;
158
170
  };
171
+ /** Retries taken after a transient fault. */
159
172
  readonly retries: {
160
173
  readonly transient: number;
161
174
  };
175
+ /**
176
+ * Chunk-aligned combines of every operator (intersect, union and andNot): how many ran, the chunk keys they fetched
177
+ * and the keys they skipped without a fetch.
178
+ */
162
179
  readonly intersect: {
163
180
  readonly calls: number;
164
181
  readonly fetchedChunks: number;
165
182
  readonly skippedChunks: number;
166
183
  };
184
+ /** By operation name, how many calls were timed and their total time in ms. */
167
185
  readonly ops: Readonly<Record<MetricOpName, {
168
186
  readonly count: number;
169
187
  readonly totalMs: number;
@@ -181,7 +199,10 @@ export declare class CountingMetricsSink implements IMetricsSink {
181
199
  private intersect;
182
200
  private ops;
183
201
  private static zeroOps;
202
+ /** Adds the event to the totals. An `advisory` event is a note, not a quantity, and adds nothing. */
184
203
  onEvent(event: MetricEvent): void;
204
+ /** A copy of the totals so far, which later events do not change. */
185
205
  snapshot(): MetricsSnapshot;
206
+ /** Sets every total back to zero. */
186
207
  reset(): void;
187
208
  }
@@ -27,15 +27,34 @@ import type { CrbmStorageChunkSource } from './crbm-storage-source.js';
27
27
  * What a pin holds for one segment: the generation, the version identifying those exact bytes, and the pinned
28
28
  * object's fingerprint, so a read can tell that the generation it opens is still that object.
29
29
  *
30
- * `fingerprint` is required for a pin of a generation (`null` only when `generation` is `null`): without it a read
31
- * could not tell a name purged and loaded again from the segment it pinned. `seg.pin()` always records it, and a
32
- * pin of a generation without a string `version` and `fingerprint` is refused with a `ValidationError`.
30
+ * It is one of two shapes, told apart by `generation`: a pin of a generation holds all three fields, and a pin of a
31
+ * segment that had no generation holds `null` in all three. Narrowing on `generation` (or on either other field)
32
+ * narrows the record whole, so a numbered pin passes to `pinAt` as it is:
33
+ *
34
+ * ```ts
35
+ * const at = first.pinnedAt;
36
+ * if (at !== undefined && at.generation !== null) await store.segment('active-30d').pinAt(at);
37
+ * ```
38
+ *
39
+ * `fingerprint` is never `null` for a pin of a generation: without it a read could not tell a name purged and loaded
40
+ * again from the segment it pinned. `seg.pin()` always records it, and a pin of a generation without a string
41
+ * `version` and `fingerprint` is refused with a `ValidationError`.
33
42
  */
34
- export interface PinnedAt {
35
- readonly generation: number | null;
36
- readonly version: string | null;
37
- readonly fingerprint: string | null;
38
- }
43
+ export type PinnedAt = {
44
+ /** The pinned generation. */
45
+ readonly generation: number;
46
+ /** The version of that generation's object. */
47
+ readonly version: string;
48
+ /** The fingerprint of that generation's object. */
49
+ readonly fingerprint: string;
50
+ } | {
51
+ /** `null`: the segment had no generation when it was pinned. */
52
+ readonly generation: null;
53
+ /** `null`, as `generation` is. */
54
+ readonly version: null;
55
+ /** `null`, as `generation` is. */
56
+ readonly fingerprint: null;
57
+ };
39
58
  export declare class PinnedStorageChunkSource implements StorageChunkSource {
40
59
  private readonly inner;
41
60
  /**
@@ -17,3 +17,12 @@ export declare function renewPointer(row: Pick<RegistryRecord, 'currentGen'>): R
17
17
  * cannot key a cache on it. A TypeScript registry fails to compile instead; this catches one in JavaScript.
18
18
  */
19
19
  export declare function pointerIdOf(row: RegistryRecord): Token;
20
+ /**
21
+ * Whether `now` shows a write this call sent as the one that put the pointer where it is: an active row of the same
22
+ * incarnation as `before` (a `create` has none to compare), whose pointer names `generation` and whose `pointerId` is
23
+ * the token of one of the writes in `sent`, which a registry that reports its writes' tokens gave before sending each.
24
+ * The token decides and nothing else is asked, so another writer's write of the same pointer is not taken for this
25
+ * call's. A write that renews the `pointerId` over this call's own landed write (an erasure's renewal, or a write of
26
+ * the pointer back onto the same generation) reads as not this call's: the two cannot be told apart. Pure: no I/O.
27
+ */
28
+ export declare function pointerSetBy(now: RegistryRecord | null, before: RegistryRecord | null, generation: number, sent: ReadonlySet<Token>): boolean;
@@ -25,16 +25,29 @@ export type Token = string;
25
25
  * segment in such a namespace would be invisible to an erasure, a consistency check, an export and a retention sweep.
26
26
  * A driver takes the prefix, since the due index's own rows go through it. Only that exact prefix is reserved:
27
27
  * `cbm.dueX` and `cbmdue.eu` are ordinary namespaces, and a segment name is never restricted.
28
+ *
29
+ * **A key other than `segment` and `namespace` throws `ValidationError`**, in every call of the store and of the free
30
+ * functions behind it that takes a ref, and at a driver port (which also takes a `generation`, a `chunkKey` and a
31
+ * registry row's own fields): a misspelt `namespace` would otherwise address the segment of that name in the default
32
+ * namespace. A `segments()` row is the one exception at the store: its `currentGen` and `status` are taken and read for
33
+ * nothing. A segment handle, a registry row from `registry.get()` or `list()` (a driver port takes one, a store call does
34
+ * not), and a result row such as a consistency issue, a ledger entry, an export manifest entry or an audit event, is not
35
+ * a ref at the store: pass `{ segment, namespace }`.
28
36
  */
29
37
  export interface SegmentRef {
38
+ /** The segment's namespace, or `undefined` for the default one. */
30
39
  readonly namespace?: string;
40
+ /** The segment's name. */
31
41
  readonly segment: string;
32
42
  }
43
+ /** One chunk of a segment: the segment, and the chunk's key. */
33
44
  export interface ChunkRef extends SegmentRef {
45
+ /** The chunk's key: an integer from 0 to 65,535, the high 16 bits of the ids the chunk holds. */
34
46
  readonly chunkKey: number;
35
47
  }
36
48
  /** Identifies one immutable `.crbm` object — a single generation of a segment. */
37
49
  export interface GenKey extends SegmentRef {
50
+ /** The generation number of the object. */
38
51
  readonly generation: number;
39
52
  }
40
53
  /**
@@ -43,6 +56,7 @@ export interface GenKey extends SegmentRef {
43
56
  * holds for an operand's stream.
44
57
  */
45
58
  export interface SegmentSize {
59
+ /** The size of the current generation's object, in bytes. */
46
60
  readonly sizeBytes: number;
47
61
  }
48
62
  /**
@@ -52,8 +66,11 @@ export interface SegmentSize {
52
66
  * can key what it keeps by the generation the bytes came from rather than the one it resolved earlier.
53
67
  */
54
68
  export interface ChunkRead {
69
+ /** The chunk key that was asked for. */
55
70
  readonly key: number;
71
+ /** The chunk's bytes, or `null` where the generation holds none. */
56
72
  readonly bytes: Uint8Array | null;
73
+ /** The version of the generation the bytes were read from, or `null` when the segment has no generation. */
57
74
  readonly version: string | null;
58
75
  }
59
76
  /** Options of {@link StorageChunkSource.getChunks}. */
@@ -91,6 +108,7 @@ export interface ReadChunksOptions {
91
108
  export interface StorageChunkSource {
92
109
  /** Read-only bytes for the chunk, or `null` if absent. Callers must not mutate the buffer. */
93
110
  getChunk(ref: ChunkRef): Promise<Uint8Array | null>;
111
+ /** The chunk keys the segment's current generation holds, or an empty list when it has none. */
94
112
  listChunkKeys(ref: SegmentRef): Promise<number[]>;
95
113
  /**
96
114
  * Optional: several chunks of one segment as a stream, read **from one generation** with fewer storage requests than
@@ -330,11 +348,20 @@ export interface StorageDeleteOptions {
330
348
  * retry can ride it out and a write's caller can tell it from a deterministic failure.
331
349
  */
332
350
  export interface IStorageDriver {
351
+ /** What this driver can do. Range reads and `maxObjectBytes` are required; the other capabilities are optional. */
333
352
  capabilities(): StorageCaps;
334
353
  /**
335
354
  * Stream a new immutable generation. The driver opens a destination, hands `write` a {@link BlobSink},
336
355
  * then atomically commits (and computes the content hash). Throws {@link WriteConflictError} if the key
337
- * already exists (write-once).
356
+ * already exists (write-once). Call `write` once: the object committed is the bytes that one call wrote, because
357
+ * the caller records the write identity of that call's object (its footer) for every later proof; a driver that
358
+ * runs `write` again after a failed attempt must commit only the last run's bytes.
359
+ *
360
+ * What a second run of `write` does depends on what is written. A load of ids, of a bitmap or of a combine's chunks
361
+ * holds its chunks in memory, so each run writes the whole object again, under a write identity of its own, and the
362
+ * load records the run that was committed. A write that streams its chunks, a load with `sorted: true` and an
363
+ * erasure's rewrite, reads them once, so its second run throws {@link IntegrityError} before writing a byte: commit
364
+ * nothing then, and the call throws that error, with the first run's error as its `cause`.
338
365
  */
339
366
  putImmutable(key: GenKey, write: (sink: BlobSink) => Promise<void>): Promise<{
340
367
  size: number;
@@ -366,8 +393,9 @@ export interface IStorageDriver {
366
393
  list(ref: SegmentRef): AsyncIterable<GenKey>;
367
394
  }
368
395
  /**
369
- * Lifecycle status of a segment. `active` is the steady state; `destroyed` is the post-crypto-shred tombstone
370
- * (the row is kept for audit but the segment is logically gone).
396
+ * Lifecycle status of a segment. `active` is the steady state; `destroyed` is the tombstone left by a crypto-shred, a
397
+ * cleartext destroy or a drop (the row is kept for audit but the segment is logically gone: no read resolves a
398
+ * generation through it).
371
399
  */
372
400
  export type RegistryStatus = 'active' | 'destroyed';
373
401
  /**
@@ -402,7 +430,9 @@ export type RegistrySummary = ClearRegistrySummary | SealedRegistrySummary;
402
430
  export type GenerationMetadata = Readonly<Record<string, string | number>>;
403
431
  /** What a generation is, as one read answers it: its number, its id count and the metadata it was written with. */
404
432
  export interface GenerationSummary {
433
+ /** The generation's number. */
405
434
  readonly generation: number;
435
+ /** The number of ids the generation holds. */
406
436
  readonly cardinality: number;
407
437
  /** Absent when the generation has none. */
408
438
  readonly metadata?: GenerationMetadata;
@@ -414,9 +444,8 @@ export interface ClearRegistrySummary {
414
444
  /** How many ids the generation holds: an integer from 0 to 2^32. */
415
445
  readonly cardinality: number;
416
446
  /**
417
- * The fingerprint of the generation's object, `<size>:<footer checksum>` in decimal, as `CrbmReader.fingerprint` and
418
- * a pin's `pinnedAt.fingerprint` spell it: the object's size in bytes, which `stat()` reports, and the CRC32C its
419
- * footer stores.
447
+ * The fingerprint of the generation's object, `<size>:<footer checksum>` in decimal, as a pin's `pinnedAt.fingerprint`
448
+ * spells it: the object's size in bytes, which `stat()` reports, and the CRC32C its footer stores.
420
449
  */
421
450
  readonly fingerprint: string;
422
451
  /**
@@ -485,8 +514,9 @@ export interface RegistryRecord extends SegmentRef {
485
514
  readonly currentGen: number | null;
486
515
  /**
487
516
  * Per-segment data-key (DEK) wrappings for encryption-at-rest: the DEK envelope-wrapped under one or more KEKs
488
- * (active + optional recovery). Reading unwraps with any held KEK; **crypto-shred deletes this whole list**,
489
- * making the segment's at-rest bytes permanently unrecoverable. Absent ⇒ the segment is cleartext.
517
+ * (active + optional recovery). Reading unwraps with any held KEK; **crypto-shred deletes this whole list** from
518
+ * the current row, so the segment's at-rest bytes are unrecoverable once no retained copy of the row (a noncurrent
519
+ * version, a backup) still holds it, or every KEK that wrapped it is destroyed. Absent ⇒ the segment is cleartext.
490
520
  */
491
521
  readonly wrappedDeks?: readonly WrappedDek[];
492
522
  /**
@@ -495,6 +525,10 @@ export interface RegistryRecord extends SegmentRef {
495
525
  * keystore. Clearable on crypto-shred.
496
526
  */
497
527
  readonly keyId?: string;
528
+ /**
529
+ * `'active'`, or `'destroyed'` for the tombstone of a segment that was crypto-shredded, destroyed in the clear or
530
+ * dropped.
531
+ */
498
532
  readonly status: RegistryStatus;
499
533
  /**
500
534
  * Governance policy. `retention.expiresAt` drives the retention sweep (see {@link GovernanceMeta}); `residency`
@@ -502,6 +536,7 @@ export interface RegistryRecord extends SegmentRef {
502
536
  * `list()` projection — a fleet sweep reads the policy from the enumeration rather than per-segment.
503
537
  */
504
538
  readonly retention?: GovernanceMeta;
539
+ /** Residency metadata, a plain object. It is stored and returned as given; nothing reads it yet. */
505
540
  readonly residency?: GovernanceMeta;
506
541
  /**
507
542
  * A cached description of the current generation (see {@link RegistrySummary}). Optional: a row without one is
@@ -538,6 +573,7 @@ export interface RegistryRecord extends SegmentRef {
538
573
  readonly leases?: readonly LeaseEntry[];
539
574
  /** Epoch-ms of creation / last mutation (from the driver's injected clock). */
540
575
  readonly createdAt: number;
576
+ /** Epoch-ms of the row's last write, from the driver's clock. */
541
577
  readonly updatedAt: number;
542
578
  /**
543
579
  * Opaque OCC token — compare-by-equality, ABA-safe. A shipped registry gives no two writes under one name the same
@@ -565,11 +601,15 @@ export interface RegistryRecord extends SegmentRef {
565
601
  export interface NewRegistryRecord {
566
602
  /** `null` ⇒ the segment has no Storage generation yet — see {@link RegistryRecord.currentGen}. */
567
603
  readonly currentGen: number | null;
604
+ /** The segment's wrapped data keys, which make it an encrypted segment. Absent for a cleartext one. */
568
605
  readonly wrappedDeks?: readonly WrappedDek[];
606
+ /** An external key reference. Reserved: the in-process keystore does not use it. */
569
607
  readonly keyId?: string;
570
608
  /** Defaults to `'active'`. */
571
609
  readonly status?: RegistryStatus;
610
+ /** The retention policy, a plain object; its `expiresAt` is what the retention sweep reads. */
572
611
  readonly retention?: GovernanceMeta;
612
+ /** Residency metadata, a plain object. It is stored and returned as given; nothing reads it yet. */
573
613
  readonly residency?: GovernanceMeta;
574
614
  /** Must name `currentGen` when given. */
575
615
  readonly summary?: RegistrySummary;
@@ -614,6 +654,11 @@ export interface RegistryWriteOptions {
614
654
  * when the write names a resolved field. A write that does not land leaves no row carrying it. A call refused before
615
655
  * it has a token to send, on a conflict its own read finds, reports none.
616
656
  *
657
+ * A publish relies on the report coming first, both ways. A write that faults after its report may have landed, and
658
+ * is settled by reading the row. A `TransientError` before any report is taken for a fault before sending, such as
659
+ * the driver's own read of the row: the publish throws it at once, with no read of the row and no fresh write. So
660
+ * nothing that can land is sent before the report, a client's own resend included.
661
+ *
617
662
  * The token is unique to that write attempt, also against another writer racing the same version: two writers that
618
663
  * send a write against the same row version never carry the same token, so a version counter alone is not enough.
619
664
  * Otherwise a loser's write would pass for the winner's.
@@ -670,8 +715,8 @@ declare const STORAGE_BACKEND_BRAND: unique symbol;
670
715
  * Where a store keeps everything: the generations, and the pointer saying which one is current.
671
716
  *
672
717
  * **Branded on purpose.** The two fields alone are not enough to qualify, and that is the whole point. This
673
- * shape is also {@link LoadDeps}, so before the brand any `{ storage, registry }`
674
- * object literal satisfied it — including one assembling halves from two *unrelated* stores, which the store
718
+ * same two fields are the heart of the deps core's standalone functions take, so without the brand any `{ storage, registry }`
719
+ * object literal would satisfy it — including one assembling halves from two *unrelated* stores, which the store
675
720
  * accepted and then answered **empty** for a segment that holds data, because it read a pointer from a place
676
721
  * nothing had ever been written. That is the exact silent-empty failure one-class-per-backend exists to
677
722
  * remove, and it was reachable in five lines.
@@ -700,6 +745,38 @@ export interface StorageBackend {
700
745
  * exception propagates from the store's constructor, so a backend's bug fails fast rather than going unseen.
701
746
  */
702
747
  attachMetrics?(sink: IMetricsSink): void;
748
+ /**
749
+ * Optional. Reads the setup the backend cannot change but can be broken by, and says whether it will delete or
750
+ * archive what the store keeps: today, the bucket's lifecycle rules over the generation objects and the
751
+ * `<prefix>registry/` rows. It is meant for a check run by hand or at deploy time, not for the read or write path, and
752
+ * costs one request, plus the SDK's own retries. It never rejects: a check that could not be made resolves to `unchecked` with the
753
+ * reason in `detail`. A backend without it, a third-party one or a {@link brandAsBackend} wrapper that does not
754
+ * forward it, offers no lifecycle check, and `store.preflight()` reads that as `unchecked`.
755
+ *
756
+ * The cloud rules stay in the driver. A `detail` is for a person, carries no credential, and never changes what a
757
+ * caller does: act on `status`.
758
+ */
759
+ checkLifecycle?(): Promise<LifecycleVerdict>;
760
+ }
761
+ /**
762
+ * What {@link StorageBackend.checkLifecycle} found.
763
+ *
764
+ * - `ok`: no rule that could delete or archive the store's objects. `detail` may still carry advice, such as a missing
765
+ * rule that cleans up abandoned multipart uploads, which is never a `problem` by itself.
766
+ * - `problem`: a rule that deletes or archives the store's current objects, or one that cannot be told apart from it.
767
+ * Where a rule is unclear, the check reports a `problem`: a condition it does not interpret, or a value it does not
768
+ * recognise (a storage class, an action).
769
+ * - `unchecked`: the check could not be made (no permission to read the configuration, an error, or a backend whose
770
+ * rules are out of the data plane's reach). `detail` says what to grant or how to check by hand.
771
+ */
772
+ export interface LifecycleVerdict {
773
+ /**
774
+ * `'ok'` when no rule would delete or archive the store's objects, `'problem'` when one would or may, `'unchecked'`
775
+ * when the check could not be made.
776
+ */
777
+ readonly status: 'ok' | 'problem' | 'unchecked';
778
+ /** A sentence or two for a person. Never a credential. */
779
+ readonly detail: string;
703
780
  }
704
781
  /**
705
782
  * The brand key, for the type-only `declare` field a backend class carries.
@@ -749,7 +826,7 @@ export declare function isStorageBackend(value: unknown): value is StorageBacken
749
826
  * probability (ABA-safe across delete→recreate).
750
827
  *
751
828
  * **What a driver must do** — callers rely on each of these, and the registry conformance suite holds every
752
- * shipped driver to them:
829
+ * shipped driver to them, except the rule on `unsettledConflict`, which the suite does not test:
753
830
  *
754
831
  * - **`create` and `compareAndSwap` are atomic conditional writes.** A `create` over a live row, and a
755
832
  * `compareAndSwap` whose token is not the stored one, throw {@link WriteConflictError} and change nothing.
@@ -779,9 +856,19 @@ export declare function isStorageBackend(value: unknown): value is StorageBacken
779
856
  * Either send each write once, with the backend client's retry off for that request, or recognise your own
780
857
  * write when the precondition fails by reading an id you stored with it back. The shipped registries do
781
858
  * one or the other.
859
+ * - **Raise a conflict that may follow your own write's landing with `unsettledConflict`** (from the driver kit). A
860
+ * driver whose client may have sent the write more than once cannot say, after a conflict, that this write did not
861
+ * land first: a row is overwritten, so another writer's write found on the read-back may have gone over this one, and
862
+ * a row that is gone may have been removed after it landed. Raise every conflict it cannot rule that out for (another
863
+ * writer's write on the read-back, a compare-and-swap that finds the row gone, a read-back that finds nothing) as
864
+ * `unsettledConflict`, so a load refused after it reports `unanswered: true`. A conflict that proves the write did
865
+ * not land (a write sent once, a check made before sending anything) is a plain {@link WriteConflictError}. A
866
+ * registry that wraps another rethrows the conflict it was given, unchanged: the check that reads the mark is
867
+ * internal, so a conflict built anew loses it.
782
868
  * - **Raise a transient fault as {@link TransientError}.**
783
869
  */
784
870
  export interface IRegistryDriver {
871
+ /** What this registry can do. Strongly consistent reads are required; the other capabilities are optional. */
785
872
  capabilities(): RegCaps;
786
873
  /**
787
874
  * `true` when every `create` and `compareAndSwap` of this registry reports the token its write carries to
@@ -792,8 +879,15 @@ export interface IRegistryDriver {
792
879
  *
793
880
  * The flag may be a getter or a field. A registry that wraps or extends another and overrides `create` or
794
881
  * `compareAndSwap` either passes `options` through to the write it forwards, or sets `reportsWriteTokens` to
795
- * `false`. One that does neither reads every write of its own that landed after an error or a lost answer as not its
796
- * own (`superseded`), and deletes nothing.
882
+ * `false`. One that does neither reads a write of its own that landed and then met a conflict as not its own
883
+ * (`superseded`), and takes every `TransientError` of its writes, a throttle and a lost answer alike, for a fault
884
+ * before sending: the publish throws it at once, with no read of the row and no fresh write. Either way it deletes
885
+ * nothing.
886
+ *
887
+ * A publish relies on the report coming first, both ways. A write that faults after its report may have landed, and
888
+ * is settled by reading the row. A `TransientError` before any report is taken for a fault before sending, such as
889
+ * the driver's own read of the row: the publish throws it at once, with no read of the row and no fresh write. So
890
+ * nothing that can land is sent before the report, a client's own resend included.
797
891
  */
798
892
  readonly reportsWriteTokens?: boolean;
799
893
  /**
@@ -0,0 +1,80 @@
1
+ /**
2
+ * `runPreflight` — the first-run check of a store's storage and registry, behind `store.preflight()`.
3
+ *
4
+ * ```
5
+ * registry ── get(probe row) must be null, canWrite !== false
6
+ * write ───── put marker A under the probe key
7
+ * write-once put marker B under the same key: must throw WriteConflictError
8
+ * read ────── getTail: must return A (read-after-write); gives the version
9
+ * list ────── list the probe segment: must name the probe
10
+ * delete ──── (finally) delete this run's own key, with the tail's version when it has one
11
+ * lifecycle ─ backend.checkLifecycle(), bounded by a timer, unchecked when the backend has none
12
+ * ```
13
+ *
14
+ * The probe key is `cbm.due.preflight/segments/probe.<n>.crbm`, where `<n>` is a 48-bit random number. That
15
+ * namespace sits inside the reserved `cbm.due.` prefix an application can never name, so no segment of the application's can collide with it, and the registry
16
+ * writes nothing: the probe is a storage object only, which no registry row names and no scan of the registry
17
+ * reaches. Storage is only ever listed one segment at a time, from rows or from refs the application gave.
18
+ *
19
+ * A finding never throws. A step that fails (a transient fault included) is a `problem` whose detail says so, and the
20
+ * checks that cannot run without it are `unchecked`; the lifecycle step is the exception, whose failures, its timeout
21
+ * included, are `unchecked`. Every step runs through one guard, so nothing a step meets (an error whose getters throw,
22
+ * a verdict whose getters throw, a clock whose sleep rejects, an entropy source that throws) escapes. The only throw
23
+ * is a store with nothing to check, which the facade refuses before it gets here.
24
+ *
25
+ * Not checked: read-after-delete, the registry's `If-Match` on compare-and-swap, multipart abort permission,
26
+ * versioning and backups.
27
+ *
28
+ * Time arrives through the injected {@link Clock}: the lifecycle step is bounded by {@link LIFECYCLE_TIMEOUT_MS},
29
+ * counted in short sleeps of {@link LIFECYCLE_TICK_MS} so that a check that answers at once leaves no long timer
30
+ * holding a short-lived process open.
31
+ */
32
+ import type { Clock, Entropy } from './determinism.js';
33
+ import type { IRegistryDriver, IStorageDriver, LifecycleVerdict } from './ports.js';
34
+ /**
35
+ * What one check found.
36
+ *
37
+ * - `ok`: the check ran and found nothing wrong.
38
+ * - `problem`: the check ran and found something wrong, or the request it makes failed (a transient fault included:
39
+ * `detail` says to run again then). The lifecycle check is the exception: its failures, its timeout included, are
40
+ * `unchecked`.
41
+ * - `unchecked`: the check could not be made, because a step it depends on failed (`write-once`, `read`, `list` and
42
+ * `delete` after a failed `write`), or the backend offers no such check, or the lifecycle check did not answer in
43
+ * time. `ok` on the result does not vouch for it.
44
+ */
45
+ export interface PreflightCheck {
46
+ /** Which check this is. A run lists all seven, in this order. */
47
+ readonly name: 'registry' | 'write' | 'write-once' | 'read' | 'list' | 'delete' | 'lifecycle';
48
+ /** What the check found. */
49
+ readonly status: 'ok' | 'problem' | 'unchecked';
50
+ /** A sentence or two for a person: what was found, and what to do. The name and message of a failed step's error. */
51
+ readonly detail: string;
52
+ }
53
+ /** What `store.preflight()` found: every check, and whether none of them is a `problem`. */
54
+ export interface PreflightResult {
55
+ /** True when no check is a `problem`. A check that is `unchecked` does not make it false. */
56
+ readonly ok: boolean;
57
+ /** All seven checks, in the order the type of `PreflightCheck.name` lists them. */
58
+ readonly checks: PreflightCheck[];
59
+ }
60
+ /** What a run needs: the store's raw drivers, an entropy source for the probe's number, and a clock for the bound. */
61
+ export interface PreflightDeps {
62
+ readonly storage: IStorageDriver;
63
+ readonly registry: IRegistryDriver;
64
+ readonly clock: Clock;
65
+ readonly entropy: Entropy;
66
+ /** The backend's lifecycle check, bound to the backend; absent for a backend that has none. */
67
+ readonly checkLifecycle?: (() => Promise<LifecycleVerdict>) | undefined;
68
+ }
69
+ /** The probe's namespace, inside the reserved prefix; no application ref can name it. */
70
+ export declare const PREFLIGHT_NAMESPACE = "cbm.due.preflight";
71
+ /** The probe's segment name. */
72
+ export declare const PREFLIGHT_SEGMENT = "probe";
73
+ /** How long the lifecycle step may take before it is read as `unchecked`: 30 seconds of the injected clock. */
74
+ export declare const LIFECYCLE_TIMEOUT_MS = 30000;
75
+ /** The length of each sleep the bound is counted in. */
76
+ export declare const LIFECYCLE_TICK_MS = 250;
77
+ /**
78
+ * Run the seven checks. Never rejects: see the module comment.
79
+ */
80
+ export declare function runPreflight(deps: PreflightDeps): Promise<PreflightResult>;