@cloudbitmaps/core 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -8
- package/dist/chunk-7GT632UF.js +641 -0
- package/dist/chunk-7GT632UF.js.map +7 -0
- package/dist/chunk-CGILDXGP.js +73 -0
- package/dist/chunk-CGILDXGP.js.map +7 -0
- package/dist/chunk-GKHERVVQ.js +121 -0
- package/dist/chunk-GKHERVVQ.js.map +7 -0
- package/dist/{chunk-IGLIZBR7.js → chunk-MVUD4R72.js} +253 -765
- package/dist/chunk-MVUD4R72.js.map +7 -0
- package/dist/core/audit.d.ts +53 -15
- package/dist/core/bit-route.d.ts +9 -0
- package/dist/core/blob.d.ts +4 -0
- package/dist/core/budget.d.ts +8 -3
- package/dist/core/check.d.ts +27 -0
- package/dist/core/codec.d.ts +1 -1
- package/dist/core/combine-expr.d.ts +9 -5
- package/dist/core/combine-feed.d.ts +11 -3
- package/dist/core/combine-many.d.ts +28 -13
- package/dist/core/consistency.d.ts +11 -2
- package/dist/core/cooperative.d.ts +7 -0
- package/dist/core/crbm-storage-source.d.ts +7 -4
- package/dist/core/crypto.d.ts +5 -0
- package/dist/core/determinism.d.ts +2 -1
- package/dist/core/discover.d.ts +5 -3
- package/dist/core/engine.d.ts +4 -4
- package/dist/core/erase-id.d.ts +52 -19
- package/dist/core/erasure.d.ts +62 -28
- package/dist/core/errors.d.ts +83 -9
- package/dist/core/leases.d.ts +2 -0
- package/dist/core/load-input.d.ts +48 -0
- package/dist/core/load.d.ts +71 -16
- package/dist/core/metrics.d.ts +22 -1
- package/dist/core/pinned-storage-source.d.ts +27 -8
- package/dist/core/pointer-id.d.ts +9 -0
- package/dist/core/ports.d.ts +107 -13
- package/dist/core/preflight.d.ts +80 -0
- package/dist/core/registry-scan.d.ts +0 -5
- package/dist/core/retention-sweep.d.ts +61 -16
- package/dist/core/retention.d.ts +18 -7
- package/dist/core/retry.d.ts +5 -2
- package/dist/core/rollback.d.ts +23 -12
- package/dist/core/validate.d.ts +33 -2
- package/dist/driver-kit.d.ts +9 -4
- package/dist/driver-kit.js +63 -27
- package/dist/driver-kit.js.map +2 -2
- package/dist/drivers/_shared/object-registry.d.ts +24 -1
- package/dist/drivers/_shared/options.d.ts +15 -7
- package/dist/drivers/backends.d.ts +19 -2
- package/dist/drivers/crypto.d.ts +18 -3
- package/dist/export/index.d.ts +25 -10
- package/dist/index.d.ts +19 -17
- package/dist/index.js +2457 -2283
- package/dist/index.js.map +4 -4
- package/dist/node.d.ts +14 -0
- package/dist/node.js +1064 -0
- package/dist/node.js.map +7 -0
- package/package.json +16 -2
- package/dist/chunk-IGLIZBR7.js.map +0 -7
- package/dist/core/absent-operands.d.ts +0 -8
- package/dist/core/bulk-load.d.ts +0 -148
- package/dist/core/cas-retry.d.ts +0 -27
- package/dist/core/chunk-checks.d.ts +0 -39
- package/dist/core/chunk-stream.d.ts +0 -22
- package/dist/core/chunk-window.d.ts +0 -37
- package/dist/core/combine-ledger.d.ts +0 -43
- package/dist/core/crbm/crc32c.d.ts +0 -10
- package/dist/core/crbm/fingerprint.d.ts +0 -23
- package/dist/core/crbm/format.d.ts +0 -150
- package/dist/core/crbm/frame.d.ts +0 -10
- package/dist/core/crbm/plan-reads.d.ts +0 -50
- package/dist/core/crbm/varint.d.ts +0 -14
- package/dist/core/crbm/writer.d.ts +0 -72
- package/dist/core/crbm-open.d.ts +0 -154
- package/dist/core/crbm-write.d.ts +0 -75
- package/dist/core/due-index.d.ts +0 -53
- package/dist/core/generation-gc.d.ts +0 -293
- package/dist/core/generation-list.d.ts +0 -11
- package/dist/core/id-window.d.ts +0 -46
- package/dist/core/invalidation-scope.d.ts +0 -77
- package/dist/core/item-pull.d.ts +0 -29
- package/dist/core/known-generations-shape.d.ts +0 -13
- package/dist/core/known-generations.d.ts +0 -147
- package/dist/core/ledger-seed.d.ts +0 -17
- package/dist/core/metadata.d.ts +0 -42
- package/dist/core/order.d.ts +0 -2
- package/dist/core/publish.d.ts +0 -207
- package/dist/core/reader-defaults.d.ts +0 -29
- package/dist/core/summary.d.ts +0 -110
- package/dist/core/token.d.ts +0 -41
- package/dist/drivers/_shared/entropy.d.ts +0 -5
- package/dist/drivers/_shared/registry.d.ts +0 -107
- package/dist/drivers/localfs/exact-case.d.ts +0 -34
- package/dist/drivers/localfs/fs-util.d.ts +0 -55
- package/dist/drivers/localfs/paths.d.ts +0 -40
- package/dist/drivers/localfs/registry.d.ts +0 -50
- package/dist/drivers/localfs/storage.d.ts +0 -28
- package/dist/drivers/memory.d.ts +0 -64
- package/dist/testing/fuzz-core.d.ts +0 -15
package/dist/core/audit.d.ts
CHANGED
|
@@ -73,10 +73,13 @@ export type AuditEvent = {
|
|
|
73
73
|
* segment's registry row changed while the load was writing). `cardinality` is what the refused generation
|
|
74
74
|
* would have contained, and `0` for a load that lost its generation number and wrote nothing.
|
|
75
75
|
*
|
|
76
|
-
* `unanswered` is present, and `true`, on a `'superseded'` refusal
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
76
|
+
* `unanswered` is present, and `true`, on a `'superseded'` refusal after a registry write whose answer did not
|
|
77
|
+
* say whether it landed: one that ended without an answer, or a conflict its registry marked as one that may
|
|
78
|
+
* follow the write's own landing (Azure Blob marks every conflict of a row write, since its client may have sent
|
|
79
|
+
* the write twice). That write may have landed, so the generation may have been the segment's current one for a
|
|
80
|
+
* while before another write replaced it, or, where a write renewed the row's `pointerId` over it, may be current
|
|
81
|
+
* still. Absent when every write was answered and every conflict was one its registry vouches the write did not
|
|
82
|
+
* land.
|
|
80
83
|
*/
|
|
81
84
|
readonly kind: 'segment.load-refused';
|
|
82
85
|
readonly namespace?: string;
|
|
@@ -115,11 +118,12 @@ export type AuditEvent = {
|
|
|
115
118
|
* not hold, or an encrypted one under a cleartext row (a first load's that lost the race, or crashed). The one
|
|
116
119
|
* emitter is `eraseIdFromSegment`, once a listing of the bucket shows nothing left holding the id, so the record
|
|
117
120
|
* exists only for a finished erasure. A call that ends otherwise throws or reports `erased: false` with another
|
|
118
|
-
* reason and emits nothing, even when it had deleted a holder; its re-run emits when it finishes, and
|
|
119
|
-
*
|
|
120
|
-
*
|
|
121
|
-
* segment's pointer on a generation it had
|
|
122
|
-
* the record although the call cannot
|
|
121
|
+
* reason and emits nothing, even when it had deleted a holder; its re-run emits when it finishes, and, when
|
|
122
|
+
* another collection deleted the last holder meanwhile, emits nothing and answers `'not-member'`, which
|
|
123
|
+
* `eraseSubject`'s ledger shows by listing nothing for the segment. One finished erasure emits it and then
|
|
124
|
+
* throws `NotFoundError`: one that found, after its deletes, the segment's pointer on a generation it had
|
|
125
|
+
* deleted (a rollback moved it there meanwhile), so the deletion is on the record although the call cannot
|
|
126
|
+
* answer.
|
|
123
127
|
*
|
|
124
128
|
* `fromGeneration` is the newest generation the call found holding the id, and is absent when it found the id in
|
|
125
129
|
* none and deleted only objects it could not search: then the ledger lists no entry for the segment, and this
|
|
@@ -137,24 +141,52 @@ export type AuditEvent = {
|
|
|
137
141
|
readonly collected: readonly number[];
|
|
138
142
|
} | {
|
|
139
143
|
/**
|
|
140
|
-
* A segment was **crypto-shredded
|
|
141
|
-
*
|
|
142
|
-
*
|
|
144
|
+
* A segment was **crypto-shredded**: its wrapped DEK(s) are gone from the segment's **current** registry row,
|
|
145
|
+
* so nothing that opens the segment through that row can decrypt its Storage bytes, in the bucket or in any
|
|
146
|
+
* backup, replica or WORM copy of its objects. It destroys no KEK: a noncurrent version, a backup or a
|
|
147
|
+
* point-in-time copy of the row still holds the wrapped DEK(s), and a registry restore to a point before the
|
|
148
|
+
* shred makes the segment readable again wherever its objects survive. The destruction is complete once no
|
|
149
|
+
* retained copy of the row holds them, or once every KEK that wrapped them is destroyed. A process that had
|
|
150
|
+
* already opened the segment keeps its unwrapped key in memory until told (`store.invalidate(ref)`). Emitted
|
|
151
|
+
* only for a genuine key shred, never for a cleartext tombstone (which leaves the Storage bytes readable) or
|
|
152
|
+
* an idempotent re-run.
|
|
143
153
|
*/
|
|
144
154
|
readonly kind: 'segment.erase';
|
|
145
155
|
readonly namespace?: string;
|
|
146
156
|
readonly segment: string;
|
|
147
157
|
/** The incarnation of the row the shred turned into a tombstone. */
|
|
148
158
|
readonly incarnation?: string;
|
|
159
|
+
} | {
|
|
160
|
+
/**
|
|
161
|
+
* A segment was tombstoned **without a key shred**: `destroySegment` or `eraseNamespace` with `allowCleartext`
|
|
162
|
+
* turned the row of a segment that holds no wrapped key into a `destroyed` tombstone. No read resolves the
|
|
163
|
+
* segment any more, but nothing was deleted and no key was destroyed, so its objects stay readable in the
|
|
164
|
+
* bucket, in any backup and through any copy of the row from before the tombstone.
|
|
165
|
+
*
|
|
166
|
+
* Deliberately not {@link AuditEvent} `segment.erase`, which attests a key shred, nor `segment.dispose`, which
|
|
167
|
+
* attests that a drop swept the storage: this one is the record that the row was tombstoned, which is what a
|
|
168
|
+
* registry restore undoes and what the restore procedure looks for. `objectsReadable` is always `true`; it
|
|
169
|
+
* is a field so a consumer that forwards events cannot read this one as an erasure receipt. Emitted once per
|
|
170
|
+
* tombstone this call wrote, not for the idempotent re-run, and not for a segment that was encrypted
|
|
171
|
+
* (`segment.erase`), or dropped (`segment.dispose`).
|
|
172
|
+
*/
|
|
173
|
+
readonly kind: 'segment.tombstone';
|
|
174
|
+
readonly namespace?: string;
|
|
175
|
+
readonly segment: string;
|
|
176
|
+
/** The incarnation of the row the call turned into a tombstone. */
|
|
177
|
+
readonly incarnation?: string;
|
|
178
|
+
/** Always `true`: the segment's objects were not deleted and no key was destroyed. */
|
|
179
|
+
readonly objectsReadable: true;
|
|
149
180
|
} | {
|
|
150
181
|
/**
|
|
151
182
|
* A segment was **disposed of** — tombstoned and its storage reclaimed (its Storage generations deleted) by
|
|
152
183
|
* `dropSegment`, *without* a key shred.
|
|
153
184
|
*
|
|
154
185
|
* Deliberately a separate kind from {@link AuditEvent} `segment.erase`, and the distinction is the point.
|
|
155
|
-
* `segment.erase` attests that
|
|
156
|
-
*
|
|
157
|
-
*
|
|
186
|
+
* `segment.erase` attests that the current row no longer holds the key, which is the one erasure claim that
|
|
187
|
+
* reaches WORM copies of the objects, once no retained copy of the row holds it. Deleting an object is
|
|
188
|
+
* strictly weaker: a noncurrent version, a cross-region replica or a PITR snapshot can still hold the
|
|
189
|
+
* cleartext. Emitting one kind for both would make a compliance dashboard
|
|
158
190
|
* over-attest, so a cleartext drop gets this instead.
|
|
159
191
|
*
|
|
160
192
|
* An **encrypted** segment dropped via `dropSegment` emits **both** — `segment.erase` for the key shred and
|
|
@@ -178,6 +210,10 @@ export type AuditEvent = {
|
|
|
178
210
|
};
|
|
179
211
|
/** Sink for {@link AuditEvent}s. Injected via the lifecycle options; omit it and nothing is recorded. */
|
|
180
212
|
export interface IAuditSink {
|
|
213
|
+
/**
|
|
214
|
+
* Receives one audit event, called synchronously by the operation that caused it. What it throws, or rejects with, is
|
|
215
|
+
* swallowed and never breaks that operation.
|
|
216
|
+
*/
|
|
181
217
|
onEvent(event: AuditEvent): void;
|
|
182
218
|
}
|
|
183
219
|
/** The default sink: records nothing. */
|
|
@@ -200,8 +236,10 @@ export declare function ignoreRejection(returned: unknown): void;
|
|
|
200
236
|
/** A ready-made sink that records events into an in-memory list — handy for tests + simple audit trails. */
|
|
201
237
|
export declare class RecordingAuditSink implements IAuditSink {
|
|
202
238
|
private readonly recorded;
|
|
239
|
+
/** Records the event, in emission order. */
|
|
203
240
|
onEvent(event: AuditEvent): void;
|
|
204
241
|
/** An independent copy of the recorded events, in emission order. */
|
|
205
242
|
snapshot(): AuditEvent[];
|
|
243
|
+
/** Forgets every recorded event. */
|
|
206
244
|
reset(): void;
|
|
207
245
|
}
|
package/dist/core/bit-route.d.ts
CHANGED
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BitRouteSplitter — maps a 32-bit id to its chunk (top 16 bits) and remainder
|
|
3
|
+
* (bottom 16 bits), per the data model.
|
|
4
|
+
*/
|
|
5
|
+
import { ValidationError } from './errors.js';
|
|
1
6
|
export declare const U32_MAX = 4294967295;
|
|
2
7
|
export declare const CHUNK_COUNT = 65536;
|
|
3
8
|
interface BitRoute {
|
|
@@ -6,6 +11,10 @@ interface BitRoute {
|
|
|
6
11
|
/** Bottom 16 bits — the value within the chunk (0..65535). */
|
|
7
12
|
readonly remainder: number;
|
|
8
13
|
}
|
|
14
|
+
/** Was `err` thrown by {@link splitId} for an id outside the u32 range? */
|
|
15
|
+
export declare function isBadIdError(err: unknown): err is ValidationError;
|
|
16
|
+
/** The refusal {@link splitId} throws for a value that is not a u32 id, for a caller that checks the range itself. */
|
|
17
|
+
export declare function refusalOfId(id: unknown): ValidationError;
|
|
9
18
|
/** Split a u32 id into `{ chunkKey, remainder }`. Throws `ValidationError` for non-u32 input. */
|
|
10
19
|
export declare function splitId(id: number): BitRoute;
|
|
11
20
|
/**
|
package/dist/core/blob.d.ts
CHANGED
|
@@ -3,6 +3,10 @@
|
|
|
3
3
|
* the write resolves, so a sink that holds bytes past that copies them.
|
|
4
4
|
*/
|
|
5
5
|
export interface BlobSink {
|
|
6
|
+
/**
|
|
7
|
+
* Appends `bytes` to the object. Once it resolves the caller may reuse the buffer, so a sink that keeps the bytes
|
|
8
|
+
* copies them.
|
|
9
|
+
*/
|
|
6
10
|
write(bytes: Uint8Array): Promise<void>;
|
|
7
11
|
}
|
|
8
12
|
/**
|
package/dist/core/budget.d.ts
CHANGED
|
@@ -15,6 +15,11 @@
|
|
|
15
15
|
* search generation by generation is also charged one unit for each generation it opens beyond the current one,
|
|
16
16
|
* since that is O(generations) work the segment count alone does not show.
|
|
17
17
|
*/
|
|
18
|
+
/**
|
|
19
|
+
* The cap on one operation's fan-out: its `maxRequests` units, counted in storage chunk fetches for a read and in
|
|
20
|
+
* segments scanned for an admin scan. `eraseSubject` also charges one unit for each generation beyond the current one
|
|
21
|
+
* that it opens, in a segment it has to search generation by generation.
|
|
22
|
+
*/
|
|
18
23
|
export interface Budget {
|
|
19
24
|
/** Max units of fan-out (Storage chunk fetches, or segments scanned) one op may issue before it is refused. */
|
|
20
25
|
readonly maxRequests: number;
|
|
@@ -30,13 +35,13 @@ export type BudgetOption = Partial<Budget> | false;
|
|
|
30
35
|
/**
|
|
31
36
|
* Resolve a budget option against a fallback to a concrete `Budget`, or `null` when disabled. Validates: anything but
|
|
32
37
|
* `false`, an absent value or an object of `{ maxRequests }` is refused, since a cap of the wrong shape (`5`,
|
|
33
|
-
* `{ maxRequest: 5 }`) would otherwise read as no override and leave the fallback in force.
|
|
34
|
-
*
|
|
38
|
+
* `{ maxRequest: 5 }`) would otherwise read as no override and leave the fallback in force. A key whose value is `undefined` reads
|
|
39
|
+
* as absent; a `null` budget and a `maxRequests` of `null` are refused.
|
|
35
40
|
*/
|
|
36
41
|
export declare function resolveBudget(opt: BudgetOption | undefined, fallback: Budget): Budget | null;
|
|
37
42
|
/**
|
|
38
43
|
* Resolve a **per-op** budget override against the store's own resolved budget (never the raw global default):
|
|
39
|
-
* `undefined`
|
|
44
|
+
* `undefined` inherits the store budget as-is (including a disabled `null`); a partial `{}` / omitted
|
|
40
45
|
* `maxRequests` inherits the store's *tightening* (so a per-op passthrough of an absent config value can't
|
|
41
46
|
* silently lift a tight tenant ceiling back to the generous default); `{ maxRequests }` replaces it; `false`
|
|
42
47
|
* disables. Falls back to {@link DEFAULT_BUDGET} only when the store itself has no budget (disabled).
|
package/dist/core/check.d.ts
CHANGED
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A value as a message shows it: a number or boolean as itself, a bigint with an `n` (so `5n` is not mistaken for
|
|
3
|
+
* `5`), a string quoted (so `'200'` is not mistaken for `200`), and anything else by its type.
|
|
4
|
+
*/
|
|
5
|
+
export declare function shown(value: unknown): string;
|
|
1
6
|
/**
|
|
2
7
|
* `value` if it is a safe integer from `min` to `max` (inclusive; `max` defaults to the largest safe integer), else a
|
|
3
8
|
* `ValidationError` naming `what`: `<what> must be an integer from <min> to <max>; got <value>`.
|
|
@@ -8,6 +13,11 @@ export declare function requireInteger(value: unknown, bounds: {
|
|
|
8
13
|
readonly what: string;
|
|
9
14
|
readonly note?: string;
|
|
10
15
|
}): number;
|
|
16
|
+
/**
|
|
17
|
+
* Refuse a `requireEncryption` that is present and not exactly a boolean. Every site that acts on the flag compares it
|
|
18
|
+
* with `=== true`, so a truthy string would be taken for `true` by one and for `false` by another.
|
|
19
|
+
*/
|
|
20
|
+
export declare function checkRequireEncryption(value: unknown): void;
|
|
11
21
|
/** `value` if it is a safe integer of 1 or more, else a `ValidationError` naming `what`. */
|
|
12
22
|
export declare function requirePositiveInteger(value: unknown, what: string): number;
|
|
13
23
|
/**
|
|
@@ -20,3 +30,20 @@ export declare function requireFiniteNumber(value: unknown, bounds: {
|
|
|
20
30
|
readonly what: string;
|
|
21
31
|
readonly note?: string;
|
|
22
32
|
}): number;
|
|
33
|
+
/**
|
|
34
|
+
* Refuse an options bag that is not an object, or that holds a key not in `keys`, naming each such key and the keys it
|
|
35
|
+
* takes. `hint`, when given, ends the message, to point at where a likely mistake belongs (`an S3 client goes in
|
|
36
|
+
* \`client\``). Only the bag's own enumerable string keys are read.
|
|
37
|
+
*/
|
|
38
|
+
export declare function refuseUnknownOptions(name: string, options: unknown, keys: readonly string[], hint?: string): void;
|
|
39
|
+
/**
|
|
40
|
+
* The option `name`'s `value`, refused with `ValidationError` unless it is absent or a boolean: a truthy string or a
|
|
41
|
+
* `1` is not taken for `true`, and `null` is not taken for absent. An absent value comes back `undefined`, for the
|
|
42
|
+
* caller's default.
|
|
43
|
+
*/
|
|
44
|
+
export declare function requireBool(name: string, value: unknown): boolean | undefined;
|
|
45
|
+
/**
|
|
46
|
+
* The option `name`'s `value`, refused with `ValidationError` unless it is absent or a string with something in it
|
|
47
|
+
* (one that is empty or only whitespace is refused, as is `null`). An absent value comes back `undefined`.
|
|
48
|
+
*/
|
|
49
|
+
export declare function requireNonEmptyString(name: string, value: unknown): string | undefined;
|
package/dist/core/codec.d.ts
CHANGED
|
@@ -103,7 +103,7 @@ export interface CodecBitmap {
|
|
|
103
103
|
/**
|
|
104
104
|
* This set as storage chunks: ascending by chunk key, empty chunks left out, each payload exactly the bytes that
|
|
105
105
|
* {@link CodecInterface.fromValues} of that chunk's low 16 bits, then {@link optimize}, then {@link serialize}
|
|
106
|
-
* would give. So a generation written from these
|
|
106
|
+
* would give. So a generation written from these holds chunk for chunk what the same ids write. It may re-encode
|
|
107
107
|
* this bitmap in place first, as `optimize()` does; membership is untouched.
|
|
108
108
|
*
|
|
109
109
|
* **Flavor-author surface, and optional.** A load from a bitmap or from serialized bytes writes these, never
|
|
@@ -1,10 +1,14 @@
|
|
|
1
|
-
/**
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
/**
|
|
2
|
+
* An expression over the operands of a batch combine (a store's `materializeMany`): a string names an operand; a node
|
|
3
|
+
* holds exactly one of `and`, `or` or `andNot` over a non-empty list of expressions. `andNot` subtracts every entry after
|
|
4
|
+
* the first from the first, so it takes at least two. Plain JSON, so it can be logged and sent between tasks.
|
|
5
|
+
*/
|
|
6
|
+
export type Expr = string | {
|
|
7
|
+
and: Expr[];
|
|
4
8
|
} | {
|
|
5
|
-
or:
|
|
9
|
+
or: Expr[];
|
|
6
10
|
} | {
|
|
7
|
-
andNot:
|
|
11
|
+
andNot: Expr[];
|
|
8
12
|
};
|
|
9
13
|
/** The most operator nodes one expression may nest. A literal that contains itself is refused by it. */
|
|
10
14
|
export declare const MAX_EXPR_DEPTH = 64;
|
|
@@ -1,6 +1,14 @@
|
|
|
1
1
|
import type { CodecBitmap, CodecInterface } from './codec.js';
|
|
2
|
-
/**
|
|
3
|
-
|
|
2
|
+
/**
|
|
3
|
+
* One record of the feed of a batch combine (a store's `materializeMany`): the ids some of the fed operands hold at one
|
|
4
|
+
* chunk key. `key` is `id >>> 16`, an integer from 0 to 65,535, never below the previous record's; a key may arrive as
|
|
5
|
+
* several consecutive records, each carrying some of the operands, and one record per key is the usual shape. `operands`
|
|
6
|
+
* maps a declared fed name to its ids at the key: a real `Uint32Array` (not an `Int32Array`, `Buffer`, array or
|
|
7
|
+
* look-alike), strictly ascending, every id inside the key. A name is given at most once per key, and an empty array is
|
|
8
|
+
* the same as leaving the name out. The record is copied as it arrives, so the caller may drop or reuse its arrays once
|
|
9
|
+
* it has yielded it.
|
|
10
|
+
*/
|
|
11
|
+
export interface MaterializeManyFeedRecord {
|
|
4
12
|
/** The chunk key, `id >>> 16`, from 0 to 65,535. Non-decreasing over the feed; a key may arrive as several records. */
|
|
5
13
|
readonly key: number;
|
|
6
14
|
/** Ids by operand name: ascending, unique, every one inside `key`. An operand is named at most once per key. */
|
|
@@ -11,7 +19,7 @@ export interface CombineManyFeed {
|
|
|
11
19
|
/** The fed operand names, declared up front. */
|
|
12
20
|
readonly names: readonly string[];
|
|
13
21
|
/** The records, read once, in key order. */
|
|
14
|
-
readonly records: AsyncIterable<
|
|
22
|
+
readonly records: AsyncIterable<MaterializeManyFeedRecord>;
|
|
15
23
|
/**
|
|
16
24
|
* The ids each fed operand holds over the whole feed, keyed by exactly `names`: an object, or a function called once
|
|
17
25
|
* after the last record. A count that differs from what the feed held refuses every fed output.
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import type { CodecBitmap, CodecInterface } from './codec.js';
|
|
2
2
|
import type { IMetricsSink } from './metrics.js';
|
|
3
3
|
import type { CombineManyFeed } from './combine-feed.js';
|
|
4
|
-
import type {
|
|
4
|
+
import type { Expr } from './combine-expr.js';
|
|
5
5
|
import type { Clock } from './determinism.js';
|
|
6
6
|
import type { Budget } from './budget.js';
|
|
7
7
|
import type { LoadGuard } from './load.js';
|
|
8
8
|
import type { GenerationMetadata, SegmentRef, StorageChunkSource } from './ports.js';
|
|
9
|
-
export type {
|
|
9
|
+
export type { Expr } from './combine-expr.js';
|
|
10
10
|
export { MAX_EXPR_DEPTH, MAX_EXPR_NODES } from './combine-expr.js';
|
|
11
|
-
export type { CombineManyFeed,
|
|
11
|
+
export type { CombineManyFeed, MaterializeManyFeedRecord } from './combine-feed.js';
|
|
12
12
|
/** The range requests a pass keeps in flight at once across all its operands, inside a client's socket pool. */
|
|
13
13
|
export declare const MAX_RANGES_IN_FLIGHT = 64;
|
|
14
14
|
/** One named stored operand, as the pass reads it. */
|
|
@@ -48,9 +48,9 @@ export interface CombineManyOperand {
|
|
|
48
48
|
}
|
|
49
49
|
/** What one output asks for. */
|
|
50
50
|
export interface CombineManyOutput<R> {
|
|
51
|
-
readonly expr:
|
|
51
|
+
readonly expr: Expr;
|
|
52
52
|
/** Subtracted from the result: a root `andNot`. */
|
|
53
|
-
readonly exclude?: readonly
|
|
53
|
+
readonly exclude?: readonly Expr[];
|
|
54
54
|
readonly allowEmpty?: boolean;
|
|
55
55
|
readonly guard?: LoadGuard;
|
|
56
56
|
readonly metadata?: GenerationMetadata;
|
|
@@ -126,8 +126,8 @@ export type CombineManyOutcome<R> = {
|
|
|
126
126
|
readonly ok: false;
|
|
127
127
|
readonly error: unknown;
|
|
128
128
|
};
|
|
129
|
-
/** What
|
|
130
|
-
export interface
|
|
129
|
+
/** What a batch combine (a store's `materializeMany`) did to one operand: see {@link MaterializeManyStats.operands}. */
|
|
130
|
+
export interface MaterializeManyOperandStats {
|
|
131
131
|
/** Whether an output of the call read it. An operand no output names is not read. */
|
|
132
132
|
readonly read: boolean;
|
|
133
133
|
/** Whether the operand was read at a pinned generation. */
|
|
@@ -155,14 +155,18 @@ export interface CombineManyOperandStats {
|
|
|
155
155
|
/** Chunk reads made of it, over all groups. */
|
|
156
156
|
readonly chunkReads: number;
|
|
157
157
|
}
|
|
158
|
-
/** What
|
|
159
|
-
export interface
|
|
158
|
+
/** What a batch combine (a store's `materializeMany`) did to one output: see {@link MaterializeManyStats.outputs}. */
|
|
159
|
+
export interface MaterializeManyOutputStats {
|
|
160
160
|
/** The names of the operands its expression reads: the stored ones in call order, then the fed ones in `feed.names` order. */
|
|
161
161
|
readonly operands: readonly string[];
|
|
162
162
|
/** The group that computed it (from 0), or `null` when it never ran. */
|
|
163
163
|
readonly group: number | null;
|
|
164
164
|
/** Clock time the group's pass began and the output's publish settled (or it failed), when it ran. */
|
|
165
165
|
readonly startedAt: number | null;
|
|
166
|
+
/**
|
|
167
|
+
* Clock time (ms) when its publish settled or failed, or `null` when the output never ran. Read from the same clock
|
|
168
|
+
* as `startedAt`.
|
|
169
|
+
*/
|
|
166
170
|
readonly endedAt: number | null;
|
|
167
171
|
/** Chunks the output holds. */
|
|
168
172
|
readonly chunks: number;
|
|
@@ -171,9 +175,14 @@ export interface CombineManyOutputStats {
|
|
|
171
175
|
/** The most its buffer held resident, in the ledger's own measure; compare with `boundBytes` to see how tight a plan was. */
|
|
172
176
|
readonly peakBytes: number;
|
|
173
177
|
}
|
|
174
|
-
|
|
178
|
+
/**
|
|
179
|
+
* What a batch combine (a store's `materializeMany`) did: the groups it ran, its requests by class, the chunk-read budget
|
|
180
|
+
* it used, the most it held resident, and, for each operand, the generation it was pinned at, began at and ended at.
|
|
181
|
+
*/
|
|
182
|
+
export interface MaterializeManyStats {
|
|
175
183
|
/** Groups run, the re-runs of outputs a lying index made too big included. */
|
|
176
184
|
readonly groups: number;
|
|
185
|
+
/** The requests the call made, by class. */
|
|
177
186
|
readonly requests: {
|
|
178
187
|
/** Range requests sent for operand chunks (retries of one request count once). */
|
|
179
188
|
readonly rangeReads: number;
|
|
@@ -205,6 +214,7 @@ export interface CombineManyStats {
|
|
|
205
214
|
readonly put: number;
|
|
206
215
|
};
|
|
207
216
|
};
|
|
217
|
+
/** The chunk-read budget: the limit in force, the reads the plan needed and the reads made. */
|
|
208
218
|
readonly budget: {
|
|
209
219
|
/** The limit used: `null` when lifted. */
|
|
210
220
|
readonly maxRequests: number | null;
|
|
@@ -213,6 +223,9 @@ export interface CombineManyStats {
|
|
|
213
223
|
/** Chunk reads made. */
|
|
214
224
|
readonly used: number;
|
|
215
225
|
};
|
|
226
|
+
/**
|
|
227
|
+
* The memory the call held: its limit, the most the ledger held at once and what the plan holds for the whole call.
|
|
228
|
+
*/
|
|
216
229
|
readonly memory: {
|
|
217
230
|
readonly maxBufferedBytes: number;
|
|
218
231
|
/** The most the ledger held at once. */
|
|
@@ -220,6 +233,7 @@ export interface CombineManyStats {
|
|
|
220
233
|
/** What the plan holds for the whole call, charged to the ledger: operand indexes, and each output's keys and expression. */
|
|
221
234
|
readonly planBytes: number;
|
|
222
235
|
};
|
|
236
|
+
/** How many chunk keys the call skipped reading. */
|
|
223
237
|
readonly chunks: {
|
|
224
238
|
/** Index keys not read because no output needed them there, summed over groups. */
|
|
225
239
|
readonly pruned: number;
|
|
@@ -233,12 +247,13 @@ export interface CombineManyStats {
|
|
|
233
247
|
readonly ids: number;
|
|
234
248
|
};
|
|
235
249
|
/** By operand name, an object with no prototype, so any name is a key. */
|
|
236
|
-
readonly operands: Readonly<Record<string,
|
|
237
|
-
|
|
250
|
+
readonly operands: Readonly<Record<string, MaterializeManyOperandStats>>;
|
|
251
|
+
/** One entry per output of the call, in the order the call gave them. */
|
|
252
|
+
readonly outputs: readonly MaterializeManyOutputStats[];
|
|
238
253
|
}
|
|
239
254
|
export interface CombineManyRun<R> {
|
|
240
255
|
readonly outputs: readonly CombineManyOutcome<R>[];
|
|
241
|
-
readonly stats:
|
|
256
|
+
readonly stats: MaterializeManyStats;
|
|
242
257
|
}
|
|
243
258
|
/** A request that passed every check that needs no I/O: what {@link runCombineMany} runs. */
|
|
244
259
|
export interface CompiledCombineMany<R> {
|
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
* the affected segment then throw. This scan
|
|
8
8
|
* detects it up front (run it at startup after a restore) instead of discovering it on the first read.
|
|
9
9
|
*
|
|
10
|
-
* Read-only; bounded fan-out. `destroyed`
|
|
11
|
-
*
|
|
10
|
+
* Read-only; bounded fan-out. `destroyed` segments are skipped: no generation resolves through one, so a missing
|
|
11
|
+
* object under it is not a torn restore. A segment whose Storage/registry can't be read this pass is recorded in
|
|
12
12
|
* `errored` (never aborts the scan). One segment can add several entries to `inconsistent` (its current object, and with
|
|
13
13
|
* `generations` one per known generation whose object is not the one recorded) and can also be in `errored`, when a
|
|
14
14
|
* read failed after those were found; with `generations` a segment costs up to one read per entry of its ledger.
|
|
@@ -35,8 +35,11 @@ import type { IKeystore } from './crypto.js';
|
|
|
35
35
|
* something a modest operator box survives. Raisable, because a ceiling you cannot lift is a landmine.
|
|
36
36
|
*/
|
|
37
37
|
export { DEFAULT_MAX_SCAN_SEGMENTS } from './registry-scan.js';
|
|
38
|
+
/** One problem the consistency check found with one segment; `issue` says which. */
|
|
38
39
|
export interface ConsistencyIssue {
|
|
40
|
+
/** The segment's name. */
|
|
39
41
|
readonly segment: string;
|
|
42
|
+
/** The segment's namespace, or `undefined` for the default one. */
|
|
40
43
|
readonly namespace?: string;
|
|
41
44
|
/**
|
|
42
45
|
* The registry's `currentGen` for the segment. For `missing-storage-generation` it is the generation whose `.crbm` is
|
|
@@ -70,11 +73,17 @@ export interface ConsistencyIssue {
|
|
|
70
73
|
}
|
|
71
74
|
/** A segment that could not be checked this pass (Storage/registry read fault) — not proof of a torn restore. */
|
|
72
75
|
export interface ConsistencyErrorEntry {
|
|
76
|
+
/** The name of the segment that could not be checked. */
|
|
73
77
|
readonly segment: string;
|
|
78
|
+
/** That segment's namespace, or `undefined` for the default one. */
|
|
74
79
|
readonly namespace?: string;
|
|
75
80
|
/** The read error's message (never the raw error, so the report stays serializable/loggable). */
|
|
76
81
|
readonly error: string;
|
|
77
82
|
}
|
|
83
|
+
/**
|
|
84
|
+
* What a consistency check found: how many segments it scanned, the issues, the segments it could not read, and, when
|
|
85
|
+
* asked for, the checks it could not make.
|
|
86
|
+
*/
|
|
78
87
|
export interface ConsistencyReport {
|
|
79
88
|
/** Registered segments scanned (a `destroyed` segment counts as scanned but is never an issue). */
|
|
80
89
|
readonly checked: number;
|
|
@@ -51,6 +51,13 @@ export interface Yielder {
|
|
|
51
51
|
* (~60 yields across a 1M-id load) while still capping any single blocking stretch in the low tens of ms.
|
|
52
52
|
*/
|
|
53
53
|
export declare const YIELD_EVERY = 1024;
|
|
54
|
+
/**
|
|
55
|
+
* Ids between a load's ingest-loop yields, sorted or not. 16x coarser than the per-chunk cadence because the per-id
|
|
56
|
+
* work is ~40 ns (a `splitId` and an array push) against ~1.3 µs per chunk — matching cadences would put the yield cost
|
|
57
|
+
* on the wrong side of the work it interrupts. Sized against the *slowest* ingest, an in-memory async generator at
|
|
58
|
+
* ~220 ns/id, so the resulting stretch stays a few ms there and well under 1 ms on a plain array.
|
|
59
|
+
*/
|
|
60
|
+
export declare const YIELD_EVERY_IDS: number;
|
|
54
61
|
/**
|
|
55
62
|
* Build a periodic yield.
|
|
56
63
|
*
|
|
@@ -377,7 +377,8 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
377
377
|
* the one read path with no resilience.
|
|
378
378
|
*
|
|
379
379
|
* **`status` is re-checked here**, on every open rather than once at pin time. A pin taken before a
|
|
380
|
-
* crypto-shred must not keep unwrapping a DEK the shred
|
|
380
|
+
* crypto-shred must not keep unwrapping a DEK whose wrapping the shred removed; a destroyed segment resolves no
|
|
381
|
+
* generation
|
|
381
382
|
* for anyone, and a pin's read of one fails. That holds from the pin's next open: a reader already open keeps
|
|
382
383
|
* the key it unwrapped until it is evicted or invalidated.
|
|
383
384
|
*
|
|
@@ -419,7 +420,8 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
419
420
|
/**
|
|
420
421
|
* Whether the segment is a registered name at all. With a registry: a row that is not a `destroyed` tombstone,
|
|
421
422
|
* including one with no generation yet (minted by `setRetention` before the first load), which somebody deliberately
|
|
422
|
-
* created.
|
|
423
|
+
* created. No generation resolves through a tombstone, so it answers `false`. Without a registry there is no
|
|
424
|
+
* registration to
|
|
423
425
|
* consult, so the honest answer is whether the bucket holds any generation of it.
|
|
424
426
|
*
|
|
425
427
|
* Deliberately NOT served from the snapshot memo: the memo resolves to `null` for both states this exists to
|
|
@@ -569,8 +571,9 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
569
571
|
*
|
|
570
572
|
* A *second* NotFound is pathological (GC outrunning resolution) and propagates rather than fabricating an
|
|
571
573
|
* absent answer — never return a wrong result. `ifGone` is returned when the segment has no committed
|
|
572
|
-
* generation to serve: storage is empty, or the row
|
|
573
|
-
*
|
|
574
|
+
* generation to serve: storage is empty, or the row is destroyed (crypto-shredded, destroyed in the clear or
|
|
575
|
+
* dropped), in which case
|
|
576
|
+
* reading empty is the documented outcome rather than a failure.
|
|
574
577
|
*
|
|
575
578
|
* **Bounded to exactly two resolve-and-open round trips.** That is a cost contract, not a detail: the retry
|
|
576
579
|
* re-reads the *registry*, so an unbounded one would hammer the shared, throttle-prone resource in a
|
package/dist/core/crypto.d.ts
CHANGED
|
@@ -8,8 +8,11 @@
|
|
|
8
8
|
*/
|
|
9
9
|
/** One AEAD result: a fresh per-message nonce, the ciphertext, and the authentication tag. */
|
|
10
10
|
export interface AeadSealed {
|
|
11
|
+
/** The fresh per-message nonce. */
|
|
11
12
|
readonly nonce: Uint8Array;
|
|
13
|
+
/** The encrypted bytes. */
|
|
12
14
|
readonly ciphertext: Uint8Array;
|
|
15
|
+
/** The authentication tag. */
|
|
13
16
|
readonly tag: Uint8Array;
|
|
14
17
|
}
|
|
15
18
|
/**
|
|
@@ -36,7 +39,9 @@ export interface Aead {
|
|
|
36
39
|
* plaintext key material — safe to store in the registry next to the (encrypted) data.
|
|
37
40
|
*/
|
|
38
41
|
export interface WrappedDek {
|
|
42
|
+
/** The id of the key-encryption key (KEK) this wrapping is under. */
|
|
39
43
|
readonly keyId: string;
|
|
44
|
+
/** The wrapped data key, an opaque base64 string. */
|
|
40
45
|
readonly wrapped: string;
|
|
41
46
|
}
|
|
42
47
|
/**
|
|
@@ -42,7 +42,8 @@ export interface Rng {
|
|
|
42
42
|
}
|
|
43
43
|
/**
|
|
44
44
|
* Injected entropy: `length` cryptographically strong random bytes, for an identity that must never repeat (a
|
|
45
|
-
* registry row's incarnation id,
|
|
45
|
+
* registry row's incarnation id, the random part every registry write adds to its token, and a generation's write
|
|
46
|
+
* identity, the 12 bytes a cleartext object's footer carries). Deliberately not an
|
|
46
47
|
* {@link Rng}: an `Rng` is seedable for simulation, and a seeded source hands every process the same "random"
|
|
47
48
|
* identity. Production wiring uses the platform's Web Crypto; a test injects a deterministic source to replay a run.
|
|
48
49
|
*/
|
package/dist/core/discover.d.ts
CHANGED
|
@@ -20,9 +20,11 @@ export interface SegmentInfo extends SegmentRef {
|
|
|
20
20
|
*/
|
|
21
21
|
readonly currentGen: number | null;
|
|
22
22
|
/**
|
|
23
|
-
* `destroyed` is a tombstone: the row stays so sweeps can still find it, and
|
|
24
|
-
*
|
|
25
|
-
*
|
|
23
|
+
* `destroyed` is a tombstone: the row stays so sweeps can still find it, and no generation resolves through it. A
|
|
24
|
+
* `dropSegment` deletes the generations (check its `generationsRemaining`); a crypto-shred leaves them in the bucket
|
|
25
|
+
* and removes the DEK wrappings from the row, which makes every copy of the bytes unreadable once no retained copy of
|
|
26
|
+
* the row still holds them, or every KEK that wrapped them is destroyed. A cleartext segment destroyed with
|
|
27
|
+
* `allowCleartext` has no key to remove: its objects stay in the bucket and stay readable.
|
|
26
28
|
*/
|
|
27
29
|
readonly status: RegistryRecord['status'];
|
|
28
30
|
}
|
package/dist/core/engine.d.ts
CHANGED
|
@@ -346,11 +346,11 @@ export declare class SegmentEngine {
|
|
|
346
346
|
* for the range's edge, never a chunk the cache holds or another operand's. The reads, the budget, the metric and the
|
|
347
347
|
* errors are the id reads' own.
|
|
348
348
|
*/
|
|
349
|
-
intersectChunks(segs: readonly SegmentRef[], options?: CombineOptions): AsyncGenerator<CombinedChunk>;
|
|
349
|
+
intersectChunks(segs: readonly SegmentRef[], options?: CombineOptions, label?: string): AsyncGenerator<CombinedChunk>;
|
|
350
350
|
/** {@link union}, as chunks: see {@link intersectChunks}. */
|
|
351
|
-
unionChunks(segs: readonly SegmentRef[], options?: CombineOptions): AsyncGenerator<CombinedChunk>;
|
|
351
|
+
unionChunks(segs: readonly SegmentRef[], options?: CombineOptions, label?: string): AsyncGenerator<CombinedChunk>;
|
|
352
352
|
/** {@link andNot}, as chunks: see {@link intersectChunks}. */
|
|
353
|
-
andNotChunks(seg: SegmentRef, excludes: readonly SegmentRef[], options?: Omit<CombineOptions, 'exclude'
|
|
353
|
+
andNotChunks(seg: SegmentRef, excludes: readonly SegmentRef[], options?: Omit<CombineOptions, 'exclude'>, label?: string): AsyncGenerator<CombinedChunk>;
|
|
354
354
|
/**
|
|
355
355
|
* Everything a combine decides before it reads a chunk: validation, each operand's keys and generation, the
|
|
356
356
|
* absent-operand refusal, key alignment, the budget and the metric. `null` when the range is empty. The per-id
|
|
@@ -546,7 +546,7 @@ export declare class SegmentEngine {
|
|
|
546
546
|
* the read listed the segment's keys: the read then lists them again, and caches as a read begun after the call.
|
|
547
547
|
*
|
|
548
548
|
* The decoded-chunk cache is keyed by generation, which handles a *publish* (the new generation misses) but
|
|
549
|
-
* not a *destruction*: a segment that was erased from, dropped,
|
|
549
|
+
* not a *destruction*: a segment that was erased from, dropped, destroyed or retired has no newer generation
|
|
550
550
|
* whose key would miss, so its decoded chunks stay resident and keep answering. That is why the erasing
|
|
551
551
|
* process itself could report `erased: true` and then still answer `true` for the same id, out of RAM, with
|
|
552
552
|
* no storage read to intercept.
|