@cloudbitmaps/core 0.18.3 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/dist/{chunk-ML6AT53O.js → chunk-TNWTIAON.js} +50 -9
  2. package/dist/chunk-TNWTIAON.js.map +7 -0
  3. package/dist/core/audit.d.ts +13 -3
  4. package/dist/core/bit-route.d.ts +5 -5
  5. package/dist/core/blob.d.ts +4 -1
  6. package/dist/core/budget.d.ts +7 -2
  7. package/dist/core/combine-feed.d.ts +3 -7
  8. package/dist/core/combine-many.d.ts +22 -29
  9. package/dist/core/consistency.d.ts +0 -1
  10. package/dist/core/cost.d.ts +13 -2
  11. package/dist/core/crbm/reader.d.ts +4 -3
  12. package/dist/core/crbm-storage-source.d.ts +86 -13
  13. package/dist/core/discover.d.ts +3 -5
  14. package/dist/core/engine.d.ts +15 -0
  15. package/dist/core/erase-id.d.ts +7 -5
  16. package/dist/core/erasure.d.ts +22 -2
  17. package/dist/core/errors.d.ts +10 -2
  18. package/dist/core/load-input.d.ts +1 -2
  19. package/dist/core/load.d.ts +48 -3
  20. package/dist/core/metrics.d.ts +19 -7
  21. package/dist/core/ports.d.ts +21 -62
  22. package/dist/core/retention-sweep.d.ts +9 -4
  23. package/dist/core/retention.d.ts +17 -0
  24. package/dist/core/retry.d.ts +6 -0
  25. package/dist/driver-kit.d.ts +1 -1
  26. package/dist/driver-kit.js +8 -101
  27. package/dist/driver-kit.js.map +2 -2
  28. package/dist/drivers/_shared/object-registry.d.ts +1 -19
  29. package/dist/drivers/_shared/registry.d.ts +5 -2
  30. package/dist/drivers/localfs/fs-util.d.ts +7 -0
  31. package/dist/drivers/localfs/registry.d.ts +1 -0
  32. package/dist/index.d.ts +5 -9
  33. package/dist/index.js +864 -600
  34. package/dist/index.js.map +4 -4
  35. package/package.json +1 -1
  36. package/dist/chunk-ML6AT53O.js.map +0 -7
  37. package/dist/core/combine-held.d.ts +0 -43
  38. package/dist/core/registry-reap.d.ts +0 -22
@@ -47,7 +47,7 @@ export type AuditEvent = {
47
47
  * downstream system reconciling "the segment should now contain X" needs as much as it needs the publish.
48
48
  *
49
49
  * `reason` is `'empty'` (an empty result over a non-empty segment, with no `allowEmpty`),
50
- * `'min-cardinality'`, `'min-retained'`, or `'superseded'` (another load took the generation number, or the
50
+ * `'min-cardinality'`, `'min-retained'`, `'max-growth'`, or `'superseded'` (another load took the generation number, or the
51
51
  * segment's registry row changed while the load was writing). `cardinality` is what the refused generation
52
52
  * would have contained, and `0` for a load that lost its generation number and wrote nothing.
53
53
  *
@@ -60,7 +60,7 @@ export type AuditEvent = {
60
60
  readonly namespace?: string;
61
61
  readonly segment: string;
62
62
  readonly generation: number;
63
- readonly reason: 'empty' | 'min-cardinality' | 'min-retained' | 'superseded';
63
+ readonly reason: 'empty' | 'min-cardinality' | 'min-retained' | 'max-growth' | 'superseded';
64
64
  readonly cardinality: number;
65
65
  readonly unanswered?: true;
66
66
  } | {
@@ -121,9 +121,19 @@ export interface IAuditSink {
121
121
  export declare const NOOP_AUDIT: IAuditSink;
122
122
  /**
123
123
  * Wrap a sink so a throwing/buggy `onEvent` can never break the lifecycle operation it observes — audit is
124
- * strictly observation. Returns {@link NOOP_AUDIT} unchanged (so the no-op case skips even the try/catch alloc).
124
+ * strictly observation. That covers an `async onEvent` that rejects, whose rejection would otherwise surface as an
125
+ * unhandled one, which ends a Node process by default. Returns {@link NOOP_AUDIT} unchanged (so the no-op case
126
+ * skips even the try/catch alloc).
125
127
  */
126
128
  export declare function safeAudit(sink: IAuditSink): IAuditSink;
129
+ /**
130
+ * Refuse an `audit` that is not a sink. One without an `onEvent` method (a bare callback, a method named `emit`) would
131
+ * receive nothing and say nothing, since {@link safeAudit} swallows what calling it throws. Called at the top of each
132
+ * entry that takes one, before anything irreversible; `undefined` and `null` read as none.
133
+ */
134
+ export declare function checkedAuditSink(sink: unknown, op: string): void;
135
+ /** Observe the rejection of what a sink's `onEvent` returned, when it returned a promise, so it is never unhandled. */
136
+ export declare function ignoreRejection(returned: unknown): void;
127
137
  /** A ready-made sink that records events into an in-memory list — handy for tests + simple audit trails. */
128
138
  export declare class RecordingAuditSink implements IAuditSink {
129
139
  private readonly recorded;
@@ -8,14 +8,14 @@ export interface BitRoute {
8
8
  }
9
9
  /** Split a u32 id into `{ chunkKey, remainder }`. Throws `ValidationError` for non-u32 input. */
10
10
  export declare function splitId(id: number): BitRoute;
11
- /**
12
- * Recombine a chunk key + remainder back into the u32 id. Precondition: `chunkKey` is a validated
13
- * 16-bit value (tier-derived keys are validated upstream in the engine before reaching here, per
14
- * invariant 5); `remainder` is masked to 16 bits.
15
- */
16
11
  /**
17
12
  * Largest legal value inside a chunk payload. A chunk covers `CHUNK_COUNT` ids, so remainders run
18
13
  * `[0, CHUNK_COUNT - 1]`. Exported so the engine can assert it on untrusted tier bytes.
19
14
  */
20
15
  export declare const MAX_REMAINDER: number;
16
+ /**
17
+ * Recombine a chunk key + remainder back into the u32 id. Precondition: `chunkKey` is a validated
18
+ * 16-bit value (tier-derived keys are validated upstream in the engine before reaching here, per
19
+ * invariant 5); `remainder` is masked to 16 bits.
20
+ */
21
21
  export declare function joinId(chunkKey: number, remainder: number): number;
@@ -1,4 +1,7 @@
1
- /** Streaming append target for a single object being written. */
1
+ /**
2
+ * Streaming append target for a single object being written. A caller may reuse the buffer it passed to `write` once
3
+ * the write resolves, so a sink that holds bytes past that copies them.
4
+ */
2
5
  export interface BlobSink {
3
6
  write(bytes: Uint8Array): Promise<void>;
4
7
  }
@@ -27,11 +27,16 @@ export interface Budget {
27
27
  export declare const DEFAULT_BUDGET: Budget;
28
28
  /** A store/per-op budget option: partial overrides, or `false` to disable entirely. */
29
29
  export type BudgetOption = Partial<Budget> | false;
30
- /** Resolve a budget option against a fallback to a concrete `Budget`, or `null` when disabled. Validates. */
30
+ /**
31
+ * Resolve a budget option against a fallback to a concrete `Budget`, or `null` when disabled. Validates: anything but
32
+ * `false`, an absent value or an object of `{ maxRequests }` is refused, since a cap of the wrong shape (`5`,
33
+ * `{ maxRequest: 5 }`) would otherwise read as no override and leave the fallback in force. `null` and a key whose value
34
+ * is `undefined` read as absent.
35
+ */
31
36
  export declare function resolveBudget(opt: BudgetOption | undefined, fallback: Budget): Budget | null;
32
37
  /**
33
38
  * Resolve a **per-op** budget override against the store's own resolved budget (never the raw global default):
34
- * `undefined` inherits the store budget as-is (including a disabled `null`); a partial `{}` / omitted
39
+ * `undefined` or `null` inherits the store budget as-is (including a disabled `null`); a partial `{}` / omitted
35
40
  * `maxRequests` inherits the store's *tightening* (so a per-op passthrough of an absent config value can't
36
41
  * silently lift a tight tenant ceiling back to the generous default); `{ maxRequests }` replaces it; `false`
37
42
  * disables. Falls back to {@link DEFAULT_BUDGET} only when the store itself has no budget (disabled).
@@ -49,13 +49,6 @@ export interface CheckedFeed {
49
49
  }
50
50
  /** Check a feed's shape, before any record is read. `stored` are the stored operand names, which a fed one may not repeat. */
51
51
  export declare function checkFeed(feed: unknown, stored: ReadonlySet<string>): CheckedFeed;
52
- /**
53
- * The view a real `Uint32Array` holds, as a plain `Uint32Array` over the same bytes, or `undefined` for anything else.
54
- * The brand is the `%TypedArray%` `@@toStringTag` getter, which reads the internal slot: it names only a real typed array
55
- * of that kind (a subclass or another realm's included) and throws for a spoof, a proxy or any other object. The view's
56
- * extent is then read through the intrinsic getters, so a subclass that overrides `length` or the iterator is never asked.
57
- */
58
- export declare function plainUint32(value: unknown): Uint32Array | 'detached' | 'out of bounds' | undefined;
59
52
  /** Thrown inside the cursor when it was closed under an operation in flight; never reaches a caller. */
60
53
  export declare class FeedClosed extends Error {
61
54
  }
@@ -128,6 +121,9 @@ export declare class FeedCursor {
128
121
  /** The next record, checked and converted (`ops` is `null` when its key is outside the range); `undefined` at the end. */
129
122
  private pull;
130
123
  private end;
124
+ /** How many records the feed has handed over, the one being read included: what names a record in an error. */
125
+ private records;
126
+ /** Refuse the record being read, named by its position in the feed: its key narrows the ids it holds to a range. */
131
127
  private refuse;
132
128
  private charge;
133
129
  private giveBack;
@@ -1,5 +1,5 @@
1
1
  import type { CodecBitmap, CodecInterface } from './codec.js';
2
- import type { HeldChunks } from './combine-held.js';
2
+ import type { IMetricsSink } from './metrics.js';
3
3
  import type { CombineManyFeed } from './combine-feed.js';
4
4
  import type { CombineExpr } from './combine-expr.js';
5
5
  import type { Clock } from './determinism.js';
@@ -18,9 +18,9 @@ export interface CombineManyOperand {
18
18
  readonly name: string;
19
19
  readonly ref: SegmentRef;
20
20
  /**
21
- * Throws the typed error when this operand's handle can no longer be read: a lease that ended or was released, or a
22
- * deadline that passed. Called before each chunk key the operand is read at, so a lapse fails the outputs that read
23
- * it and never reads empty.
21
+ * Throws the typed error when this operand's handle can no longer be read: a lease that ended or was released.
22
+ * Called before each chunk key the operand is read at, so a lapse fails the outputs that read it and never reads
23
+ * empty.
24
24
  */
25
25
  readonly check?: () => void;
26
26
  /** The generation the operand is pinned to: `null` when pinned to a segment with none, `undefined` when not pinned. */
@@ -44,25 +44,6 @@ export interface CombineManyOperand {
44
44
  readonly token: string;
45
45
  readonly fingerprint?: string | null;
46
46
  } | null>;
47
- /**
48
- * Set for an operand held in memory: its chunks are read from here, never from the source, and `ref`, `pinned*` and
49
- * `current` are not used. It has no registry row, so nothing about it can move; what can invalidate it is an erasure in
50
- * the store that made it ({@link CombineManyHeld.moved}) or its release.
51
- */
52
- readonly held?: CombineManyHeld;
53
- }
54
- /** An operand held in memory, as the pass reads it. */
55
- export interface CombineManyHeld {
56
- /** The operand's chunks. Once released, reading one throws. */
57
- readonly chunks: HeldChunks;
58
- /**
59
- * Whether an erasure has run in the store since the operand began to be made, one still running included, so what it holds may include an erased id. Checked
60
- * before the call's first request, before each chunk key is read and immediately before each publish of an output that
61
- * reads it.
62
- */
63
- readonly moved: () => boolean;
64
- /** Whether holding no id is accepted: an include then empties an AND and an exclude subtracts nothing. */
65
- readonly mayBeEmpty: boolean;
66
47
  }
67
48
  /** What one output asks for. */
68
49
  export interface CombineManyOutput<R> {
@@ -74,7 +55,7 @@ export interface CombineManyOutput<R> {
74
55
  readonly metadata?: GenerationMetadata;
75
56
  /** Overrides the call's `keep`. */
76
57
  readonly keep?: number;
77
- /** Throws to refuse the publish: the destination's lease or deadline, checked again just before it. */
58
+ /** Throws to refuse the publish: the destination's lease, checked again just before it. */
78
59
  readonly beforePublish?: () => void;
79
60
  /**
80
61
  * The requests a settled publish made that its value proves, for the call's totals by class; absent, a publish adds
@@ -116,6 +97,12 @@ export interface CombineManyRequest<R> {
116
97
  * outputs as one group; a fed output publishes only after the whole feed was read and checked.
117
98
  */
118
99
  readonly feed?: CombineManyFeed;
100
+ /**
101
+ * The outputs' publish callbacks judge their finished chunks and must write nothing. Each is still charged the object
102
+ * a write would hold, so a dry run admits, defers and refuses for memory exactly where a publish would, and none
103
+ * counts as a publish in the stats.
104
+ */
105
+ readonly dryRun?: boolean;
119
106
  }
120
107
  export interface CombineManyDeps {
121
108
  /** Reads the operands: already pinned where they are, and retrying as the store's reads do. */
@@ -123,6 +110,12 @@ export interface CombineManyDeps {
123
110
  readonly codec: CodecInterface;
124
111
  readonly clock: Clock;
125
112
  readonly maxBitmapBytes?: number;
113
+ /**
114
+ * Where the call reports a `storage.get` event for each range request it sends, as a combine does; absent, it reports
115
+ * none. The call never looks up the chunk cache, so it reports no `cache` event. A sink that throws is ignored, as
116
+ * {@link safeMetrics} makes it.
117
+ */
118
+ readonly metrics?: IMetricsSink;
126
119
  }
127
120
  /** What one output came to. */
128
121
  export type CombineManyOutcome<R> = {
@@ -262,11 +255,6 @@ export declare function compileCombineMany<R>(request: CombineManyRequest<R>): C
262
255
  * without checking the expressions again.
263
256
  */
264
257
  export declare function rebindCombineMany<R>(compiled: CompiledCombineMany<R>, operands: readonly CombineManyOperand[]): CompiledCombineMany<R>;
265
- /**
266
- * The resident bytes a group needs, kept as outputs are added so that one is priced in work proportional to its own
267
- * operands, not to the group's size: its buffered outputs, its operand streams, the chunks being evaluated, the objects
268
- * being published beside the buffers, and the key lists of its own plan.
269
- */
270
258
  /** What {@link GroupCost} reads of an output. */
271
259
  export interface GroupCostItem {
272
260
  readonly bound: number;
@@ -287,6 +275,11 @@ export declare const combineWork: {
287
275
  operandSteps: number;
288
276
  planBuilds: number;
289
277
  };
278
+ /**
279
+ * The resident bytes a group needs, kept as outputs are added so that one is priced in work proportional to its own
280
+ * operands, not to the group's size: its buffered outputs, its operand streams, the chunks being evaluated, the objects
281
+ * being published beside the buffers, and the key lists of its own plan.
282
+ */
290
283
  export declare class GroupCost {
291
284
  private readonly streamBytes;
292
285
  private readonly publishConcurrency;
@@ -18,7 +18,6 @@
18
18
  */
19
19
  import type { IStorageDriver, IRegistryDriver } from './ports.js';
20
20
  import type { IKeystore } from './crypto.js';
21
- /** Default in-flight fan-out for the consistency scan — bounded, no thundering herd. */
22
21
  /**
23
22
  * Default ceiling on how many registry records one consistency check may hold resident: 250,000.
24
23
  *
@@ -83,6 +83,8 @@ export interface RedisSizing {
83
83
  * because an `m6g` or `r6g` with the same memory costs less. It is the cheapest cluster of THIS kind; Redis bought
84
84
  * another way costs less — one replica a shard a third less, ElastiCache for Valkey 20% less a node, reserved nodes
85
85
  * less again — so against those the saving the verdict reports is smaller. Pass their prices to compare with them.
86
+ * @deprecated Moves to `@cloudbitmaps/tools`, a package of its own, with a `size` on `stat()` so a grounded report
87
+ * needs nothing internal. It is a planning tool, and its price list is as old as the release that ships it.
86
88
  */
87
89
  export declare const ELASTICACHE_REDIS_US_EAST_1_ONDEMAND: RedisSizing;
88
90
  /**
@@ -90,6 +92,9 @@ export declare const ELASTICACHE_REDIS_US_EAST_1_ONDEMAND: RedisSizing;
90
92
  * at $0.158 an hour each: 3 × 730 h × $0.158 = $346.02, published to the dollar. The per-request crossovers on the
91
93
  * benchmarks page are drawn against this one cluster, whatever the data size; pass it as `pricing.redis` to compare
92
94
  * with it. It is not the cheapest cluster for any size of data: three cache.m6g.large hold the same memory for less.
95
+ *
96
+ * @deprecated Moves to `@cloudbitmaps/tools`, a package of its own, with a `size` on `stat()` so a grounded report
97
+ * needs nothing internal. It is a planning tool, and its price list is as old as the release that ships it.
93
98
  */
94
99
  export declare const ONE_REDIS_HA_CLUSTER: Readonly<{
95
100
  monthlyUSD: number;
@@ -97,6 +102,8 @@ export declare const ONE_REDIS_HA_CLUSTER: Readonly<{
97
102
  /**
98
103
  * Default profile — **AWS us-east-1, on-demand**, mid-2026, from the fact-checked published pricing rather than
99
104
  * copied from a blog post, with Redis sized to the data. Override it for your region, cloud, or committed term.
105
+ * @deprecated Moves to `@cloudbitmaps/tools`, a package of its own, with a `size` on `stat()` so a grounded report
106
+ * needs nothing internal. It is a planning tool, and its price list is as old as the release that ships it.
100
107
  */
101
108
  export declare const AWS_US_EAST_1_ONDEMAND: PricingProfile;
102
109
  /** Sustained access pattern. All rates default to 0; unspecified ⇒ that op contributes nothing. */
@@ -186,8 +193,8 @@ export interface Workload {
186
193
  */
187
194
  readonly genTtlMs?: number;
188
195
  /**
189
- * Segments the retention sweep (`retireExpired`) retires a month. Each costs 9 registry reads, 3 writes and a
190
- * delete with {@link Workload.conditionalDelete} on, and 8 reads and 3 writes with it off, priced at the GET and PUT
196
+ * Segments the retention sweep (`retireExpired`) retires a month. Each costs 7 registry reads, 2 writes and a
197
+ * delete with {@link Workload.conditionalDelete} on, and 6 reads and 2 writes with it off, priced at the GET and PUT
191
198
  * rates, a delete unbilled, as S3 leaves it. Default
192
199
  * **0**.
193
200
  */
@@ -324,12 +331,16 @@ export declare const RETENTION_SWEEP_REQUESTS: Readonly<{
324
331
  * **Planning** cost estimate — pure, no instance or live data needed (sizing, sales, what-if). Segment sizes
325
332
  * are taken as given (or roughly derived from cardinality); use the grounded `segment.costReport()` for
326
333
  * exact, real sizes. See {@link CostReport}.
334
+ * @deprecated Moves to `@cloudbitmaps/tools`, a package of its own, with a `size` on `stat()` so a grounded report
335
+ * needs nothing internal. It is a planning tool, and its price list is as old as the release that ships it.
327
336
  */
328
337
  export declare function estimateCost(input: EstimateInput): CostReport;
329
338
  /**
330
339
  * **Grounded** report from a real segment byte total (from the `.crbm` index) + a supplied workload. Used by
331
340
  * `Segment.costReport()` in the facade. `grounded` defaults to true (the size is exact, not estimated); the
332
341
  * caller passes `grounded: false` + a note when the Storage source can't measure size.
342
+ * @deprecated Moves to `@cloudbitmaps/tools`, a package of its own, with a `size` on `stat()` so a grounded report
343
+ * needs nothing internal. It is a planning tool, and its price list is as old as the release that ships it.
333
344
  */
334
345
  export declare function groundedReport(input: {
335
346
  readonly storageBytes: number;
@@ -140,8 +140,8 @@ export declare class CrbmReader {
140
140
  /**
141
141
  * Range-GET one chunk's payload, verifying its CRC32C before returning. Returns `null` if the chunk is
142
142
  * absent. Throws `IntegrityError` on a CRC mismatch (the bytes must never reach the native
143
- * deserializer). The returned buffer is a **read-only view** owned by the reader/driver — callers must
144
- * not mutate it.
143
+ * deserializer). The returned bytes may be shared (a small generation's chunks are kept by the reader and handed out
144
+ * again), so callers must not change them.
145
145
  */
146
146
  getChunk(chunkKey: number): Promise<Uint8Array | null>;
147
147
  /**
@@ -189,7 +189,8 @@ export declare class CrbmReader {
189
189
  * others; `options.now` times each request for `options.onRequest`, which hears of every range request that was sent, once, when
190
190
  * it settles: taken, abandoned after the consumer stopped, or failed (a reader has no clock of its own, so `ms` is 0
191
191
  * without one).
192
- * A plain chunk is a writable view into the range it was read in: copy a chunk to keep it.
192
+ * A plain chunk is a view into the range it was read in, so keeping it keeps that whole range: copy a chunk to keep it
193
+ * alone. Callers must not change it, as with {@link getChunk}.
193
194
  *
194
195
  * A stream that fails (a range that errors, a chunk that fails its check) raises at once and sends nothing further:
195
196
  * the ranges it still had in flight finish in the background, are not retried and never raise. Its `settled()`
@@ -1,4 +1,5 @@
1
1
  import { type IAuditSink } from './audit.js';
2
+ import { ValidationError } from './errors.js';
2
3
  import type { Yielder } from './cooperative.js';
3
4
  import type { Clock, Rng } from './determinism.js';
4
5
  import type { PublishedKept } from './generation-gc.js';
@@ -6,7 +7,7 @@ import type { Aead, CrbmCrypto, IKeystore, WrappedDek } from './crypto.js';
6
7
  import type { ChunkRef, ChunkRead, ReadChunksOptions, StorageChunkSource, GenerationMetadata, GenerationSummary, GenKey, IStorageDriver, IRegistryDriver, RegistryRecord, RegistrySummary, SegmentRef, SegmentSize, Token } from './ports.js';
7
8
  import { CrbmReader } from './crbm/reader.js';
8
9
  import type { CrbmReaderOptions } from './crbm/reader.js';
9
- import type { CodecBitmap, CodecInterface, EncodedChunk } from './codec.js';
10
+ import type { CodecBitmap, CodecInterface } from './codec.js';
10
11
  import { ChunkLoadInput, DecodedLoadInput } from './load-input.js';
11
12
  export interface CrbmStorageChunkSourceOptions extends CrbmReaderOptions {
12
13
  /**
@@ -79,11 +80,6 @@ export interface PinnedObject {
79
80
  readonly version: string;
80
81
  readonly fingerprint?: string;
81
82
  }
82
- /**
83
- * What one resolution of a segment's pointer found: the target, and what its row says of the generation, each
84
- * computed only when asked for. The key is unwrapped once however many ask, and a failure is not remembered, so a
85
- * transient fault in the keystore is asked again.
86
- */
87
83
  /** What a pin that leases its generation does between reading the row and opening the object. */
88
84
  export interface PinLease {
89
85
  /** Write the lease against the row the pin read: its token, or `'moved'` when the pointer is no longer at the generation. */
@@ -143,7 +139,20 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
143
139
  private readonly keepChunkBytesUpTo;
144
140
  private readonly clock;
145
141
  private readonly currentGenTtlMs;
142
+ /**
143
+ * Whether the segment's pointer is re-read on a timer: it needs a clock (the TTL), a registry (the cheap row read; with
144
+ * none, resolving is a listing of the bucket) and a TTL above 0. It decides the keeping of chunk bytes, the TTL lapse
145
+ * and the refresh interval the cost report prices, so it is decided once.
146
+ */
147
+ private readonly timedRefresh;
146
148
  constructor(driver: IStorageDriver, options?: CrbmStorageChunkSourceOptions);
149
+ /**
150
+ * The last few segments' keys, newest first. A stream asks for its segment's snapshot before every cached chunk it
151
+ * serves, and a combine alternates between a few operands, so comparing two names beats encoding them again, which
152
+ * cost about a microsecond a chunk.
153
+ */
154
+ private readonly recentKeys;
155
+ private keyOf;
147
156
  /**
148
157
  * The segment's live snapshot, refreshing on the TTL. Cheap within the TTL window. The snapshot holds the resolved
149
158
  * target, and its reader once a read has asked for one.
@@ -176,6 +185,13 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
176
185
  * this.
177
186
  */
178
187
  get pointerRefreshMs(): number;
188
+ /**
189
+ * Whether a stream reading from `snap` must resolve its segment again before its next chunk, as a read of one chunk
190
+ * would: the snapshot's TTL has lapsed, an invalidation has happened since `epoch`, or the snapshot is no longer the
191
+ * one the reader cache holds for the segment (it let it go, or another read installed a newer one). Else nothing can
192
+ * have moved it, and the check costs a compare and a lookup.
193
+ */
194
+ private mayHaveMoved;
179
195
  private expired;
180
196
  /**
181
197
  * The segment's current generation number — the engine keys its chunk cache by this so a generation bump
@@ -445,13 +461,6 @@ export declare function writeCrbmGeneration(driver: IStorageDriver, key: GenKey,
445
461
  sha256: string;
446
462
  fingerprint: string;
447
463
  }>;
448
- /**
449
- * The chunks a load of ids, or of a decoded bitmap, writes: ascending by key, each non-empty and run-optimized. A decoded
450
- * bitmap writes its own chunks, which never touches an id. Ids, and the bitmap of a codec that cannot encode its own
451
- * chunks, are bucketed per chunk instead; both give the same bytes. Exported for a caller that holds the chunks itself
452
- * instead of writing them.
453
- */
454
- export declare function encodedChunksOfInput(ids: Iterable<number> | AsyncIterable<number> | DecodedLoadInput, codec: CodecInterface, clock: Clock | undefined): Promise<Iterable<EncodedChunk>>;
455
464
  /**
456
465
  * Whether the object under `key` is provably the one `fingerprint` names ({@link CrbmReader.fingerprint}), from one
457
466
  * read of its footer. False when it is another object, when it is gone, and when the read fails in any way: a caller
@@ -498,6 +507,68 @@ export declare function writeCrbmGenerationStream(driver: IStorageDriver, key: G
498
507
  clock?: Yielder;
499
508
  metadata?: GenerationMetadata;
500
509
  }): Promise<StreamWriteResult>;
510
+ /**
511
+ * Point a segment's registry `currentGen` at `key.generation` — the publish step that makes a freshly-written
512
+ * generation the authoritative latest (so registry-aware readers see it). **Forward-only and idempotent:**
513
+ * if the registry has no row it creates one; if it's already at/ahead of `key.generation` it's a no-op (an
514
+ * out-of-order/duplicate publish never regresses the pointer); otherwise it advances via compare-and-swap,
515
+ * retrying a few times under contention. Separated from the Storage write so callers can publish atomically
516
+ * after the immutable object is durable (write-then-publish).
517
+ *
518
+ * **Throws** rather than returning, in two cases, because each would otherwise create a state no reader can
519
+ * recover from: a `destroyed` (crypto-shredded) row, where advancing the pointer would name an object encrypted
520
+ * under a shredded key; and an advance carrying **new** `wrappedDeks`, which would either strand that key or make
521
+ * the row advertise encryption over cleartext generations. Contention that never clears throws
522
+ * {@link WriteConflictError} after the retries.
523
+ *
524
+ * **`expectFrom` turns forward-only into read-modify-write.** Forward-only is the right rule for a writer whose
525
+ * content does not depend on what was current — a load computes its ids upstream, so publishing over a newer
526
+ * generation loses nothing the loader knew about. It is the WRONG rule for a writer that *derived* its content
527
+ * from a particular generation: the erasure rewrite streams generation `from` and clears one bit, so publishing
528
+ * it over a newer generation silently discards whatever that generation added. Passing `expectFrom` makes the
529
+ * publish land only while the pointer is still exactly there, and return `false` otherwise, so the caller can
530
+ * report `superseded` and re-derive. The compare-and-swap itself carries the token read in the same iteration,
531
+ * so the check and the write see the same row.
532
+ *
533
+ * **`expectToken` fences the LINEAGE, and a derived writer needs both.** A generation number identifies a
534
+ * generation only within one incarnation of a name: the numbering restarts at `0` once the row is purged and
535
+ * the bucket empty, so a name that is retired and re-created has a *different* segment wearing the *same*
536
+ * `currentGen`. `expectFrom` alone matched it — and an erasure rewrite then published one incarnation's content
537
+ * over another's, deleted the live objects with its `keep: 0` collection, and returned `erased: true`. The row's
538
+ * OCC token is the identity that survives this: a shipped registry's later `create` gets a token never issued before
539
+ * under that name, but for a collision of probability 2^-128 per pair of incarnations, so no token is reused across them.
540
+ *
541
+ * Pass the token read alongside `expectFrom` and the publish lands only on the same row it was derived from.
542
+ * The check is deliberately **conservative**: the token also changes on writes that are not supersessions at
543
+ * all (a `setRetention`, a due-index reindex), so such a write makes a derived publish report `superseded` and
544
+ * the caller re-derive. That costs a re-run on a rare, unrelated write; the alternative costs a segment.
545
+ *
546
+ * **The driver sends a registry write once, and an unanswered one is reconciled by its effect.** A `create` or
547
+ * `compareAndSwap` that fails with {@link TransientError} (a throttle, a lost response, a timeout) may or may not
548
+ * have landed, and may still be in flight. The publish reads the row once more and decides from what it holds:
549
+ *
550
+ * - the pointer names `key.generation`, on the incarnation the write was made against (the row's `createdAt`), and
551
+ * `holdsOwnObject` proves the object under it the caller's: the write landed, and the publish returns `true`. Only
552
+ * the writer of a write-once object publishes its number, and a number is never taken while an object holds it;
553
+ * - the row is still the one the write was made against (the same token, or still no row): the write may land yet. The
554
+ * publish then waits, on the injected clock, and sends a **new** write from the row it just read: a request of its
555
+ * own, not a replay of the driver's, carrying the same version, so under the registry's fence at most one of the two
556
+ * lands and the other meets a conflict, which the next read settles. At most {@link UNANSWERED_RESENDS} follow, after
557
+ * a wait under 500 ms, then 1 s, then 2 s (full jitter when an `rng` is given); each is settled the same way. Still
558
+ * unanswered, or with no clock to wait on, this throws the registry's own `TransientError` and writes nothing more.
559
+ * A caller must not delete its object on it;
560
+ * - anything else: the row has moved past the state the write was conditioned on, so that write can never land, and
561
+ * the publish goes on as after a lost race, from the row it just read. A pointer at `key.generation` that is not over
562
+ * the caller's own object, or is on another incarnation, is a lost race too and answers `false`, never an already
563
+ * current re-publish.
564
+ *
565
+ * A failed read throws `TransientError` too. The same check of the pointer runs after a {@link WriteConflictError},
566
+ * so a write that landed and then met itself (a registry whose client re-sent it) is not taken for a lost race.
567
+ *
568
+ * **A `summary` goes in the same write that moves the pointer**, so a reader that sees this generation as current
569
+ * sees the count and metadata that describe it, and every attempt sends the same one. It must be the shape the row's
570
+ * keys call for: sealed for an object written with a key, clear for one written without.
571
+ */
501
572
  export declare function publishGenerationKept(registry: IRegistryDriver, key: GenKey, options?: {
502
573
  wrappedDeks?: readonly WrappedDek[];
503
574
  expectFrom?: number;
@@ -735,6 +806,8 @@ export interface LoadAhead {
735
806
  readonly aead: PromiseLike<Aead>;
736
807
  };
737
808
  }
809
+ /** What a load with `requireEncryption` meets on a segment that already has a cleartext generation. */
810
+ export declare function cleartextUnderRequiredEncryption(ref: SegmentRef, currentGen: number): ValidationError;
738
811
  /**
739
812
  * {@link bulkLoadCrbmGeneration} for a load whose number, and perhaps its key, are still being fetched: the same
740
813
  * options, with the same meaning, and the same writes. The number is joined only where the object is written.
@@ -38,10 +38,9 @@ export interface SegmentInfo extends SegmentRef {
38
38
  * distinction between "never loaded" and "loaded, and genuinely empty" is the one `count()` cannot make and
39
39
  * the main reason this function is worth having.
40
40
  *
41
- * Two states answer `true` where a read still gives you nothing, and neither is this call's job to detect: a
41
+ * One state answers `true` where a read still gives you nothing, and it is not this call's job to detect: a
42
42
  * **torn restore** (`missing-storage-generation` — a live pointer whose object was deleted) makes reads *throw*
43
- * `NotFoundError` rather than answer empty, and `runConsistencyCheck` is the call that looks for it; and a
44
- * handle carrying an expired `expiresAt` reads as empty by a rule that lives on the handle, not on the row.
43
+ * `NotFoundError` rather than answer empty, and `runConsistencyCheck` is the call that looks for it.
45
44
  *
46
45
  * Not a lock. A segment can be created or dropped between this call and whatever you do next; if the answer
47
46
  * has to hold, use the fences that exist for it — `load`'s guard, or `expectFrom`/`expectToken` on a publish.
@@ -69,8 +68,7 @@ export declare function segmentExists(ref: SegmentRef, registry: IRegistryDriver
69
68
  * disagree with the registry it reports on — a filtered enumeration that looks complete is how a retention
70
69
  * sweep ends up permanently skipping rows nobody can see. Filter on `status`/`currentGen` yourself, or use
71
70
  * {@link segmentExists} for the "does the pointer resolve" question. Internal bookkeeping rows are the one
72
- * exclusion, and only on an unscoped scan: they are not segments, and a caller naming their namespace still
73
- * sees them.
71
+ * exclusion: they are not segments, an unscoped scan skips them, and their namespace is one a caller cannot name.
74
72
  */
75
73
  export declare function listSegments(registry: IRegistryDriver, options?: {
76
74
  namespace?: string;
@@ -310,6 +310,17 @@ export declare class SegmentEngine {
310
310
  * lookup by generation number would never find a chunk cached under a stream's version.
311
311
  */
312
312
  private streamedVersion;
313
+ /**
314
+ * `hit`, a chunk cached under the version a multi-chunk read planned under, if that is still the segment's version;
315
+ * else the chunk as it is now. A read resolves the segment again before it serves each chunk, which the source's
316
+ * stream does for the chunks it delivers; a chunk served from the cache is checked here, or a read whose chunks are
317
+ * all cached would go on serving the generation it planned under, an erased id included, for as long as it is pulled.
318
+ * Within `cache.genTtlMs` the source answers from its snapshot, so the check is a lookup, not a request. `report`:
319
+ * emit the `cache` event for this lookup, which a key counted when its stream opened does not. `streamed`: the read
320
+ * this chunk belongs to, which a move carries to the version now current, so the rest of it looks there: a read whose
321
+ * stream is not yet open reads the rest as one stream, not a chunk at a time.
322
+ */
323
+ private cachedIfCurrent;
313
324
  /** The cache key of a chunk: by the version it was read under, or by segment and key alone for a source with none. */
314
325
  private chunkCacheKey;
315
326
  /** The segment's chunk keys, ascending, checked as untrusted tier data. */
@@ -376,6 +387,10 @@ export declare class SegmentEngine {
376
387
  *
377
388
  * The returned instance is **shared** (it may be the cached one, and every caller that joined a read gets the same
378
389
  * one): callers read it or clone it, never mutate it.
390
+ *
391
+ * `recheck`: `gen` was planned for a read of many chunks, so a cached chunk is served only while it is still the
392
+ * segment's version ({@link cachedIfCurrent}). A point read resolves `gen` just before it asks, and passes none.
393
+ * `report`: emit the `cache` event for this lookup; a key already counted for this read does not.
379
394
  */
380
395
  private storageChunk;
381
396
  /**
@@ -68,9 +68,10 @@
68
68
  * A live read still fetching from the collected generation heals forward to the rewrite, and a pin of it fails with
69
69
  * `NotFoundError` for any chunk it has yet to read: that is the documented cost of physical deletion on return. **Do not re-load the id while erasing it**: a load that lands after this rewrite
70
70
  * carries whatever its source held, and the library cannot know that source was meant to exclude the id.
71
- * A load already in flight is the same case: it writes its object above the pointer before it publishes, and an
72
- * erasure that finds the id in that object deletes it as it does any holder there, so a load that then publishes
73
- * leaves the pointer naming a missing object.
71
+ * A load already in flight writes its object above the pointer before it publishes. An erasure that finds the id in that
72
+ * object writes the row before deleting it, so the load's publish, fenced on the row it read, is refused
73
+ * (`published: false`) and the pointer never names a missing object (see `fenceInFlight`). A first load onto a row with
74
+ * no pointer has nothing to be fenced on, so an erasure that finds the id in its object refuses instead.
74
75
  */
75
76
  import { type IAuditSink } from './audit.js';
76
77
  import type { CodecInterface } from './codec.js';
@@ -137,8 +138,9 @@ export interface EraseIdResult {
137
138
  * re-run settles it, which is why it is the documented action for this reason and for no other.
138
139
  *
139
140
  * The first three reasons are also returned when the row changes *underneath* a call already in flight, not
140
- * only when it is read up front — a concurrent `dropSegment` gives `'destroyed'`, a retention sweep that purges
141
- * the row gives `'absent'`. A caller branching on `reason` never has to care which it was.
141
+ * only when it is read up front — a concurrent `dropSegment` gives `'destroyed'`, once the tombstone's objects have
142
+ * been searched as a fresh call searches them, and a retention sweep that purges the row gives `'absent'`. A caller
143
+ * branching on `reason` never has to care which it was.
142
144
  */
143
145
  readonly reason?: 'absent' | 'destroyed' | 'no-generation' | 'not-member' | 'superseded';
144
146
  /**
@@ -14,7 +14,7 @@
14
14
  */
15
15
  import { type IAuditSink } from './audit.js';
16
16
  import { type ChurnDeps } from './leases.js';
17
- import type { IStorageDriver, IRegistryDriver, SegmentRef } from './ports.js';
17
+ import type { IStorageDriver, IRegistryDriver, RegistryRecord, SegmentRef } from './ports.js';
18
18
  export interface EraseDeps extends ChurnDeps {
19
19
  readonly registry: IRegistryDriver;
20
20
  }
@@ -165,7 +165,7 @@ export interface DropResult {
165
165
  * but the object exists, and it holds the complete set. A single list-then-delete misses it entirely. The re-sweep
166
166
  * converges because the tombstone *is* a hard fence on **publishing**, so only loads already under way can write
167
167
  * and they are finite. One that writes after the last pass deletes its own object when its publish is refused, as a
168
- * refused load does under a `destroyed` row; only one whose process stops in between, or whose publish fails without
168
+ * refused load and a refused erasure rewrite do under a `destroyed` row; only one whose process stops in between, or whose publish fails without
169
169
  * a definite answer (a lost response, a timeout), leaves it behind, for a re-run of the drop, and
170
170
  * `generationsRemaining` cannot report an object written after this call returned.
171
171
  *
@@ -188,3 +188,23 @@ export declare function dropSegment(ref: SegmentRef, deps: DropDeps, options: {
188
188
  dryRun?: boolean;
189
189
  audit?: IAuditSink;
190
190
  }): Promise<DropResult>;
191
+ /**
192
+ * What the retention sweep adds to a drop. `wanted` is judged on every row the shred is about to replace with a
193
+ * tombstone, so a row whose expiry was extended or cleared after the sweep looked, or that someone else has already
194
+ * destroyed or purged, is left as it is rather than retired. `stamp` is the `retention` the tombstone carries, written
195
+ * in the shred's own write, so only a tombstone a sweep wrote ever carries a sweep's mark, which is what lets a later
196
+ * sweep purge it and never a crypto-shred's.
197
+ */
198
+ export interface SweepDrop {
199
+ readonly wanted: (row: RegistryRecord) => boolean;
200
+ readonly stamp: (retention: RegistryRecord['retention']) => RegistryRecord['retention'];
201
+ }
202
+ /**
203
+ * {@link dropSegment}, with what a retention sweep adds: `null` when `sweep.wanted` refused the row, with nothing
204
+ * written or deleted.
205
+ */
206
+ export declare function dropSegmentFor(ref: SegmentRef, deps: DropDeps, options: {
207
+ confirmSegment: string;
208
+ dryRun?: boolean;
209
+ audit?: IAuditSink;
210
+ }, sweep: SweepDrop | undefined): Promise<DropResult | null>;
@@ -64,8 +64,8 @@ export declare class LeaseLimitError extends CloudRoaringError {
64
64
  /**
65
65
  * An operand a call relied on changed while the call ran, so what the call built from it is not published. Carries the
66
66
  * operand's name in the call and why it is stale (`'moved'`: it was pinned at a generation and its pointer has since
67
- * moved; `'erased'`: an erasure ran in the store after a call with a feed began, or after a held operand the call reads
68
- * was made, so what the feed gave it, or what the operand holds, may hold an erased id), under the stable `code`
67
+ * moved; `'erased'`: an erasure ran in the store after a call with a feed began, so what the feed gave it may hold an
68
+ * erased id), under the stable `code`
69
69
  * `'stale-operand'`. Deterministic: run the call again against the current operand.
70
70
  */
71
71
  export declare class StaleOperandError extends CloudRoaringError {
@@ -150,4 +150,12 @@ export declare function isNotFoundError(err: unknown): err is NotFoundError;
150
150
  export declare function isIntegrityError(err: unknown): err is IntegrityError;
151
151
  /** Invalid caller input. */
152
152
  export declare function isValidationError(err: unknown): err is ValidationError;
153
+ /** This build or this store's wiring cannot do what was asked, though nothing is malformed. */
154
+ export declare function isUnsupportedError(err: unknown): err is UnsupportedError;
155
+ /** A driver lacks a capability the store's wiring needs. */
156
+ export declare function isCapabilityError(err: unknown): err is CapabilityError;
157
+ /** A call would pass its per-op request budget, so it was refused before fanning out. */
158
+ export declare function isBudgetExceededError(err: unknown): err is BudgetExceededError;
159
+ /** No key the keystore holds can unwrap an encrypted segment's data key. */
160
+ export declare function isKeyUnavailableError(err: unknown): err is KeyUnavailableError;
153
161
  export {};
@@ -57,12 +57,11 @@ export declare class ChunkLoadInput {
57
57
  * Check a load's input and decode a bitmap input, before the load makes any request.
58
58
  *
59
59
  * @returns ids unchanged, to be consumed lazily by the write, or the decoded bitmap.
60
- * @param subject names what takes the input in a refusal's message.
61
60
  * @throws {ValidationError} for a byte array passed as ids; bytes that are malformed, over the cap, or followed by
62
61
  * more bytes; a `{ bitmap }` with no `serialize` method, or whose `serialize('portable')` returns no `Uint8Array`;
63
62
  * and anything that is none of the three inputs. An error the caller's own `serialize` throws propagates as it is.
64
63
  */
65
- export declare function prepareLoadInput(input: LoadInput, codec: CodecInterface, subject?: string): Iterable<number> | AsyncIterable<number> | DecodedLoadInput;
64
+ export declare function prepareLoadInput(input: LoadInput, codec: CodecInterface): Iterable<number> | AsyncIterable<number> | DecodedLoadInput;
66
65
  /**
67
66
  * Portable bytes a caller holds, checked and decoded by the codec's safe deserializer: the one check every bitmap
68
67
  * input of a load goes through, and what a flavor offers its users to decode bytes they hold themselves.