@cloudbitmaps/core 0.18.3 → 0.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-ML6AT53O.js → chunk-RSLVTRJY.js} +50 -9
- package/dist/chunk-RSLVTRJY.js.map +7 -0
- package/dist/core/audit.d.ts +13 -3
- package/dist/core/bit-route.d.ts +5 -5
- package/dist/core/blob.d.ts +4 -1
- package/dist/core/budget.d.ts +7 -2
- package/dist/core/combine-feed.d.ts +3 -7
- package/dist/core/combine-many.d.ts +28 -29
- package/dist/core/consistency.d.ts +0 -1
- package/dist/core/cost.d.ts +13 -2
- package/dist/core/crbm/reader.d.ts +9 -3
- package/dist/core/crbm-storage-source.d.ts +115 -29
- package/dist/core/discover.d.ts +3 -5
- package/dist/core/engine.d.ts +20 -0
- package/dist/core/erase-id.d.ts +8 -5
- package/dist/core/erasure.d.ts +22 -2
- package/dist/core/errors.d.ts +10 -2
- package/dist/core/generation-gc.d.ts +2 -2
- package/dist/core/keys.d.ts +28 -0
- package/dist/core/load-input.d.ts +1 -2
- package/dist/core/load.d.ts +54 -7
- package/dist/core/metrics.d.ts +19 -7
- package/dist/core/ports.d.ts +25 -62
- package/dist/core/retention-sweep.d.ts +9 -4
- package/dist/core/retention.d.ts +17 -0
- package/dist/core/retry.d.ts +6 -0
- package/dist/driver-kit.d.ts +1 -1
- package/dist/driver-kit.js +8 -101
- package/dist/driver-kit.js.map +2 -2
- package/dist/drivers/_shared/object-registry.d.ts +1 -19
- package/dist/drivers/_shared/registry.d.ts +5 -2
- package/dist/drivers/localfs/fs-util.d.ts +7 -0
- package/dist/drivers/localfs/registry.d.ts +1 -0
- package/dist/index.d.ts +5 -9
- package/dist/index.js +975 -631
- package/dist/index.js.map +4 -4
- package/package.json +1 -1
- package/dist/chunk-ML6AT53O.js.map +0 -7
- package/dist/core/combine-held.d.ts +0 -43
- package/dist/core/registry-reap.d.ts +0 -22
package/dist/core/erase-id.d.ts
CHANGED
|
@@ -68,9 +68,11 @@
|
|
|
68
68
|
* A live read still fetching from the collected generation heals forward to the rewrite, and a pin of it fails with
|
|
69
69
|
* `NotFoundError` for any chunk it has yet to read: that is the documented cost of physical deletion on return. **Do not re-load the id while erasing it**: a load that lands after this rewrite
|
|
70
70
|
* carries whatever its source held, and the library cannot know that source was meant to exclude the id.
|
|
71
|
-
* A load already in flight
|
|
72
|
-
*
|
|
73
|
-
*
|
|
71
|
+
* A load already in flight writes its object above the pointer before it publishes. An erasure that finds the id in that
|
|
72
|
+
* object writes the row before deleting it, so the load's publish, fenced on the row it read, is refused
|
|
73
|
+
* (`published: false`) and the pointer never names a missing object (see `fenceInFlight`). On a row with no pointer
|
|
74
|
+
* the erasure writes nothing to the row, so a first load's object there may still be published; an erasure that finds
|
|
75
|
+
* the id in it refuses instead of deleting it.
|
|
74
76
|
*/
|
|
75
77
|
import { type IAuditSink } from './audit.js';
|
|
76
78
|
import type { CodecInterface } from './codec.js';
|
|
@@ -137,8 +139,9 @@ export interface EraseIdResult {
|
|
|
137
139
|
* re-run settles it, which is why it is the documented action for this reason and for no other.
|
|
138
140
|
*
|
|
139
141
|
* The first three reasons are also returned when the row changes *underneath* a call already in flight, not
|
|
140
|
-
* only when it is read up front — a concurrent `dropSegment` gives `'destroyed'`,
|
|
141
|
-
* the row gives `'absent'`. A caller
|
|
142
|
+
* only when it is read up front — a concurrent `dropSegment` gives `'destroyed'`, once the tombstone's objects have
|
|
143
|
+
* been searched as a fresh call searches them, and a retention sweep that purges the row gives `'absent'`. A caller
|
|
144
|
+
* branching on `reason` never has to care which it was.
|
|
142
145
|
*/
|
|
143
146
|
readonly reason?: 'absent' | 'destroyed' | 'no-generation' | 'not-member' | 'superseded';
|
|
144
147
|
/**
|
package/dist/core/erasure.d.ts
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
*/
|
|
15
15
|
import { type IAuditSink } from './audit.js';
|
|
16
16
|
import { type ChurnDeps } from './leases.js';
|
|
17
|
-
import type { IStorageDriver, IRegistryDriver, SegmentRef } from './ports.js';
|
|
17
|
+
import type { IStorageDriver, IRegistryDriver, RegistryRecord, SegmentRef } from './ports.js';
|
|
18
18
|
export interface EraseDeps extends ChurnDeps {
|
|
19
19
|
readonly registry: IRegistryDriver;
|
|
20
20
|
}
|
|
@@ -165,7 +165,7 @@ export interface DropResult {
|
|
|
165
165
|
* but the object exists, and it holds the complete set. A single list-then-delete misses it entirely. The re-sweep
|
|
166
166
|
* converges because the tombstone *is* a hard fence on **publishing**, so only loads already under way can write
|
|
167
167
|
* and they are finite. One that writes after the last pass deletes its own object when its publish is refused, as a
|
|
168
|
-
* refused load
|
|
168
|
+
* refused load and a refused erasure rewrite do under a `destroyed` row; only one whose process stops in between, or whose publish fails without
|
|
169
169
|
* a definite answer (a lost response, a timeout), leaves it behind, for a re-run of the drop, and
|
|
170
170
|
* `generationsRemaining` cannot report an object written after this call returned.
|
|
171
171
|
*
|
|
@@ -188,3 +188,23 @@ export declare function dropSegment(ref: SegmentRef, deps: DropDeps, options: {
|
|
|
188
188
|
dryRun?: boolean;
|
|
189
189
|
audit?: IAuditSink;
|
|
190
190
|
}): Promise<DropResult>;
|
|
191
|
+
/**
|
|
192
|
+
* What the retention sweep adds to a drop. `wanted` is judged on every row the shred is about to replace with a
|
|
193
|
+
* tombstone, so a row whose expiry was extended or cleared after the sweep looked, or that someone else has already
|
|
194
|
+
* destroyed or purged, is left as it is rather than retired. `stamp` is the `retention` the tombstone carries, written
|
|
195
|
+
* in the shred's own write, so only a tombstone a sweep wrote ever carries a sweep's mark, which is what lets a later
|
|
196
|
+
* sweep purge it and never a crypto-shred's.
|
|
197
|
+
*/
|
|
198
|
+
export interface SweepDrop {
|
|
199
|
+
readonly wanted: (row: RegistryRecord) => boolean;
|
|
200
|
+
readonly stamp: (retention: RegistryRecord['retention']) => RegistryRecord['retention'];
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* {@link dropSegment}, with what a retention sweep adds: `null` when `sweep.wanted` refused the row, with nothing
|
|
204
|
+
* written or deleted.
|
|
205
|
+
*/
|
|
206
|
+
export declare function dropSegmentFor(ref: SegmentRef, deps: DropDeps, options: {
|
|
207
|
+
confirmSegment: string;
|
|
208
|
+
dryRun?: boolean;
|
|
209
|
+
audit?: IAuditSink;
|
|
210
|
+
}, sweep: SweepDrop | undefined): Promise<DropResult | null>;
|
package/dist/core/errors.d.ts
CHANGED
|
@@ -64,8 +64,8 @@ export declare class LeaseLimitError extends CloudRoaringError {
|
|
|
64
64
|
/**
|
|
65
65
|
* An operand a call relied on changed while the call ran, so what the call built from it is not published. Carries the
|
|
66
66
|
* operand's name in the call and why it is stale (`'moved'`: it was pinned at a generation and its pointer has since
|
|
67
|
-
* moved; `'erased'`: an erasure ran in the store after a call with a feed began,
|
|
68
|
-
*
|
|
67
|
+
* moved; `'erased'`: an erasure ran in the store after a call with a feed began, so what the feed gave it may hold an
|
|
68
|
+
* erased id), under the stable `code`
|
|
69
69
|
* `'stale-operand'`. Deterministic: run the call again against the current operand.
|
|
70
70
|
*/
|
|
71
71
|
export declare class StaleOperandError extends CloudRoaringError {
|
|
@@ -150,4 +150,12 @@ export declare function isNotFoundError(err: unknown): err is NotFoundError;
|
|
|
150
150
|
export declare function isIntegrityError(err: unknown): err is IntegrityError;
|
|
151
151
|
/** Invalid caller input. */
|
|
152
152
|
export declare function isValidationError(err: unknown): err is ValidationError;
|
|
153
|
+
/** This build or this store's wiring cannot do what was asked, though nothing is malformed. */
|
|
154
|
+
export declare function isUnsupportedError(err: unknown): err is UnsupportedError;
|
|
155
|
+
/** A driver lacks a capability the store's wiring needs. */
|
|
156
|
+
export declare function isCapabilityError(err: unknown): err is CapabilityError;
|
|
157
|
+
/** A call would pass its per-op request budget, so it was refused before fanning out. */
|
|
158
|
+
export declare function isBudgetExceededError(err: unknown): err is BudgetExceededError;
|
|
159
|
+
/** No key the keystore holds can unwrap an encrypted segment's data key. */
|
|
160
|
+
export declare function isKeyUnavailableError(err: unknown): err is KeyUnavailableError;
|
|
153
161
|
export {};
|
|
@@ -45,8 +45,8 @@ export interface LoadNumber {
|
|
|
45
45
|
* `superseded`. The orphan above stays unpublished; the load whose check meets it numbers past it by the listing,
|
|
46
46
|
* and collection takes it once a generation above it is current. Like a listing's, the number can be one whose
|
|
47
47
|
* object was deleted (an erasure removes the generations above a rolled-back pointer that held the id), which is
|
|
48
|
-
* why nothing identifies a generation by its number alone: the reader cache
|
|
49
|
-
* the
|
|
48
|
+
* why nothing identifies a generation by its number alone: the reader cache and the chunk cache key on the number and
|
|
49
|
+
* the row's token, checked against the object a reader opens, and a pin on its object's fingerprint.
|
|
50
50
|
*/
|
|
51
51
|
export declare function nextLoadGeneration(ref: SegmentRef, deps: GenerationDeps, row: RegistryRecord | null): Promise<LoadNumber>;
|
|
52
52
|
/**
|
package/dist/core/keys.d.ts
CHANGED
|
@@ -1,7 +1,35 @@
|
|
|
1
1
|
import type { ChunkRef, SegmentRef } from './ports.js';
|
|
2
2
|
export declare function segmentKey(ref: SegmentRef): string;
|
|
3
3
|
export declare function segmentPrefix(ref: SegmentRef): string;
|
|
4
|
+
/**
|
|
5
|
+
* How many segments' encoded keys a {@link KeptSegmentKeys} holds before it empties: the reader cache's default
|
|
6
|
+
* segment count, a working set a long-lived reader keeps open. With 30-character names that is about 150 KB.
|
|
7
|
+
*/
|
|
8
|
+
export declare const KEPT_SEGMENT_KEYS = 1024;
|
|
9
|
+
/**
|
|
10
|
+
* Segments' encoded keys ({@link segmentKey}, or {@link segmentPrefix}: whichever `encode` gives), kept so a read that
|
|
11
|
+
* looks up chunk after chunk compares names rather than encoding them again, which costs about a third of a warm
|
|
12
|
+
* `has()`. Held by namespace and then by segment, so segments of one name in several namespaces sit side by side, and
|
|
13
|
+
* no number of segments read in turn makes every lookup encode again. Bounded: at `max` entries it empties and fills
|
|
14
|
+
* again. An encoding is the names' and nothing else, so a kept one is the one an encoding would give now.
|
|
15
|
+
*/
|
|
16
|
+
export declare class KeptSegmentKeys {
|
|
17
|
+
private readonly encode;
|
|
18
|
+
private readonly max;
|
|
19
|
+
private readonly byNamespace;
|
|
20
|
+
private kept;
|
|
21
|
+
constructor(encode: (ref: SegmentRef) => string, max?: number);
|
|
22
|
+
/** The segment's encoded key. */
|
|
23
|
+
of(ref: SegmentRef): string;
|
|
24
|
+
/** How many encoded keys are kept. */
|
|
25
|
+
get size(): number;
|
|
26
|
+
}
|
|
4
27
|
export declare function chunkRefKey(ref: ChunkRef): string;
|
|
28
|
+
/**
|
|
29
|
+
* {@link chunkRefKey}, or {@link chunkGenKey} with a `version`, from the segment's {@link segmentPrefix} already
|
|
30
|
+
* encoded: for a caller that looks up many chunks of a few segments and keeps their prefixes.
|
|
31
|
+
*/
|
|
32
|
+
export declare function chunkKeyUnder(prefix: string, chunkKey: number, version?: string | number): string;
|
|
5
33
|
/**
|
|
6
34
|
* Cache key for a chunk scoped to a specific **version** of the segment — {@link chunkRefKey} plus that
|
|
7
35
|
* version (space-delimited, injection-proof exactly as above — the name parts are already encoded). A publish bumps the version, so it naturally
|
|
@@ -57,12 +57,11 @@ export declare class ChunkLoadInput {
|
|
|
57
57
|
* Check a load's input and decode a bitmap input, before the load makes any request.
|
|
58
58
|
*
|
|
59
59
|
* @returns ids unchanged, to be consumed lazily by the write, or the decoded bitmap.
|
|
60
|
-
* @param subject names what takes the input in a refusal's message.
|
|
61
60
|
* @throws {ValidationError} for a byte array passed as ids; bytes that are malformed, over the cap, or followed by
|
|
62
61
|
* more bytes; a `{ bitmap }` with no `serialize` method, or whose `serialize('portable')` returns no `Uint8Array`;
|
|
63
62
|
* and anything that is none of the three inputs. An error the caller's own `serialize` throws propagates as it is.
|
|
64
63
|
*/
|
|
65
|
-
export declare function prepareLoadInput(input: LoadInput, codec: CodecInterface
|
|
64
|
+
export declare function prepareLoadInput(input: LoadInput, codec: CodecInterface): Iterable<number> | AsyncIterable<number> | DecodedLoadInput;
|
|
66
65
|
/**
|
|
67
66
|
* Portable bytes a caller holds, checked and decoded by the codec's safe deserializer: the one check every bitmap
|
|
68
67
|
* input of a load goes through, and what a flavor offers its users to decode bytes they hold themselves.
|
package/dist/core/load.d.ts
CHANGED
|
@@ -59,8 +59,9 @@ export interface LoadDeps {
|
|
|
59
59
|
/**
|
|
60
60
|
* What a plausible result looks like, so an implausible one is refused instead of published.
|
|
61
61
|
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
62
|
+
* `minCardinality` judges the new generation alone. `minRetained` and `maxGrowth` judge it against the **current**
|
|
63
|
+
* generation's cardinality, which the load takes from the registry row's summary of it when the row has a usable one,
|
|
64
|
+
* and otherwise from the `.crbm` index (one object-header read, never the payload), and only when a bound needs it.
|
|
64
65
|
*/
|
|
65
66
|
export interface LoadGuard {
|
|
66
67
|
/** Refuse a generation with fewer than this many ids. Use for "this segment is never legitimately tiny". */
|
|
@@ -76,7 +77,25 @@ export interface LoadGuard {
|
|
|
76
77
|
* permissive, which is where an inversion would eventually be written and not noticed.
|
|
77
78
|
*/
|
|
78
79
|
readonly minRetained?: number;
|
|
80
|
+
/**
|
|
81
|
+
* Refuse a generation larger than this multiple of the current one — `1.5` means "at most half as large again".
|
|
82
|
+
* The ceiling to `minRetained`'s floor: a source that lands duplicated or joined wrong grows a segment as quietly as
|
|
83
|
+
* a partial one shrinks it. A shrinking load is never refused by this bound, and neither is the first load of a
|
|
84
|
+
* segment nor one onto an empty segment, which have no size to multiply. `1` means "never grow".
|
|
85
|
+
*
|
|
86
|
+
* `0` means no bound, as on the two fields above. Setting it makes the load read the current size and fence its
|
|
87
|
+
* publish on it, as `minRetained` does, `allowEmpty: true` or not. The comparison is in floating point, so do not
|
|
88
|
+
* rely on the exact boundary: `100 × 1.15` is `114.99999999999999`.
|
|
89
|
+
*/
|
|
90
|
+
readonly maxGrowth?: number;
|
|
79
91
|
}
|
|
92
|
+
/**
|
|
93
|
+
* Check a guard's bounds before any round trip, and copy them: the one check `load` and each output of
|
|
94
|
+
* `materializeMany` share. The copy is what the load judges, so a guard object the caller reuses or changes while the
|
|
95
|
+
* call runs cannot change the bound it is judged by. `where` prefixes each field's name in the message (`''`, or
|
|
96
|
+
* `'outputs[3].'`).
|
|
97
|
+
*/
|
|
98
|
+
export declare function checkedGuard(guard: LoadGuard | undefined, where: string): LoadGuard | undefined;
|
|
80
99
|
export interface LoadOptions {
|
|
81
100
|
/**
|
|
82
101
|
* Publish an empty generation over a non-empty one. Off by default: an empty result is far more often an
|
|
@@ -119,15 +138,43 @@ export interface LoadOptions {
|
|
|
119
138
|
readonly metadata?: GenerationMetadata;
|
|
120
139
|
readonly audit?: IAuditSink;
|
|
121
140
|
}
|
|
141
|
+
/** The bounds a guard can refuse a generation for: every {@link LoadRefusal} but a lost race. */
|
|
142
|
+
export type GuardRefusal = Exclude<LoadRefusal, 'superseded'>;
|
|
143
|
+
/**
|
|
144
|
+
* The bound a generation of `cardinality` ids breaks over a current one of `before` (`null`: none, or not read), in the
|
|
145
|
+
* order a load checks them, or `undefined`. Pure: the one judgement a load and a dry run share.
|
|
146
|
+
*/
|
|
147
|
+
export declare function guardRefusal(cardinality: number, before: number | null, guard: LoadGuard | undefined, allowEmpty: boolean | undefined): GuardRefusal | undefined;
|
|
148
|
+
/** What a dry run learns about one write: the size it would replace, and the bound that would refuse it. */
|
|
149
|
+
export interface LoadJudgement {
|
|
150
|
+
/**
|
|
151
|
+
* The current generation's size: `null` when there is none or its object is gone, and, where no bound needs the size,
|
|
152
|
+
* when its object could not be read, as the load would not read it. Always asked for.
|
|
153
|
+
*/
|
|
154
|
+
readonly cardinalityBefore: number | null;
|
|
155
|
+
/** The bound a load of this size would be refused for now, absent when it would publish. */
|
|
156
|
+
readonly wouldRefuse?: GuardRefusal;
|
|
157
|
+
/** The requests the judgement made: the row, and the object's index when the row had no usable summary. */
|
|
158
|
+
readonly reads: number;
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Judge a write of `cardinality` ids into `ref` as a load would, and write nothing: read the row and the current size
|
|
162
|
+
* as the load reads them, and apply the same bounds. A `destroyed` segment throws the `ValidationError` a load's write
|
|
163
|
+
* would, and an encrypted one with no keystore the `KeyUnavailableError`. What it says holds for now only: a load
|
|
164
|
+
* made later is judged against the segment as it is then.
|
|
165
|
+
*/
|
|
166
|
+
export declare function judgeLoad(ref: SegmentRef, cardinality: number, deps: LoadDeps, options?: Pick<LoadOptions, 'allowEmpty' | 'guard'>): Promise<LoadJudgement>;
|
|
122
167
|
/** Why a load did not become current. */
|
|
123
|
-
export type LoadRefusal = 'empty' | 'min-cardinality' | 'min-retained'
|
|
168
|
+
export type LoadRefusal = 'empty' | 'min-cardinality' | 'min-retained' | 'max-growth'
|
|
124
169
|
/**
|
|
125
170
|
* Another writer got there first. Either another load wrote the same generation number first, so the write-once
|
|
126
171
|
* put refused this one and it wrote nothing (`size: 0`); or the segment's registry row changed while the load was
|
|
127
|
-
* writing: another load published, a retention change, a rollback or an erasure wrote the row,
|
|
128
|
-
* deleted. Once another write has changed the row, the object stays in
|
|
129
|
-
*
|
|
130
|
-
*
|
|
172
|
+
* writing: another load published, a retention change, a rollback or an erasure wrote the row, the row was
|
|
173
|
+
* deleted, or a row appeared where the load found none. Once another write has changed the row, the object stays in
|
|
174
|
+
* the bucket, still billed, until a later load's collection deletes it (at the latest the listing a load runs
|
|
175
|
+
* every sixteenth generation) or the segment is dropped, because its generation number may by then name another
|
|
176
|
+
* incarnation's live object; once the row is `destroyed` (dropped or crypto-shredded), the object is deleted, and
|
|
177
|
+
* once the row is gone, so is an object its footer proves this load's own.
|
|
131
178
|
*/
|
|
132
179
|
| 'superseded';
|
|
133
180
|
export interface LoadResult {
|
package/dist/core/metrics.d.ts
CHANGED
|
@@ -14,8 +14,15 @@
|
|
|
14
14
|
* per-series metric labels unless they're known low-cardinality and PII-free. See the observability
|
|
15
15
|
* guide.
|
|
16
16
|
*/
|
|
17
|
-
/**
|
|
18
|
-
|
|
17
|
+
/**
|
|
18
|
+
* The operations that emit an `op` latency event (timed with the injected clock, at the facade). `materializeMany` is
|
|
19
|
+
* timed from its first request, the pins, to its last publish: a call its input checks refuse emits none.
|
|
20
|
+
*
|
|
21
|
+
* A name added here stops two kinds of code compiling: a sink that switches on `name` with an exhaustive `never` check
|
|
22
|
+
* (add a case, or use a `default` branch, which needs no change), and a `Record<MetricOpName, …>` built by hand, a
|
|
23
|
+
* `MetricsSnapshot` literal among them (add the key).
|
|
24
|
+
*/
|
|
25
|
+
export type MetricOpName = 'has' | 'count' | 'intersectInto' | 'unionInto' | 'andNotInto' | 'materializeMany';
|
|
19
26
|
/**
|
|
20
27
|
* One observability event. A discriminated union on `kind` — new variants can be added over time without
|
|
21
28
|
* breaking consumers (they switch on the kinds they care about). `namespace` is the segment's namespace
|
|
@@ -27,10 +34,11 @@ export type MetricEvent = {
|
|
|
27
34
|
readonly namespace?: string;
|
|
28
35
|
readonly segment: string;
|
|
29
36
|
/**
|
|
30
|
-
* One event per request for chunks: a range request of a combine or `
|
|
31
|
-
* needs from a stretch of the object, or the one chunk of a read made on its own, however
|
|
32
|
-
* on it.
|
|
33
|
-
*
|
|
37
|
+
* One event per request for chunks: a range request of a combine, `iterate` or `materializeMany`, which carries
|
|
38
|
+
* every chunk the read needs from a stretch of the object, or the one chunk of a read made on its own, however
|
|
39
|
+
* many callers were waiting on it. A `materializeMany` range serves every output of its group, so its events
|
|
40
|
+
* count the requests the call sent, not requests per output. Bytes returned: the range's, the gaps between its
|
|
41
|
+
* chunks included (0 if a single chunk was absent — a GET still happened).
|
|
34
42
|
*/
|
|
35
43
|
readonly bytes: number;
|
|
36
44
|
/**
|
|
@@ -45,7 +53,7 @@ export type MetricEvent = {
|
|
|
45
53
|
* no cached chunk. A combine or `iterate` looks up every chunk it needs when it opens its streams. A caller that
|
|
46
54
|
* missed while another caller's read of the same chunk was open waits on that read and adds no `storage.get`, and a
|
|
47
55
|
* range carries several chunks, so the misses can outnumber the `storage.get` events; the `storage.get` count is
|
|
48
|
-
* the number of requests for chunks.
|
|
56
|
+
* the number of requests for chunks. `materializeMany` never looks up the cache, so it emits none.
|
|
49
57
|
*/
|
|
50
58
|
readonly kind: 'cache';
|
|
51
59
|
readonly hit: boolean;
|
|
@@ -57,6 +65,10 @@ export type MetricEvent = {
|
|
|
57
65
|
readonly attempt: number;
|
|
58
66
|
readonly delayMs: number;
|
|
59
67
|
} | {
|
|
68
|
+
/**
|
|
69
|
+
* One per chunk-aligned combine of one operator. `materializeMany` emits none: an output is an expression over
|
|
70
|
+
* several operators, and the call's `stats.chunks.pruned` counts the index keys it did not read.
|
|
71
|
+
*/
|
|
60
72
|
readonly kind: 'intersect';
|
|
61
73
|
/**
|
|
62
74
|
* Which chunk-aligned combine this was. **Optional** — absent means `'intersect'`.
|
package/dist/core/ports.d.ts
CHANGED
|
@@ -10,7 +10,10 @@
|
|
|
10
10
|
import type { BlobSink } from './blob.js';
|
|
11
11
|
import type { WrappedDek } from './crypto.js';
|
|
12
12
|
import type { IMetricsSink } from './metrics.js';
|
|
13
|
-
/**
|
|
13
|
+
/**
|
|
14
|
+
* Opaque optimistic-concurrency token — unique per write with overwhelming probability, since it carries random parts
|
|
15
|
+
* (see {@link RegistryRecord.token}), and compared by equality only.
|
|
16
|
+
*/
|
|
14
17
|
export type Token = string;
|
|
15
18
|
/**
|
|
16
19
|
* The address of one segment: a `namespace` and a `segment` name, both any non-empty string of at most 256
|
|
@@ -191,6 +194,10 @@ export interface StorageChunkSource {
|
|
|
191
194
|
* Note that putting the incarnation in the *object key* would not fix this: the new incarnation still starts
|
|
192
195
|
* at generation 0, so the cache key collides either way. The identity has to reach the cache.
|
|
193
196
|
*
|
|
197
|
+
* Within one incarnation a number can be taken again once its object is deleted, so two objects must never share
|
|
198
|
+
* a version either, however long after its pointer read a source opens the object: the `.crbm` source's version
|
|
199
|
+
* ends with the fingerprint of the object it opened.
|
|
200
|
+
*
|
|
194
201
|
* Returns `null` when the segment resolves to no generation. A source that only ever serves one immutable
|
|
195
202
|
* generation may omit this, and the engine falls back to the generation alone.
|
|
196
203
|
*/
|
|
@@ -214,8 +221,14 @@ export interface StorageCaps {
|
|
|
214
221
|
* shipped driver to them:
|
|
215
222
|
*
|
|
216
223
|
* - **`putImmutable` is write-once and reports a collision as {@link WriteConflictError}.** It never overwrites a
|
|
217
|
-
* stored object. `load` reads that error as a lost
|
|
218
|
-
* that threw anything else for an existing key would
|
|
224
|
+
* stored object, two concurrent writes of one key included: exactly one lands. `load` reads that error as a lost
|
|
225
|
+
* race for the generation number (`superseded`), so a driver that threw anything else for an existing key would
|
|
226
|
+
* surface as a failed load. A write whose writer fails stores nothing, and the key can be written after.
|
|
227
|
+
* - **Keys keep every name apart.** One segment in two namespaces, two names an encoding could fold into one (`a/b`
|
|
228
|
+
* and `a_b`), and a name that extends another (`s` and `s.1`) are separate objects, each listed and deleted alone;
|
|
229
|
+
* generations are listed and deleted by number, never by a prefix of one.
|
|
230
|
+
* - **A bucket or container that does not exist is not a missing object**: it fails with an error that is neither
|
|
231
|
+
* {@link NotFoundError}, which would read as an empty segment, nor a {@link TransientError}, which a retry cannot fix.
|
|
219
232
|
* - **A missing object makes `getRange` and `getTail` throw {@link NotFoundError}** — never an empty or short
|
|
220
233
|
* result. Heal-forward (a read whose generation a sweep collected re-resolves), the erasure's holder probe and
|
|
221
234
|
* its verify of the rewrite, and a pin's replaced-object check all branch on that one error. (A zero-length
|
|
@@ -234,6 +247,8 @@ export interface StorageCaps {
|
|
|
234
247
|
* holding the id, `dropSegment`'s `generationsRemaining`, the retention sweep's check that a tombstone's storage
|
|
235
248
|
* is gone, and rollback's post-move check prove a deletion or a presence by listing, so a listing that lags
|
|
236
249
|
* reports an erasure as incomplete or a stale object as gone.
|
|
250
|
+
* - **`list` yields every generation, however many pages the service splits it into.** One that stops after its
|
|
251
|
+
* first page drops generations from every collection and every check above.
|
|
237
252
|
* - **Never replay a conditional write without telling the replay apart.** A `putImmutable` that lands and then
|
|
238
253
|
* loses its response, sent again, finds its own object and would report a collision. Either send each write
|
|
239
254
|
* once, with the backend client's retry off for that request, or recognise your own write when the
|
|
@@ -486,47 +501,6 @@ export interface RegCaps {
|
|
|
486
501
|
*/
|
|
487
502
|
readonly conditionalDelete?: boolean;
|
|
488
503
|
}
|
|
489
|
-
/** What {@link IRegistryDriver.reapLegacyTombstones} is asked to do. */
|
|
490
|
-
export interface ReapLegacyTombstonesOptions {
|
|
491
|
-
/** Only rows in this namespace; every namespace, the library's bookkeeping namespaces included, when absent. */
|
|
492
|
-
readonly namespace?: string;
|
|
493
|
-
/** Count what would be removed and remove nothing. */
|
|
494
|
-
readonly dryRun: boolean;
|
|
495
|
-
/** The most rows to remove (or, in a dry run, to count as removable) in this call. */
|
|
496
|
-
readonly limit: number;
|
|
497
|
-
}
|
|
498
|
-
/** What a reap of the tombstones a registry holds found. Every count is of rows, not requests. */
|
|
499
|
-
export interface ReapLegacyTombstonesResult {
|
|
500
|
-
/** The call was a dry run: nothing was removed, and `wouldReap` says what a real run removes. */
|
|
501
|
-
readonly dryRun: boolean;
|
|
502
|
-
/** Rows read. Rows the `limit` stopped the run before reading are not counted. */
|
|
503
|
-
readonly examined: number;
|
|
504
|
-
/** Rows removed. Always 0 in a dry run. */
|
|
505
|
-
readonly reaped: number;
|
|
506
|
-
/** Rows a dry run found removable. Always 0 in a real run. */
|
|
507
|
-
readonly wouldReap: number;
|
|
508
|
-
/**
|
|
509
|
-
* The `limit` was spent and the listing had more keys: rows were left unread, and they may or may not be removable,
|
|
510
|
-
* so run again. A run is not resumable: each lists and reads from the start, so a dry run with a `limit` reports the
|
|
511
|
-
* same first rows every time, and `wouldReap` is a total only when this is `false`.
|
|
512
|
-
*/
|
|
513
|
-
readonly limited: boolean;
|
|
514
|
-
/** Rows read and left, by why. */
|
|
515
|
-
readonly skipped: {
|
|
516
|
-
/** A live (`active`, not deleted) row. */
|
|
517
|
-
readonly live: number;
|
|
518
|
-
/** A `destroyed` row that is not `deleted`: a crypto-shred's or a drop's tombstone, kept as the attestation of an erasure. */
|
|
519
|
-
readonly destroyed: number;
|
|
520
|
-
/**
|
|
521
|
-
* A deleted row that carries an incarnation id, written by a 0.12 or later release. On a registry that removes rows,
|
|
522
|
-
* `delete` and the sweep remove such a row themselves; one left while the registry could not remove rows (GCS by
|
|
523
|
-
* default, S3 on a custom endpoint) stays, and this call, which refuses on such a registry, does not remove it.
|
|
524
|
-
*/
|
|
525
|
-
readonly incarnated: number;
|
|
526
|
-
/** A row that changed, or was already gone (a 404), between its read and its delete, so the delete (fenced on the version read) refused. */
|
|
527
|
-
readonly raced: number;
|
|
528
|
-
};
|
|
529
|
-
}
|
|
530
504
|
/**
|
|
531
505
|
* Refuse, with `UnsupportedError`, a write that would write an object before its row when the registry reports it
|
|
532
506
|
* cannot write a row (see {@link RegCaps.canWrite}). Called before the write's first request.
|
|
@@ -623,8 +597,8 @@ export declare function brandAsBackend<T extends {
|
|
|
623
597
|
export declare function isStorageBackend(value: unknown): value is StorageBackend;
|
|
624
598
|
/**
|
|
625
599
|
* Per-segment registry — the authoritative source of `currentGen`, the discovery index, and the wrapped-DEK
|
|
626
|
-
* holder. One row per segment under OCC:
|
|
627
|
-
* delete→recreate).
|
|
600
|
+
* holder. One row per segment under OCC: an equality-compared {@link Token} that is never reused, with overwhelming
|
|
601
|
+
* probability (ABA-safe across delete→recreate).
|
|
628
602
|
*
|
|
629
603
|
* **What a driver must do** — callers rely on each of these, and the registry conformance suite holds every
|
|
630
604
|
* shipped driver to them:
|
|
@@ -643,7 +617,11 @@ export declare function isStorageBackend(value: unknown): value is StorageBacken
|
|
|
643
617
|
* **With `expected` it is fenced**: it lands only while the row still carries that token, and otherwise throws
|
|
644
618
|
* {@link WriteConflictError} and leaves the row.
|
|
645
619
|
* - **Reads are strongly consistent** (`RegCaps.strongRead`), and `list` yields every existing row,
|
|
646
|
-
* `destroyed` tombstones included, with every field (see {@link IRegistryDriver.list})
|
|
620
|
+
* `destroyed` tombstones included, with every field (see {@link IRegistryDriver.list}), however many pages the
|
|
621
|
+
* service splits the listing into.
|
|
622
|
+
* - **A bucket or container that does not exist is not an absent row**: a read, a listing and a write fail with an
|
|
623
|
+
* error that is neither {@link NotFoundError} nor a {@link TransientError}, and a read never answers `null`, which would
|
|
624
|
+
* read as a segment with no generation.
|
|
647
625
|
* - **Never replay a conditional write without telling the replay apart.** A `create` or `compareAndSwap`
|
|
648
626
|
* that lands and then loses its response, sent again, finds its own write and would report a conflict.
|
|
649
627
|
* Either send each write once, with the backend client's retry off for that request, or recognise your own
|
|
@@ -701,20 +679,5 @@ export interface IRegistryDriver {
|
|
|
701
679
|
* created after the decision. A third-party driver that ignores `expected` keeps the unfenced behaviour.
|
|
702
680
|
*/
|
|
703
681
|
delete(ref: SegmentRef, expected?: Token): Promise<void>;
|
|
704
|
-
/**
|
|
705
|
-
* Optional: remove the `deleted: true` rows with no incarnation id, which a release before 0.12 left, or a 0.12 or
|
|
706
|
-
* later release left by deleting a row born before 0.12, so that no later full `list` reads them. Only a registry that
|
|
707
|
-
* keeps such rows in its backend implements it; the object registries do.
|
|
708
|
-
*
|
|
709
|
-
* It removes a row only if it is `deleted: true` with no incarnation id (whatever its `status`), and only by a delete
|
|
710
|
-
* the backend applies under the version it read (see {@link RegCaps.conditionalDelete}), so a `create` that lands over
|
|
711
|
-
* the row first wins. A registry that cannot delete under that precondition throws `CapabilityError` before its first
|
|
712
|
-
* request. It never touches a live row, a `destroyed` row that is not `deleted`, or a deleted row with an incarnation
|
|
713
|
-
* id, so it does not remove every row a full `list` reads. If `conditionalDelete` was set on a backend that ignores the
|
|
714
|
-
* precondition, the delete is unfenced.
|
|
715
|
-
*
|
|
716
|
-
* Not resumable: every call lists and reads from the start (R reads for R rows, whatever the `limit`).
|
|
717
|
-
*/
|
|
718
|
-
reapLegacyTombstones?(options: ReapLegacyTombstonesOptions): Promise<ReapLegacyTombstonesResult>;
|
|
719
682
|
}
|
|
720
683
|
export {};
|
|
@@ -177,7 +177,8 @@ export type RetireEntry = {
|
|
|
177
177
|
* expiring is the silence that costs a retention commitment.
|
|
178
178
|
* `'limit'` — eligible, but this cycle's `limit` was already spent. Re-run to continue.
|
|
179
179
|
* `'tombstone-not-empty'` — a tombstone whose Storage generations are not gone even after a GC
|
|
180
|
-
* attempt, so its row is kept
|
|
180
|
+
* attempt, so its row is kept, and, unless the collection lost a race to a write of the row, a refused purge
|
|
181
|
+
* (see {@link RetireExpiredResult.purgeFaults}): the row is what keeps the segment reachable by the generation collection and
|
|
181
182
|
* refused by every writer. Several causes, all self-healing: the storage really could not be reclaimed,
|
|
182
183
|
* the collection *declined* because the row changed underneath it (`WriteConflictError`, which this
|
|
183
184
|
* sweep swallows deliberately), or this was a `dryRun`, which reports the reason without attempting the
|
|
@@ -193,7 +194,10 @@ export type RetireEntry = {
|
|
|
193
194
|
export interface RetireExpiredResult {
|
|
194
195
|
/** Registry rows enumerated. */
|
|
195
196
|
readonly scanned: number;
|
|
196
|
-
/**
|
|
197
|
+
/**
|
|
198
|
+
* Rows whose policy said "expired", counted until the `limit` stopped the scan: the first row it deferred is counted
|
|
199
|
+
* and the rows after it are not read, so with `limited: true` this is a lower bound.
|
|
200
|
+
*/
|
|
197
201
|
readonly eligible: number;
|
|
198
202
|
/**
|
|
199
203
|
* Segments **actually retired**. Zero under `dryRun` — see `wouldRetire`. Kept honest because this is the field
|
|
@@ -208,8 +212,9 @@ export interface RetireExpiredResult {
|
|
|
208
212
|
/** True when `limit` cut the cycle short — **more segments are still eligible**. Re-run. */
|
|
209
213
|
readonly limited: boolean;
|
|
210
214
|
/**
|
|
211
|
-
* Deletes this sweep attempted that
|
|
212
|
-
*
|
|
215
|
+
* Deletes this sweep attempted that were **refused for a reason other than a lost race**: a tombstone's purge
|
|
216
|
+
* (including one whose objects were still there after a collection that raised nothing), or the removal of a
|
|
217
|
+
* due-index pointer. A policy that denies delete, an Azure blob with a snapshot, or any
|
|
213
218
|
* raw provider error is one; a write that landed between the sweep's read and its fenced delete is not (that is
|
|
214
219
|
* `failed: contended` in the ledger). Each leaves its row or pointer in place, so a purge that keeps failing never
|
|
215
220
|
* frees the name.
|
package/dist/core/retention.d.ts
CHANGED
|
@@ -44,6 +44,14 @@ export interface SetRetentionResult {
|
|
|
44
44
|
*/
|
|
45
45
|
readonly indexed: boolean;
|
|
46
46
|
}
|
|
47
|
+
/**
|
|
48
|
+
* The key of the retention sweep's mark on a tombstone it wrote, which is what lets a later sweep purge the row. Only the
|
|
49
|
+
* sweep's own tombstone write sets it, and every other tombstone write removes it ({@link withoutSweepMark}), so a
|
|
50
|
+
* restored or hand-edited row cannot hand the sweep a tombstone it did not write.
|
|
51
|
+
*/
|
|
52
|
+
export declare const SWEEP_MARK = "retiredBySweepAt";
|
|
53
|
+
/** `meta` without the sweep's mark, or `undefined` when it carries none, so a write that changes nothing is not made. */
|
|
54
|
+
export declare function withoutSweepMark(meta: GovernanceMeta | undefined): GovernanceMeta | undefined | null;
|
|
47
55
|
/**
|
|
48
56
|
* Read the policy off a registry row's `retention` metadata. Returns `null` when there is no policy, and
|
|
49
57
|
* `'invalid'` when the key is present but unusable (wrong type, or below the epoch-seconds floor).
|
|
@@ -56,6 +64,15 @@ export interface SetRetentionResult {
|
|
|
56
64
|
export declare function readRetentionPolicy(meta: GovernanceMeta | undefined): RetentionPolicy | null | 'invalid';
|
|
57
65
|
/** The stored policy for one segment, or `null` if it has none (and `'invalid'` for a malformed one). */
|
|
58
66
|
export declare function getSegmentRetention(ref: SegmentRef, deps: RetentionDeps): Promise<RetentionPolicy | null | 'invalid'>;
|
|
67
|
+
/**
|
|
68
|
+
* Record when a segment becomes eligible for retirement, creating its registry row if it has none.
|
|
69
|
+
*
|
|
70
|
+
* Idempotent and safe to re-run: writing the same instant twice is a no-op in effect. Other keys already in the
|
|
71
|
+
* row's `retention` metadata are preserved — this owns one key, not the whole object.
|
|
72
|
+
*
|
|
73
|
+
* Refuses a `destroyed` segment: a tombstone has nothing left to retire, and putting a policy on one would make
|
|
74
|
+
* a sweep repeatedly "retire" bytes that are already gone.
|
|
75
|
+
*/
|
|
59
76
|
export declare function setSegmentRetention(ref: SegmentRef, deps: RetentionDeps, policy: RetentionPolicy): Promise<SetRetentionResult>;
|
|
60
77
|
/**
|
|
61
78
|
* Remove a segment's expiry so no sweep will retire it. Returns whether a policy was actually removed (`false`
|
package/dist/core/retry.d.ts
CHANGED
|
@@ -75,6 +75,12 @@ export type ReadRetry = RetryDeps & {
|
|
|
75
75
|
* The write itself is never passed here.
|
|
76
76
|
*/
|
|
77
77
|
export declare function retryRead<T>(op: () => Promise<T>, retry: ReadRetry | undefined): Promise<T>;
|
|
78
|
+
/**
|
|
79
|
+
* `policy`, checked when it is wired rather than at the first retry: a delay or factor that is not a finite number, a
|
|
80
|
+
* negative delay, a factor below 1 or a jitter mode it does not know would make the backoff `NaN`, which a timer reads
|
|
81
|
+
* as 1 ms, so bounded backoff silently became a hot retry loop.
|
|
82
|
+
*/
|
|
83
|
+
export declare function checkedRetryPolicy(policy: RetryPolicy): RetryPolicy;
|
|
78
84
|
/**
|
|
79
85
|
* Run `op`, retrying transient failures per `policy`. Resolves with `op`'s result, or rejects with the last
|
|
80
86
|
* error once attempts are exhausted (or immediately for a non-retryable error). The thrown error is always
|
package/dist/driver-kit.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
export type { GenKey, IRegistryDriver, IStorageDriver, SegmentRef, StorageBackend, StorageCaps, } from './core/ports.js';
|
|
2
2
|
export { brandAsBackend, STORAGE_BACKEND } from './core/ports.js';
|
|
3
3
|
export type { IMetricsSink, MetricEvent } from './core/metrics.js';
|
|
4
|
-
export { IntegrityError, isNotFoundError, isValidationError, isWriteConflictError, NotFoundError, TransientError, ValidationError, WriteConflictError, } from './core/errors.js';
|
|
4
|
+
export { IntegrityError, isIntegrityError, isNotFoundError, isTransientError, isValidationError, isWriteConflictError, NotFoundError, TransientError, ValidationError, WriteConflictError, } from './core/errors.js';
|
|
5
5
|
export { validateSegmentRef } from './core/validate.js';
|
|
6
6
|
export type { BlobSink } from './core/blob.js';
|
|
7
7
|
export { MAX_ROW_BYTES, ObjectStoreRegistry, ObjectVersionRaced, } from './drivers/_shared/object-registry.js';
|