@cloudbitmaps/core 0.1.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +12 -0
- package/README.md +22 -0
- package/dist/azure/index.cjs +368 -0
- package/dist/azure/index.cjs.map +1 -0
- package/dist/azure/index.d.cts +48 -0
- package/dist/azure/index.d.ts +48 -0
- package/dist/azure/index.js +304 -0
- package/dist/azure/index.js.map +1 -0
- package/dist/cassandra/index.cjs +265 -0
- package/dist/cassandra/index.cjs.map +1 -0
- package/dist/cassandra/index.d.cts +52 -0
- package/dist/cassandra/index.d.ts +52 -0
- package/dist/cassandra/index.js +204 -0
- package/dist/cassandra/index.js.map +1 -0
- package/dist/chunk-2YDULGXS.js +203 -0
- package/dist/chunk-2YDULGXS.js.map +1 -0
- package/dist/chunk-7LMLYSVJ.js +43 -0
- package/dist/chunk-7LMLYSVJ.js.map +1 -0
- package/dist/chunk-AS6ODRLT.js +6 -0
- package/dist/chunk-AS6ODRLT.js.map +1 -0
- package/dist/chunk-NUIDEEFZ.js +91 -0
- package/dist/chunk-NUIDEEFZ.js.map +1 -0
- package/dist/chunk-SNJVZ227.js +35 -0
- package/dist/chunk-SNJVZ227.js.map +1 -0
- package/dist/dynamodb/index.cjs +731 -0
- package/dist/dynamodb/index.cjs.map +1 -0
- package/dist/dynamodb/index.d.cts +106 -0
- package/dist/dynamodb/index.d.ts +106 -0
- package/dist/dynamodb/index.js +460 -0
- package/dist/dynamodb/index.js.map +1 -0
- package/dist/gcs/index.cjs +343 -0
- package/dist/gcs/index.cjs.map +1 -0
- package/dist/gcs/index.d.cts +46 -0
- package/dist/gcs/index.d.ts +46 -0
- package/dist/gcs/index.js +279 -0
- package/dist/gcs/index.js.map +1 -0
- package/dist/index.cjs +4031 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +1945 -0
- package/dist/index.d.ts +1945 -0
- package/dist/index.js +3642 -0
- package/dist/index.js.map +1 -0
- package/dist/mongodb/index.cjs +260 -0
- package/dist/mongodb/index.cjs.map +1 -0
- package/dist/mongodb/index.d.cts +45 -0
- package/dist/mongodb/index.d.ts +45 -0
- package/dist/mongodb/index.js +199 -0
- package/dist/mongodb/index.js.map +1 -0
- package/dist/mysql/index.cjs +281 -0
- package/dist/mysql/index.cjs.map +1 -0
- package/dist/mysql/index.d.cts +56 -0
- package/dist/mysql/index.d.ts +56 -0
- package/dist/mysql/index.js +211 -0
- package/dist/mysql/index.js.map +1 -0
- package/dist/ports-D3BrJ6ax.d.cts +357 -0
- package/dist/ports-D3BrJ6ax.d.ts +357 -0
- package/dist/postgres/index.cjs +273 -0
- package/dist/postgres/index.cjs.map +1 -0
- package/dist/postgres/index.d.cts +54 -0
- package/dist/postgres/index.d.ts +54 -0
- package/dist/postgres/index.js +203 -0
- package/dist/postgres/index.js.map +1 -0
- package/dist/redis/index.cjs +257 -0
- package/dist/redis/index.cjs.map +1 -0
- package/dist/redis/index.d.cts +42 -0
- package/dist/redis/index.d.ts +42 -0
- package/dist/redis/index.js +197 -0
- package/dist/redis/index.js.map +1 -0
- package/dist/s3/index.cjs +891 -0
- package/dist/s3/index.cjs.map +1 -0
- package/dist/s3/index.d.cts +112 -0
- package/dist/s3/index.d.ts +112 -0
- package/dist/s3/index.js +555 -0
- package/dist/s3/index.js.map +1 -0
- package/package.json +172 -0
|
@@ -0,0 +1,357 @@
|
|
|
1
|
+
/** Streaming append target for a single object being written. */
|
|
2
|
+
interface BlobSink {
|
|
3
|
+
write(bytes: Uint8Array): Promise<void>;
|
|
4
|
+
}
|
|
5
|
+
/**
|
|
6
|
+
* Random/tail read access to a single finished object. The total size comes back from `getTail` (the
|
|
7
|
+
* codec's entry point), so there's no separate `size()` — keeping the seam minimal for driver authors.
|
|
8
|
+
*/
|
|
9
|
+
interface BlobReader {
|
|
10
|
+
/** Exactly `length` bytes starting at `offset`. Out-of-range is a `ValidationError`, never a short read. */
|
|
11
|
+
getRange(offset: number, length: number): Promise<Uint8Array>;
|
|
12
|
+
/** The last `min(maxBytes, size)` bytes plus the total size — the one-GET footer+index path. */
|
|
13
|
+
getTail(maxBytes: number): Promise<{
|
|
14
|
+
bytes: Uint8Array;
|
|
15
|
+
size: number;
|
|
16
|
+
}>;
|
|
17
|
+
}
|
|
18
|
+
/** In-memory `BlobSink` that accumulates written bytes; `bytes()` returns the concatenation. */
|
|
19
|
+
declare class BufferSink implements BlobSink {
|
|
20
|
+
private readonly chunks;
|
|
21
|
+
private length;
|
|
22
|
+
write(bytes: Uint8Array): Promise<void>;
|
|
23
|
+
bytes(): Uint8Array;
|
|
24
|
+
}
|
|
25
|
+
/** In-memory `BlobReader` over a fixed byte buffer. */
|
|
26
|
+
declare class BufferReader implements BlobReader {
|
|
27
|
+
private readonly buffer;
|
|
28
|
+
constructor(buffer: Uint8Array);
|
|
29
|
+
getRange(offset: number, length: number): Promise<Uint8Array>;
|
|
30
|
+
getTail(maxBytes: number): Promise<{
|
|
31
|
+
bytes: Uint8Array;
|
|
32
|
+
size: number;
|
|
33
|
+
}>;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Encryption seams — the **pure** interfaces `core/` depends on so the codec can encrypt/decrypt without
|
|
38
|
+
* importing `node:crypto` (lint-banned here, like `Clock`/`Rng`/drivers). Concrete AES-256-GCM lives outside
|
|
39
|
+
* `core/` (`src/drivers/crypto.ts`) and is injected. The envelope scheme: a per-segment DEK wrapped under an
|
|
40
|
+
* operator-held KEK, so discarding the DEK crypto-shreds the segment,
|
|
41
|
+
* [DECISIONS #19]. Per-chunk AEAD over the `.crbm` payloads + index; a per-segment **DEK** wrapped (envelope
|
|
42
|
+
* encryption) under one or more **KEK**s by an {@link IKeystore}; the wrapped DEK(s) live in the registry, so
|
|
43
|
+
* crypto-shred = delete them.
|
|
44
|
+
*/
|
|
45
|
+
/** One AEAD result: a fresh per-message nonce, the ciphertext, and the authentication tag. */
|
|
46
|
+
interface AeadSealed {
|
|
47
|
+
readonly nonce: Uint8Array;
|
|
48
|
+
readonly ciphertext: Uint8Array;
|
|
49
|
+
readonly tag: Uint8Array;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Authenticated encryption bound to a single key (a segment's DEK). `aad` (associated data) is authenticated
|
|
53
|
+
* but not encrypted — the codec passes each chunk's `(namespace, segment, generation, chunkKey)` identity
|
|
54
|
+
* ({@link aadFor}), so a chunk's ciphertext can't be silently relocated to another segment/generation/chunk or
|
|
55
|
+
* the index, even by someone who can rewrite Cold objects. The seam is symmetric and self-framing-agnostic:
|
|
56
|
+
* the caller decides where the nonce/tag go (inline with a chunk, or in the footer for the index).
|
|
57
|
+
*/
|
|
58
|
+
interface Aead {
|
|
59
|
+
/** Encrypt `plaintext`, authenticating `aad`. Returns a fresh nonce + ciphertext + tag. */
|
|
60
|
+
seal(plaintext: Uint8Array, aad: Uint8Array): AeadSealed;
|
|
61
|
+
/**
|
|
62
|
+
* Decrypt + verify. Throws {@link IntegrityError} on **any** mismatch (wrong key, tampered ciphertext/tag,
|
|
63
|
+
* or `aad` that doesn't match what was sealed) — never returns wrong-but-plausible plaintext.
|
|
64
|
+
*/
|
|
65
|
+
open(sealed: AeadSealed, aad: Uint8Array): Uint8Array;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* A DEK wrapped under one KEK. A segment stores a **list** of these (one per KEK it was wrapped under — e.g.
|
|
69
|
+
* an active KEK plus an offline recovery KEK), so losing one KEK isn't fatal and KEK rotation needs no
|
|
70
|
+
* data re-encryption. `keyId` names which KEK; `wrapped` is the opaque wrapped-key blob (base64). Carries no
|
|
71
|
+
* plaintext key material — safe to store in the registry next to the (encrypted) data.
|
|
72
|
+
*/
|
|
73
|
+
interface WrappedDek {
|
|
74
|
+
readonly keyId: string;
|
|
75
|
+
readonly wrapped: string;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Envelope key management — the injected seam between the codec and however the operator holds key material
|
|
79
|
+
* (in-process BYOK by default; KMS/Vault as optional adapters later). Async so a remote keystore (KMS) fits
|
|
80
|
+
* the same interface. Never exposes raw key bytes to `core/`: it returns an {@link Aead} bound to the DEK.
|
|
81
|
+
*/
|
|
82
|
+
interface IKeystore {
|
|
83
|
+
/**
|
|
84
|
+
* Mint a fresh per-segment DEK, wrapped under the active KEK (and any recovery KEK). Returns the wrapping(s)
|
|
85
|
+
* to persist in the registry plus an {@link Aead} to encrypt this segment's chunks with.
|
|
86
|
+
*/
|
|
87
|
+
createDek(): Promise<{
|
|
88
|
+
wrapped: WrappedDek[];
|
|
89
|
+
aead: Aead;
|
|
90
|
+
}>;
|
|
91
|
+
/**
|
|
92
|
+
* Reconstruct the {@link Aead} from a segment's stored wrappings by unwrapping with any KEK currently held.
|
|
93
|
+
* Throws {@link KeyUnavailableError} if it holds none of the referenced KEKs (lost key, or a shredded
|
|
94
|
+
* segment whose wrappings were deleted).
|
|
95
|
+
*/
|
|
96
|
+
openDek(wrapped: readonly WrappedDek[]): Promise<Aead>;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* What the encrypting codec needs: the {@link Aead} for this segment's DEK plus a way to build the AAD for a
|
|
100
|
+
* given scope (`'index'` or a chunkKey). Built per (segment, generation) by the cold-source bridge so the
|
|
101
|
+
* codec itself stays segment-agnostic.
|
|
102
|
+
*/
|
|
103
|
+
interface CrbmCrypto {
|
|
104
|
+
readonly aead: Aead;
|
|
105
|
+
aadFor(scope: number | 'index'): Uint8Array;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Build the AEAD associated-data binding a chunk (or the index) to its exact location:
|
|
109
|
+
* `v1 ‖ len(namespace) ‖ namespace ‖ len(segment) ‖ segment ‖ generation ‖ scope ‖ chunkKey`. Length-prefixed
|
|
110
|
+
* so distinct `(namespace, segment)` pairs can never collide. Pure (no crypto) — just the authenticated label.
|
|
111
|
+
*/
|
|
112
|
+
declare function aadFor(ref: {
|
|
113
|
+
readonly namespace?: string;
|
|
114
|
+
readonly segment: string;
|
|
115
|
+
}, generation: number, scope: number | 'index'): Uint8Array;
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Storage-driver contracts the engine depends on.
|
|
119
|
+
*
|
|
120
|
+
* Phase 1 uses the subset needed by the in-memory engine: the per-chunk Warm store (under OCC) and a
|
|
121
|
+
* per-chunk read view of Cold. Drivers move opaque bytes + an OCC token; they never understand roaring,
|
|
122
|
+
* `adds`/`removes`, or the `.crbm` layout. The full `IColdDriver` (blob/range) + `IRegistryDriver` arrive
|
|
123
|
+
* with generations in Phase 2/3 — the `.crbm` reader will implement `ColdChunkSource`.
|
|
124
|
+
*/
|
|
125
|
+
|
|
126
|
+
/** Opaque optimistic-concurrency token — unique per write, compared by equality only. */
|
|
127
|
+
type Token = string;
|
|
128
|
+
/** Sentinel for `putConditional` meaning "the row must not exist yet" (create). */
|
|
129
|
+
declare const NO_ROW: unique symbol;
|
|
130
|
+
type NoRow = typeof NO_ROW;
|
|
131
|
+
interface SegmentRef {
|
|
132
|
+
readonly namespace?: string;
|
|
133
|
+
readonly segment: string;
|
|
134
|
+
}
|
|
135
|
+
interface ChunkRef extends SegmentRef {
|
|
136
|
+
readonly chunkKey: number;
|
|
137
|
+
}
|
|
138
|
+
/** Identifies one immutable `.crbm` object — a single generation of a segment. */
|
|
139
|
+
interface GenKey extends SegmentRef {
|
|
140
|
+
readonly generation: number;
|
|
141
|
+
}
|
|
142
|
+
interface WarmRow {
|
|
143
|
+
readonly token: Token;
|
|
144
|
+
/** Read-only: owned by the driver. Callers must NOT mutate the buffer (re-encode to write). */
|
|
145
|
+
readonly bytes: Uint8Array;
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Read consistency for a Warm fetch (gap #9). `consistent: true` (the default — omitted ⇒ true) is a
|
|
149
|
+
* strong, read-your-writes read, as the OCC read-modify-write path requires. `consistent: false` requests an
|
|
150
|
+
* **eventually-consistent** read — on DynamoDB that is ~½ the RCU cost; a driver that is always strongly
|
|
151
|
+
* consistent (in-memory, LocalFs) ignores it. The engine uses it only on read paths (`has`/`count`/`iterate`/
|
|
152
|
+
* `intersect`) when the store opts into `warmReadConsistency: 'eventual'`, trading read-after-write for cost.
|
|
153
|
+
*/
|
|
154
|
+
interface WarmReadOptions {
|
|
155
|
+
readonly consistent?: boolean;
|
|
156
|
+
}
|
|
157
|
+
/** Per-chunk Warm store under optimistic concurrency. Returned byte buffers are read-only. */
|
|
158
|
+
interface IWarmDriver {
|
|
159
|
+
get(ref: ChunkRef, opts?: WarmReadOptions): Promise<WarmRow | null>;
|
|
160
|
+
/** OCC write. `expected` = `NO_ROW` to create, else the token previously read. Throws `WriteConflictError` on mismatch. */
|
|
161
|
+
putConditional(ref: ChunkRef, bytes: Uint8Array, expected: Token | NoRow): Promise<{
|
|
162
|
+
token: Token;
|
|
163
|
+
}>;
|
|
164
|
+
/**
|
|
165
|
+
* Fenced delete — only if the stored token still equals `expected`; throws `WriteConflictError`
|
|
166
|
+
* otherwise. **Used by compaction (Phase 4); the Phase-1 engine never calls it** — drivers still
|
|
167
|
+
* implement it to satisfy the conformance suite.
|
|
168
|
+
*/
|
|
169
|
+
deleteConditional(ref: ChunkRef, expected: Token): Promise<void>;
|
|
170
|
+
/** All dirty chunks of a segment, ascending `chunkKey`. Yielded `bytes` are read-only. */
|
|
171
|
+
listChunks(ref: SegmentRef, opts?: WarmReadOptions): AsyncIterable<{
|
|
172
|
+
chunkKey: number;
|
|
173
|
+
} & WarmRow>;
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* A segment's grounded on-disk footprint (Phase 5b) — the current generation's Cold object bytes, read
|
|
177
|
+
* cheaply from the `.crbm` footer/index (no payload reads). Powers the grounded `costReport()`.
|
|
178
|
+
*/
|
|
179
|
+
interface SegmentSize {
|
|
180
|
+
readonly sizeBytes: number;
|
|
181
|
+
}
|
|
182
|
+
/** Per-chunk read view of the immutable Cold tier (implemented by the `.crbm` reader from Phase 2). */
|
|
183
|
+
interface ColdChunkSource {
|
|
184
|
+
/** Read-only bytes for the chunk, or `null` if absent. Callers must not mutate the buffer. */
|
|
185
|
+
getChunk(ref: ChunkRef): Promise<Uint8Array | null>;
|
|
186
|
+
listChunkKeys(ref: SegmentRef): Promise<number[]>;
|
|
187
|
+
/**
|
|
188
|
+
* Optional (Phase 5b): the current generation's grounded size, cheaply (from the already-parsed
|
|
189
|
+
* `.crbm` index — no payload reads), or `null` if the segment has no Cold generation. Powers the
|
|
190
|
+
* grounded `costReport()`.
|
|
191
|
+
*/
|
|
192
|
+
sizeOf?(ref: SegmentRef): Promise<SegmentSize | null>;
|
|
193
|
+
/**
|
|
194
|
+
* Optional (Phase 5c): the current generation's **per-chunk cardinality** (`chunkKey → count`), read
|
|
195
|
+
* from the already-parsed `.crbm` index with **no payload reads**, or `null` if the segment has no Cold
|
|
196
|
+
* generation. Powers the cheap `count()` — a warm-delta-free chunk is counted straight from the index
|
|
197
|
+
* instead of fetching + deserializing it. A source with no index (e.g. the in-memory source) omits this,
|
|
198
|
+
* and `count()` falls back to fetching + merging every chunk.
|
|
199
|
+
*/
|
|
200
|
+
cardinalities?(ref: SegmentRef): Promise<ReadonlyMap<number, number> | null>;
|
|
201
|
+
/**
|
|
202
|
+
* Optional (Phase B, gap #4): the segment's **current generation number** as this source resolves it right now
|
|
203
|
+
* (registry `currentGen`, or the highest cold generation), or `null` if the segment has no Cold generation. The
|
|
204
|
+
* engine keys its HOT chunk cache by this so a generation bump (a compaction commit) is observed — a new
|
|
205
|
+
* generation misses the cache instead of serving a stale decoded chunk, and an erased id can't resurrect from a
|
|
206
|
+
* cached pre-compaction chunk. Cheap: served from the source's own (short-TTL-refreshed) snapshot, **not** a
|
|
207
|
+
* fresh backend read per call. A source that pins one immutable generation for its whole lifetime and never
|
|
208
|
+
* refreshes may omit this — the engine then keys the cache without a generation, exactly as before.
|
|
209
|
+
*/
|
|
210
|
+
currentGeneration?(ref: SegmentRef): Promise<number | null>;
|
|
211
|
+
}
|
|
212
|
+
/** Capabilities a Cold driver advertises; validated at wiring time, fail-fast. */
|
|
213
|
+
interface ColdCaps {
|
|
214
|
+
/** REQUIRED — the format relies on byte-range reads. */
|
|
215
|
+
readonly rangeRead: true;
|
|
216
|
+
/** Largest single object the backend accepts (informs single-object-vs-shard, B7 — future). */
|
|
217
|
+
readonly maxObjectBytes: number;
|
|
218
|
+
/** Optional: enables the pure-object `LATEST`-pointer registry variant (S3 conditional put). */
|
|
219
|
+
readonly conditionalPut?: boolean;
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* Immutable object storage for `.crbm` generations. A "dumb byte mover": it
|
|
223
|
+
* understands neither roaring nor the `.crbm` layout, only opaque bytes addressed by a {@link GenKey}.
|
|
224
|
+
* The core never reuses a key, so puts are write-once.
|
|
225
|
+
*/
|
|
226
|
+
interface IColdDriver {
|
|
227
|
+
capabilities(): ColdCaps;
|
|
228
|
+
/**
|
|
229
|
+
* Stream a new immutable generation. The driver opens a destination, hands `write` a {@link BlobSink},
|
|
230
|
+
* then atomically commits (and computes the content hash). Throws if the key already exists (write-once).
|
|
231
|
+
*/
|
|
232
|
+
putImmutable(key: GenKey, write: (sink: BlobSink) => Promise<void>): Promise<{
|
|
233
|
+
size: number;
|
|
234
|
+
sha256: string;
|
|
235
|
+
}>;
|
|
236
|
+
/** Range read; the caller bounds-checks. Out-of-range is rejected, never a short/adjacent read. */
|
|
237
|
+
getRange(key: GenKey, offset: number, length: number): Promise<Uint8Array>;
|
|
238
|
+
/** Speculative tail read: the last `min(maxBytes, size)` bytes + the total object size. */
|
|
239
|
+
getTail(key: GenKey, maxBytes: number): Promise<{
|
|
240
|
+
bytes: Uint8Array;
|
|
241
|
+
size: number;
|
|
242
|
+
}>;
|
|
243
|
+
delete(key: GenKey): Promise<void>;
|
|
244
|
+
/** Enumerate the generations present for a segment (orphan sweep / latest-gen resolution). */
|
|
245
|
+
list(ref: SegmentRef): AsyncIterable<GenKey>;
|
|
246
|
+
}
|
|
247
|
+
/**
|
|
248
|
+
* Lifecycle status of a segment. `active` is the steady state;
|
|
249
|
+
* `compacting`/`erasing` are transient flags a daemon sets while it works (Phase 4d/4e); `destroyed` is the
|
|
250
|
+
* post-crypto-shred tombstone (the row is kept for audit but the segment is logically gone). The 4c registry
|
|
251
|
+
* stores and round-trips the field; the *transitions* are driven by their owning features.
|
|
252
|
+
*/
|
|
253
|
+
type RegistryStatus = 'active' | 'compacting' | 'erasing' | 'destroyed';
|
|
254
|
+
/** Free-form, JSON-serializable governance metadata (retention/residency policy); shape lands in Phase 6. */
|
|
255
|
+
type GovernanceMeta = Record<string, unknown>;
|
|
256
|
+
/**
|
|
257
|
+
* One registry row — the authoritative per-segment record. Exactly one per segment.
|
|
258
|
+
*/
|
|
259
|
+
interface RegistryRecord extends SegmentRef {
|
|
260
|
+
/** **The** authoritative LATEST pointer: which immutable Cold generation is current. */
|
|
261
|
+
readonly currentGen: number;
|
|
262
|
+
/**
|
|
263
|
+
* Per-segment data-key (DEK) wrappings for encryption-at-rest (Phase 4e): the DEK envelope-wrapped under one
|
|
264
|
+
* or more KEKs (active + optional recovery). Reading unwraps with any held KEK; **crypto-shred deletes this
|
|
265
|
+
* whole list**, making the segment's at-rest bytes permanently unrecoverable. Absent ⇒ the segment is
|
|
266
|
+
* cleartext. See [DECISIONS #19].
|
|
267
|
+
*/
|
|
268
|
+
readonly wrappedDeks?: readonly WrappedDek[];
|
|
269
|
+
/**
|
|
270
|
+
* Optional **external** key reference (reserved) — e.g. a KMS key ARN for a future KMS keystore adapter that
|
|
271
|
+
* keeps wrapped material in the KMS rather than in-band {@link wrappedDeks}. Unused by the in-process BYOK
|
|
272
|
+
* keystore. Clearable on crypto-shred.
|
|
273
|
+
*/
|
|
274
|
+
readonly keyId?: string;
|
|
275
|
+
/**
|
|
276
|
+
* Dirty-row hint written by `findCompactable`. Discovery still recomputes the live dirty count by scanning
|
|
277
|
+
* Warm each cycle (the count that decides candidacy), but it now **reads** this hint to gate a change-guarded
|
|
278
|
+
* CAS — the hint is only rewritten when the count actually moved, so an all-idle fleet issues no per-segment
|
|
279
|
+
* registry writes. It does not yet drive candidacy — turning the Warm scan itself into O(dirty) is a
|
|
280
|
+
* deferred cheap-enumeration fix.
|
|
281
|
+
*/
|
|
282
|
+
readonly dirtyChunkCount: number;
|
|
283
|
+
readonly status: RegistryStatus;
|
|
284
|
+
/**
|
|
285
|
+
* Daemon health (Phase D, gap #2): epoch-ms of the last **successfully committed** compaction, and the
|
|
286
|
+
* count of **consecutive** compaction failures (reset to 0 by a successful commit). `lastCompactedAt`
|
|
287
|
+
* powers a dead-man's-switch / staleness alarm; `consecutiveFailures` drives **poison-segment quarantine** —
|
|
288
|
+
* discovery skips a segment past a failure threshold so one corrupt Warm row can't freeze the compaction of a
|
|
289
|
+
* segment **that has a committed generation** forever (with no alarm) while its Warm backlog grows unbounded.
|
|
290
|
+
* (A segment whose *very first* compaction keeps failing has no registry row to count against yet —
|
|
291
|
+
* bootstrap faults are surfaced as `error` results but not yet quarantined; gap #2.) Both
|
|
292
|
+
* **optional** (absent on rows written before Phase D, and on a segment never compacted) — read them as
|
|
293
|
+
* `lastCompactedAt ?? undefined`, `consecutiveFailures ?? 0`.
|
|
294
|
+
*/
|
|
295
|
+
readonly lastCompactedAt?: number;
|
|
296
|
+
readonly consecutiveFailures?: number;
|
|
297
|
+
/**
|
|
298
|
+
* Compaction lease (Phase 4d): the opaque id of the worker currently compacting this segment, and the
|
|
299
|
+
* epoch-ms at which its lease expires. `status === 'compacting'` with `leaseExpiresAt` in the future means
|
|
300
|
+
* a live daemon owns it; once `leaseExpiresAt` is past, another worker may **steal** the lease (the prior
|
|
301
|
+
* holder crashed). Both absent in the steady (`active`) state. The lease is an efficiency guard only — the
|
|
302
|
+
* 2-phase commit is correct without it (OCC-fenced swap + write-once generations + version-fenced purge).
|
|
303
|
+
*/
|
|
304
|
+
readonly leaseOwner?: string;
|
|
305
|
+
readonly leaseExpiresAt?: number;
|
|
306
|
+
/** Governance policy (Phase 6); stored + round-tripped now, semantics later. */
|
|
307
|
+
readonly retention?: GovernanceMeta;
|
|
308
|
+
readonly residency?: GovernanceMeta;
|
|
309
|
+
/** Epoch-ms of creation / last mutation (from the driver's injected clock). */
|
|
310
|
+
readonly createdAt: number;
|
|
311
|
+
readonly updatedAt: number;
|
|
312
|
+
/** Opaque OCC token — compare-by-equality, never reused (ABA-safe), exactly like {@link WarmRow.token}. */
|
|
313
|
+
readonly token: Token;
|
|
314
|
+
}
|
|
315
|
+
/** The caller-settable fields at {@link IRegistryDriver.create} (audit + token are driver-managed). */
|
|
316
|
+
interface NewRegistryRecord {
|
|
317
|
+
readonly currentGen: number;
|
|
318
|
+
readonly wrappedDeks?: readonly WrappedDek[];
|
|
319
|
+
readonly keyId?: string;
|
|
320
|
+
/** Defaults to 0. */
|
|
321
|
+
readonly dirtyChunkCount?: number;
|
|
322
|
+
/** Defaults to `'active'`. */
|
|
323
|
+
readonly status?: RegistryStatus;
|
|
324
|
+
readonly retention?: GovernanceMeta;
|
|
325
|
+
readonly residency?: GovernanceMeta;
|
|
326
|
+
}
|
|
327
|
+
/** Fields a {@link IRegistryDriver.compareAndSwap} may mutate (identity + audit + token are off-limits). */
|
|
328
|
+
type RegistryPatch = Partial<Pick<RegistryRecord, 'currentGen' | 'wrappedDeks' | 'keyId' | 'dirtyChunkCount' | 'status' | 'leaseOwner' | 'leaseExpiresAt' | 'lastCompactedAt' | 'consecutiveFailures' | 'retention' | 'residency'>>;
|
|
329
|
+
/** Capabilities a registry driver advertises; validated fail-fast at wiring time. */
|
|
330
|
+
interface RegCaps {
|
|
331
|
+
/** REQUIRED — `currentGen` feeds read correctness + compaction CAS, so reads must be strongly consistent. */
|
|
332
|
+
readonly strongRead: true;
|
|
333
|
+
}
|
|
334
|
+
/**
|
|
335
|
+
* Per-segment registry — the authoritative source of `currentGen`, the discovery index, and (Phase 4e) the
|
|
336
|
+
* wrapped-DEK holder. One row per segment under OCC: a
|
|
337
|
+
* never-reused, equality-compared {@link Token} (ABA-safe across delete→recreate, like the Warm tier).
|
|
338
|
+
*/
|
|
339
|
+
interface IRegistryDriver {
|
|
340
|
+
capabilities(): RegCaps;
|
|
341
|
+
/** The segment's record, or `null` if it doesn't exist (or was deleted). */
|
|
342
|
+
get(ref: SegmentRef): Promise<RegistryRecord | null>;
|
|
343
|
+
/** Create the row; throws {@link WriteConflictError} if it already exists (use CAS to mutate). */
|
|
344
|
+
create(ref: SegmentRef, record: NewRegistryRecord): Promise<{
|
|
345
|
+
token: Token;
|
|
346
|
+
}>;
|
|
347
|
+
/** Server-side compare-and-set: apply `patch` iff the stored token equals `expected`, else `WriteConflictError`. */
|
|
348
|
+
compareAndSwap(ref: SegmentRef, expected: Token, patch: RegistryPatch): Promise<{
|
|
349
|
+
token: Token;
|
|
350
|
+
}>;
|
|
351
|
+
/** Discovery: every live record, optionally scoped to one namespace. Order is unspecified. */
|
|
352
|
+
list(namespace?: string): AsyncIterable<RegistryRecord>;
|
|
353
|
+
/** Remove the row (tombstoned for ABA-safety — a later `create` still gets a fresh, greater token). */
|
|
354
|
+
delete(ref: SegmentRef): Promise<void>;
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
export { type Aead as A, type BlobSink as B, type ColdCaps as C, type GenKey as G, type IColdDriver as I, type NoRow as N, type RegCaps as R, type SegmentRef as S, type Token as T, type WarmReadOptions as W, type IWarmDriver as a, type ChunkRef as b, type WarmRow as c, type IRegistryDriver as d, type RegistryRecord as e, type NewRegistryRecord as f, type RegistryPatch as g, type ColdChunkSource as h, type SegmentSize as i, type IKeystore as j, type CrbmCrypto as k, type BlobReader as l, type WrappedDek as m, type AeadSealed as n, BufferReader as o, BufferSink as p, type GovernanceMeta as q, NO_ROW as r, type RegistryStatus as s, aadFor as t };
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var crypto = require('crypto');
|
|
4
|
+
|
|
5
|
+
// src/drivers/postgres/warm.ts
|
|
6
|
+
|
|
7
|
+
// src/core/errors.ts
|
|
8
|
+
var ERROR_BRAND = /* @__PURE__ */ Symbol.for("cloud-roaring.error");
|
|
9
|
+
var TRANSIENT_BRAND = /* @__PURE__ */ Symbol.for("cloud-roaring.error.transient");
|
|
10
|
+
var CloudRoaringError = class extends Error {
|
|
11
|
+
/** Cross-bundle brand — see the predicates ({@link isCloudRoaringError}, …). Non-enumerable-ish (symbol key ⇒ not in JSON). */
|
|
12
|
+
[ERROR_BRAND] = true;
|
|
13
|
+
constructor(message) {
|
|
14
|
+
super(message);
|
|
15
|
+
this.name = new.target.name;
|
|
16
|
+
}
|
|
17
|
+
};
|
|
18
|
+
var ValidationError = class extends CloudRoaringError {
|
|
19
|
+
};
|
|
20
|
+
var WriteConflictError = class extends CloudRoaringError {
|
|
21
|
+
};
|
|
22
|
+
var IntegrityError = class extends CloudRoaringError {
|
|
23
|
+
};
|
|
24
|
+
var TransientError = class extends CloudRoaringError {
|
|
25
|
+
/** A second brand so the whole transient subtree (incl. {@link TimeoutError}) is classifiable cross-bundle. */
|
|
26
|
+
[TRANSIENT_BRAND] = true;
|
|
27
|
+
constructor(message, options) {
|
|
28
|
+
super(message);
|
|
29
|
+
if (options && "cause" in options) this.cause = options.cause;
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
function hasBrand(err, brand) {
|
|
33
|
+
return typeof err === "object" && err !== null && err[brand] === true;
|
|
34
|
+
}
|
|
35
|
+
function isCloudRoaringError(err) {
|
|
36
|
+
return hasBrand(err, ERROR_BRAND);
|
|
37
|
+
}
|
|
38
|
+
function isWriteConflictError(err) {
|
|
39
|
+
return isCloudRoaringError(err) && err.name === "WriteConflictError";
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// src/core/ports.ts
|
|
43
|
+
var NO_ROW = /* @__PURE__ */ Symbol.for("cloud-roaring.no-row");
|
|
44
|
+
|
|
45
|
+
// src/core/validate.ts
|
|
46
|
+
var CHUNK_KEY_MAX = 65535;
|
|
47
|
+
var NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,255}$/;
|
|
48
|
+
function validatePart(value, field) {
|
|
49
|
+
if (typeof value !== "string" || !NAME.test(value) || value.includes("..")) {
|
|
50
|
+
throw new ValidationError(
|
|
51
|
+
`${field} must match ${String(NAME)} and contain no "..": got ${JSON.stringify(value)}`
|
|
52
|
+
);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
function validateSegmentRef(ref) {
|
|
56
|
+
validatePart(ref.segment, "segment");
|
|
57
|
+
if (ref.namespace !== void 0) validatePart(ref.namespace, "namespace");
|
|
58
|
+
}
|
|
59
|
+
function validateChunkRef(ref) {
|
|
60
|
+
validateSegmentRef(ref);
|
|
61
|
+
if (!Number.isInteger(ref.chunkKey) || ref.chunkKey < 0 || ref.chunkKey > CHUNK_KEY_MAX) {
|
|
62
|
+
throw new ValidationError(
|
|
63
|
+
`chunkKey must be an integer in [0, ${CHUNK_KEY_MAX}]; got ${ref.chunkKey}`
|
|
64
|
+
);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// src/drivers/_shared/keys.ts
|
|
69
|
+
var DEFAULT_NAMESPACE = "_default";
|
|
70
|
+
function namespacePart(namespace) {
|
|
71
|
+
return namespace ?? DEFAULT_NAMESPACE;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// src/drivers/postgres/keys.ts
|
|
75
|
+
var IDENT = /^[A-Za-z_][A-Za-z0-9_]{0,62}$/;
|
|
76
|
+
function validateAndQuoteTable(table) {
|
|
77
|
+
const parts = table.split(".");
|
|
78
|
+
if (parts.length > 2 || parts.some((p) => !IDENT.test(p))) {
|
|
79
|
+
throw new ValidationError(
|
|
80
|
+
`invalid table name ${JSON.stringify(table)} \u2014 expected an identifier or "schema.table" (letters, digits, underscore; \u226463 chars; leading letter/underscore)`
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
return parts.map((p) => `"${p}"`).join(".");
|
|
84
|
+
}
|
|
85
|
+
function normalizeKeyPrefix(prefix) {
|
|
86
|
+
if (prefix === void 0 || prefix === "") return "";
|
|
87
|
+
for (const ch of prefix) {
|
|
88
|
+
if (ch.charCodeAt(0) < 32) {
|
|
89
|
+
throw new ValidationError("keyPrefix must not contain control characters");
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return prefix;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// src/drivers/postgres/postgres-errors.ts
|
|
96
|
+
function networkCode(err) {
|
|
97
|
+
const code = err?.code;
|
|
98
|
+
return typeof code === "string" && /^E[A-Z_]+$/.test(code) ? code : void 0;
|
|
99
|
+
}
|
|
100
|
+
var CONNECTION_LOST = /connection terminated|not queryable|timeout exceeded when trying to connect/i;
|
|
101
|
+
function isConnectionLostMessage(err) {
|
|
102
|
+
return err instanceof Error && CONNECTION_LOST.test(err.message);
|
|
103
|
+
}
|
|
104
|
+
function sqlState(err) {
|
|
105
|
+
if (networkCode(err) !== void 0) return void 0;
|
|
106
|
+
const code = err?.code;
|
|
107
|
+
return typeof code === "string" && /^[0-9A-Z]{5}$/.test(code) ? code : void 0;
|
|
108
|
+
}
|
|
109
|
+
function isTransient(err) {
|
|
110
|
+
const state = sqlState(err);
|
|
111
|
+
if (state !== void 0) {
|
|
112
|
+
const cls = state.slice(0, 2);
|
|
113
|
+
if (cls === "40" || cls === "08" || cls === "53") return true;
|
|
114
|
+
if (state === "57P03" || state === "57P01" || state === "57P02") return true;
|
|
115
|
+
return false;
|
|
116
|
+
}
|
|
117
|
+
const net = networkCode(err);
|
|
118
|
+
if (net === "ECONNRESET" || net === "ECONNABORTED" || net === "ETIMEDOUT" || net === "ESOCKETTIMEDOUT" || net === "ECONNREFUSED" || net === "EHOSTUNREACH" || net === "ENETUNREACH" || net === "EPIPE" || net === "EAI_AGAIN" || net === "ENOTFOUND") {
|
|
119
|
+
return true;
|
|
120
|
+
}
|
|
121
|
+
return isConnectionLostMessage(err);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// src/drivers/postgres/warm.ts
|
|
125
|
+
var DEFAULT_TABLE = "cloud_roaring_warm";
|
|
126
|
+
var DEFAULT_LIST_BATCH = 1e3;
|
|
127
|
+
function postgresWarmTableDDL(table = DEFAULT_TABLE) {
|
|
128
|
+
const t = validateAndQuoteTable(table);
|
|
129
|
+
return `CREATE TABLE IF NOT EXISTS ${t} (
|
|
130
|
+
key_prefix text NOT NULL,
|
|
131
|
+
namespace text NOT NULL,
|
|
132
|
+
segment text NOT NULL,
|
|
133
|
+
chunk_key integer NOT NULL,
|
|
134
|
+
token text NOT NULL,
|
|
135
|
+
payload bytea NOT NULL,
|
|
136
|
+
PRIMARY KEY (key_prefix, namespace, segment, chunk_key)
|
|
137
|
+
);`;
|
|
138
|
+
}
|
|
139
|
+
var PostgresWarmDriver = class {
|
|
140
|
+
pool;
|
|
141
|
+
table;
|
|
142
|
+
// validated + quoted for interpolation
|
|
143
|
+
keyPrefix;
|
|
144
|
+
listBatch;
|
|
145
|
+
constructor(options) {
|
|
146
|
+
this.pool = options.pool;
|
|
147
|
+
this.table = validateAndQuoteTable(options.table ?? DEFAULT_TABLE);
|
|
148
|
+
this.keyPrefix = normalizeKeyPrefix(options.keyPrefix);
|
|
149
|
+
const batch = options.listPageSize ?? DEFAULT_LIST_BATCH;
|
|
150
|
+
if (!Number.isSafeInteger(batch) || batch < 1) {
|
|
151
|
+
throw new ValidationError(`listPageSize must be a positive safe integer; got ${batch}`);
|
|
152
|
+
}
|
|
153
|
+
this.listBatch = batch;
|
|
154
|
+
}
|
|
155
|
+
// Reads are always strongly consistent (single primary), so the optional `WarmReadOptions` hint would be a
|
|
156
|
+
// no-op — the structurally-optional param is simply omitted (still satisfies IWarmDriver), as in LocalFs.
|
|
157
|
+
async get(ref) {
|
|
158
|
+
validateChunkRef(ref);
|
|
159
|
+
try {
|
|
160
|
+
const res = await this.pool.query(
|
|
161
|
+
`SELECT token, payload FROM ${this.table} WHERE key_prefix = $1 AND namespace = $2 AND segment = $3 AND chunk_key = $4`,
|
|
162
|
+
[this.keyPrefix, namespacePart(ref.namespace), ref.segment, ref.chunkKey]
|
|
163
|
+
);
|
|
164
|
+
const row = res.rows[0];
|
|
165
|
+
return row === void 0 ? null : this.rowFrom(row, ref.chunkKey);
|
|
166
|
+
} catch (err) {
|
|
167
|
+
throw this.mapError(err);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
async putConditional(ref, bytes, expected) {
|
|
171
|
+
validateChunkRef(ref);
|
|
172
|
+
const token = crypto.randomUUID();
|
|
173
|
+
const payload = Buffer.from(bytes);
|
|
174
|
+
const ns = namespacePart(ref.namespace);
|
|
175
|
+
try {
|
|
176
|
+
if (expected === NO_ROW) {
|
|
177
|
+
const res = await this.pool.query(
|
|
178
|
+
`INSERT INTO ${this.table} (key_prefix, namespace, segment, chunk_key, token, payload) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (key_prefix, namespace, segment, chunk_key) DO NOTHING`,
|
|
179
|
+
[this.keyPrefix, ns, ref.segment, ref.chunkKey, token, payload]
|
|
180
|
+
);
|
|
181
|
+
if (res.rowCount !== 1) {
|
|
182
|
+
throw new WriteConflictError(`chunk ${ref.chunkKey} already exists (create-if-absent)`);
|
|
183
|
+
}
|
|
184
|
+
} else {
|
|
185
|
+
const res = await this.pool.query(
|
|
186
|
+
`UPDATE ${this.table} SET token = $5, payload = $6 WHERE key_prefix = $1 AND namespace = $2 AND segment = $3 AND chunk_key = $4 AND token = $7`,
|
|
187
|
+
[this.keyPrefix, ns, ref.segment, ref.chunkKey, token, payload, expected]
|
|
188
|
+
);
|
|
189
|
+
if (res.rowCount !== 1)
|
|
190
|
+
throw new WriteConflictError(`OCC conflict on chunk ${ref.chunkKey}`);
|
|
191
|
+
}
|
|
192
|
+
return { token };
|
|
193
|
+
} catch (err) {
|
|
194
|
+
if (isWriteConflictError(err)) throw err;
|
|
195
|
+
throw this.mapError(err);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
async deleteConditional(ref, expected) {
|
|
199
|
+
validateChunkRef(ref);
|
|
200
|
+
try {
|
|
201
|
+
const res = await this.pool.query(
|
|
202
|
+
`DELETE FROM ${this.table} WHERE key_prefix = $1 AND namespace = $2 AND segment = $3 AND chunk_key = $4 AND token = $5`,
|
|
203
|
+
[this.keyPrefix, namespacePart(ref.namespace), ref.segment, ref.chunkKey, expected]
|
|
204
|
+
);
|
|
205
|
+
if (res.rowCount !== 1) {
|
|
206
|
+
throw new WriteConflictError(`fenced-delete conflict on chunk ${ref.chunkKey}`);
|
|
207
|
+
}
|
|
208
|
+
} catch (err) {
|
|
209
|
+
if (isWriteConflictError(err)) throw err;
|
|
210
|
+
throw this.mapError(err);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
async *listChunks(ref) {
|
|
214
|
+
validateSegmentRef(ref);
|
|
215
|
+
const ns = namespacePart(ref.namespace);
|
|
216
|
+
let after = -1;
|
|
217
|
+
for (; ; ) {
|
|
218
|
+
let rows;
|
|
219
|
+
try {
|
|
220
|
+
const res = await this.pool.query(
|
|
221
|
+
`SELECT chunk_key, token, payload FROM ${this.table} WHERE key_prefix = $1 AND namespace = $2 AND segment = $3 AND chunk_key > $4 ORDER BY chunk_key ASC LIMIT $5`,
|
|
222
|
+
[this.keyPrefix, ns, ref.segment, after, this.listBatch]
|
|
223
|
+
);
|
|
224
|
+
rows = res.rows;
|
|
225
|
+
} catch (err) {
|
|
226
|
+
throw this.mapError(err);
|
|
227
|
+
}
|
|
228
|
+
for (const row of rows) {
|
|
229
|
+
const chunkKey = row.chunk_key;
|
|
230
|
+
if (typeof chunkKey !== "number") {
|
|
231
|
+
throw new IntegrityError("warm row is missing its chunk_key");
|
|
232
|
+
}
|
|
233
|
+
yield { chunkKey, ...this.rowFrom(row, chunkKey) };
|
|
234
|
+
after = chunkKey;
|
|
235
|
+
}
|
|
236
|
+
if (rows.length < this.listBatch) return;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
/**
|
|
240
|
+
* Build a {@link WarmRow} from a raw row. `token` must be a non-empty string and `payload` a Buffer — a
|
|
241
|
+
* missing/typo'd column means a corrupt or foreign row, which we reject (untrusted-data posture) rather
|
|
242
|
+
* than paper over. The returned bytes are a fresh copy (driver-owned, read-only per the contract).
|
|
243
|
+
*/
|
|
244
|
+
rowFrom(row, chunkKey) {
|
|
245
|
+
if (typeof row.token !== "string" || row.token === "") {
|
|
246
|
+
throw new IntegrityError(`warm row for chunk ${chunkKey} is missing its token`);
|
|
247
|
+
}
|
|
248
|
+
if (!(row.payload instanceof Uint8Array)) {
|
|
249
|
+
throw new IntegrityError(`warm row for chunk ${chunkKey} is missing its payload`);
|
|
250
|
+
}
|
|
251
|
+
return { token: row.token, bytes: new Uint8Array(row.payload) };
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* Reclassify a transient Postgres fault (serialization/deadlock, connection, resource, dropped socket) as
|
|
255
|
+
* a retryable {@link TransientError}; everything else propagates unchanged. Applied at every query site so
|
|
256
|
+
* callers + the retry decorator only ever see typed errors. A deterministic OCC conflict is signalled by
|
|
257
|
+
* the driver itself (0 rows affected) — never reclassified here.
|
|
258
|
+
*/
|
|
259
|
+
mapError(err) {
|
|
260
|
+
if (isTransient(err)) {
|
|
261
|
+
return new TransientError(
|
|
262
|
+
`transient Postgres fault: ${err?.code ?? "unknown"}`,
|
|
263
|
+
{ cause: err }
|
|
264
|
+
);
|
|
265
|
+
}
|
|
266
|
+
return err;
|
|
267
|
+
}
|
|
268
|
+
};
|
|
269
|
+
|
|
270
|
+
exports.PostgresWarmDriver = PostgresWarmDriver;
|
|
271
|
+
exports.postgresWarmTableDDL = postgresWarmTableDDL;
|
|
272
|
+
//# sourceMappingURL=index.cjs.map
|
|
273
|
+
//# sourceMappingURL=index.cjs.map
|