@cloudbitmaps/core 0.1.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +12 -0
- package/README.md +22 -0
- package/dist/azure/index.cjs +368 -0
- package/dist/azure/index.cjs.map +1 -0
- package/dist/azure/index.d.cts +48 -0
- package/dist/azure/index.d.ts +48 -0
- package/dist/azure/index.js +304 -0
- package/dist/azure/index.js.map +1 -0
- package/dist/cassandra/index.cjs +265 -0
- package/dist/cassandra/index.cjs.map +1 -0
- package/dist/cassandra/index.d.cts +52 -0
- package/dist/cassandra/index.d.ts +52 -0
- package/dist/cassandra/index.js +204 -0
- package/dist/cassandra/index.js.map +1 -0
- package/dist/chunk-2YDULGXS.js +203 -0
- package/dist/chunk-2YDULGXS.js.map +1 -0
- package/dist/chunk-7LMLYSVJ.js +43 -0
- package/dist/chunk-7LMLYSVJ.js.map +1 -0
- package/dist/chunk-AS6ODRLT.js +6 -0
- package/dist/chunk-AS6ODRLT.js.map +1 -0
- package/dist/chunk-NUIDEEFZ.js +91 -0
- package/dist/chunk-NUIDEEFZ.js.map +1 -0
- package/dist/chunk-SNJVZ227.js +35 -0
- package/dist/chunk-SNJVZ227.js.map +1 -0
- package/dist/dynamodb/index.cjs +731 -0
- package/dist/dynamodb/index.cjs.map +1 -0
- package/dist/dynamodb/index.d.cts +106 -0
- package/dist/dynamodb/index.d.ts +106 -0
- package/dist/dynamodb/index.js +460 -0
- package/dist/dynamodb/index.js.map +1 -0
- package/dist/gcs/index.cjs +343 -0
- package/dist/gcs/index.cjs.map +1 -0
- package/dist/gcs/index.d.cts +46 -0
- package/dist/gcs/index.d.ts +46 -0
- package/dist/gcs/index.js +279 -0
- package/dist/gcs/index.js.map +1 -0
- package/dist/index.cjs +4031 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +1945 -0
- package/dist/index.d.ts +1945 -0
- package/dist/index.js +3642 -0
- package/dist/index.js.map +1 -0
- package/dist/mongodb/index.cjs +260 -0
- package/dist/mongodb/index.cjs.map +1 -0
- package/dist/mongodb/index.d.cts +45 -0
- package/dist/mongodb/index.d.ts +45 -0
- package/dist/mongodb/index.js +199 -0
- package/dist/mongodb/index.js.map +1 -0
- package/dist/mysql/index.cjs +281 -0
- package/dist/mysql/index.cjs.map +1 -0
- package/dist/mysql/index.d.cts +56 -0
- package/dist/mysql/index.d.ts +56 -0
- package/dist/mysql/index.js +211 -0
- package/dist/mysql/index.js.map +1 -0
- package/dist/ports-D3BrJ6ax.d.cts +357 -0
- package/dist/ports-D3BrJ6ax.d.ts +357 -0
- package/dist/postgres/index.cjs +273 -0
- package/dist/postgres/index.cjs.map +1 -0
- package/dist/postgres/index.d.cts +54 -0
- package/dist/postgres/index.d.ts +54 -0
- package/dist/postgres/index.js +203 -0
- package/dist/postgres/index.js.map +1 -0
- package/dist/redis/index.cjs +257 -0
- package/dist/redis/index.cjs.map +1 -0
- package/dist/redis/index.d.cts +42 -0
- package/dist/redis/index.d.ts +42 -0
- package/dist/redis/index.js +197 -0
- package/dist/redis/index.js.map +1 -0
- package/dist/s3/index.cjs +891 -0
- package/dist/s3/index.cjs.map +1 -0
- package/dist/s3/index.d.cts +112 -0
- package/dist/s3/index.d.ts +112 -0
- package/dist/s3/index.js +555 -0
- package/dist/s3/index.js.map +1 -0
- package/package.json +172 -0
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,4031 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var crypto = require('crypto');
|
|
4
|
+
var fs = require('fs');
|
|
5
|
+
var promises = require('fs/promises');
|
|
6
|
+
var path = require('path');
|
|
7
|
+
|
|
8
|
+
// src/core/errors.ts
|
|
9
|
+
var ERROR_BRAND = /* @__PURE__ */ Symbol.for("cloud-roaring.error");
|
|
10
|
+
var TRANSIENT_BRAND = /* @__PURE__ */ Symbol.for("cloud-roaring.error.transient");
|
|
11
|
+
var CloudRoaringError = class extends Error {
|
|
12
|
+
/** Cross-bundle brand — see the predicates ({@link isCloudRoaringError}, …). Non-enumerable-ish (symbol key ⇒ not in JSON). */
|
|
13
|
+
[ERROR_BRAND] = true;
|
|
14
|
+
constructor(message) {
|
|
15
|
+
super(message);
|
|
16
|
+
this.name = new.target.name;
|
|
17
|
+
}
|
|
18
|
+
};
|
|
19
|
+
var ValidationError = class extends CloudRoaringError {
|
|
20
|
+
};
|
|
21
|
+
var WriteConflictError = class extends CloudRoaringError {
|
|
22
|
+
};
|
|
23
|
+
var IntegrityError = class extends CloudRoaringError {
|
|
24
|
+
};
|
|
25
|
+
var NotFoundError = class extends CloudRoaringError {
|
|
26
|
+
};
|
|
27
|
+
var UnsupportedError = class extends CloudRoaringError {
|
|
28
|
+
};
|
|
29
|
+
var CapabilityError = class extends CloudRoaringError {
|
|
30
|
+
};
|
|
31
|
+
var BudgetExceededError = class extends CloudRoaringError {
|
|
32
|
+
};
|
|
33
|
+
var KeyUnavailableError = class extends CloudRoaringError {
|
|
34
|
+
};
|
|
35
|
+
var TransientError = class extends CloudRoaringError {
|
|
36
|
+
/** A second brand so the whole transient subtree (incl. {@link TimeoutError}) is classifiable cross-bundle. */
|
|
37
|
+
[TRANSIENT_BRAND] = true;
|
|
38
|
+
constructor(message, options) {
|
|
39
|
+
super(message);
|
|
40
|
+
if (options && "cause" in options) this.cause = options.cause;
|
|
41
|
+
}
|
|
42
|
+
};
|
|
43
|
+
var TimeoutError = class extends TransientError {
|
|
44
|
+
};
|
|
45
|
+
function hasBrand(err, brand) {
|
|
46
|
+
return typeof err === "object" && err !== null && err[brand] === true;
|
|
47
|
+
}
|
|
48
|
+
function isCloudRoaringError(err) {
|
|
49
|
+
return hasBrand(err, ERROR_BRAND);
|
|
50
|
+
}
|
|
51
|
+
function isWriteConflictError(err) {
|
|
52
|
+
return isCloudRoaringError(err) && err.name === "WriteConflictError";
|
|
53
|
+
}
|
|
54
|
+
function isTransientError(err) {
|
|
55
|
+
return hasBrand(err, TRANSIENT_BRAND);
|
|
56
|
+
}
|
|
57
|
+
function isNotFoundError(err) {
|
|
58
|
+
return isCloudRoaringError(err) && err.name === "NotFoundError";
|
|
59
|
+
}
|
|
60
|
+
function isIntegrityError(err) {
|
|
61
|
+
return isCloudRoaringError(err) && err.name === "IntegrityError";
|
|
62
|
+
}
|
|
63
|
+
function isValidationError(err) {
|
|
64
|
+
return isCloudRoaringError(err) && err.name === "ValidationError";
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// src/core/bit-route.ts
|
|
68
|
+
var U32_MAX = 4294967295;
|
|
69
|
+
var CHUNK_COUNT = 65536;
|
|
70
|
+
var LOW_16 = 65535;
|
|
71
|
+
function splitId(id) {
|
|
72
|
+
if (!Number.isInteger(id) || id < 0 || id > U32_MAX) {
|
|
73
|
+
throw new ValidationError(`id must be an integer in 0..${U32_MAX}; got ${id}`);
|
|
74
|
+
}
|
|
75
|
+
return { chunkKey: id >>> 16, remainder: id & LOW_16 };
|
|
76
|
+
}
|
|
77
|
+
function joinId(chunkKey, remainder) {
|
|
78
|
+
return ((chunkKey & LOW_16) << 16 >>> 0) + (remainder & LOW_16);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// src/core/chunk.ts
|
|
82
|
+
function emptyDelta(codec) {
|
|
83
|
+
return { adds: codec.empty(), removes: codec.empty() };
|
|
84
|
+
}
|
|
85
|
+
function applyAdd(delta, remainder) {
|
|
86
|
+
delta.adds.add(remainder);
|
|
87
|
+
delta.removes.remove(remainder);
|
|
88
|
+
}
|
|
89
|
+
function applyRemove(delta, remainder) {
|
|
90
|
+
delta.removes.add(remainder);
|
|
91
|
+
delta.adds.remove(remainder);
|
|
92
|
+
}
|
|
93
|
+
function effective(cold, delta) {
|
|
94
|
+
const out = cold.clone();
|
|
95
|
+
out.orInPlace(delta.adds);
|
|
96
|
+
out.andNotInPlace(delta.removes);
|
|
97
|
+
return out;
|
|
98
|
+
}
|
|
99
|
+
var WARM_DELTA_VERSION = 1;
|
|
100
|
+
var VERSION_BYTES = 1;
|
|
101
|
+
var HEADER_BYTES = 4;
|
|
102
|
+
function encodeDelta(delta) {
|
|
103
|
+
const adds = delta.adds.serialize();
|
|
104
|
+
const removes = delta.removes.serialize();
|
|
105
|
+
const out = new Uint8Array(VERSION_BYTES + HEADER_BYTES + adds.length + removes.length);
|
|
106
|
+
out[0] = WARM_DELTA_VERSION;
|
|
107
|
+
new DataView(out.buffer).setUint32(VERSION_BYTES, adds.length, true);
|
|
108
|
+
out.set(adds, VERSION_BYTES + HEADER_BYTES);
|
|
109
|
+
out.set(removes, VERSION_BYTES + HEADER_BYTES + adds.length);
|
|
110
|
+
return out;
|
|
111
|
+
}
|
|
112
|
+
function decodeDelta(bytes, maxBitmapBytes, codec) {
|
|
113
|
+
if (bytes.length < VERSION_BYTES + HEADER_BYTES) {
|
|
114
|
+
throw new IntegrityError(`delta too short: ${bytes.length}B`);
|
|
115
|
+
}
|
|
116
|
+
const version = bytes[0];
|
|
117
|
+
if (version !== WARM_DELTA_VERSION) {
|
|
118
|
+
throw new UnsupportedError(
|
|
119
|
+
`Warm delta schema version ${version} is unsupported (this build reads v${WARM_DELTA_VERSION})`
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
const totalCap = 2 * maxBitmapBytes + VERSION_BYTES + HEADER_BYTES;
|
|
123
|
+
if (bytes.length > totalCap) {
|
|
124
|
+
throw new IntegrityError(`delta is ${bytes.length}B, exceeds cap ${totalCap}B`);
|
|
125
|
+
}
|
|
126
|
+
const addsStart = VERSION_BYTES + HEADER_BYTES;
|
|
127
|
+
const addsLen = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength).getUint32(
|
|
128
|
+
VERSION_BYTES,
|
|
129
|
+
true
|
|
130
|
+
);
|
|
131
|
+
const addsEnd = addsStart + addsLen;
|
|
132
|
+
if (addsLen > bytes.length - addsStart) {
|
|
133
|
+
throw new IntegrityError(
|
|
134
|
+
`delta addsLen ${addsLen} exceeds payload ${bytes.length - addsStart}`
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
const adds = codec.safeDeserialize(bytes.subarray(addsStart, addsEnd), maxBitmapBytes);
|
|
138
|
+
const removes = codec.safeDeserialize(bytes.subarray(addsEnd), maxBitmapBytes);
|
|
139
|
+
return { adds, removes };
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// src/core/budget.ts
|
|
143
|
+
var DEFAULT_BUDGET = { maxRequests: 1e6 };
|
|
144
|
+
function resolveBudget(opt, fallback) {
|
|
145
|
+
if (opt === false) return null;
|
|
146
|
+
const maxRequests = opt?.maxRequests ?? fallback.maxRequests;
|
|
147
|
+
if (!Number.isInteger(maxRequests) || maxRequests < 1) {
|
|
148
|
+
throw new ValidationError(`budget.maxRequests must be a positive integer; got ${maxRequests}`);
|
|
149
|
+
}
|
|
150
|
+
return { maxRequests };
|
|
151
|
+
}
|
|
152
|
+
function resolvePerOpBudget(opt, storeBudget) {
|
|
153
|
+
if (opt === void 0) return storeBudget;
|
|
154
|
+
return resolveBudget(opt, storeBudget ?? DEFAULT_BUDGET);
|
|
155
|
+
}
|
|
156
|
+
function checkBudget(budget, projectedRequests, op) {
|
|
157
|
+
if (budget !== null && projectedRequests > budget.maxRequests) {
|
|
158
|
+
throw new BudgetExceededError(
|
|
159
|
+
`${op} would fan out to ${projectedRequests} units, over the per-op budget of ${budget.maxRequests} \u2014 raise \`budget.maxRequests\`, override it per-op, or set \`budget: false\``
|
|
160
|
+
);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// src/core/concurrency.ts
|
|
165
|
+
async function mapWithConcurrency(items, limit, fn) {
|
|
166
|
+
if (!Number.isInteger(limit) || limit < 1) {
|
|
167
|
+
throw new ValidationError(`concurrency limit must be a positive integer; got ${limit}`);
|
|
168
|
+
}
|
|
169
|
+
const results = new Array(items.length);
|
|
170
|
+
let next = 0;
|
|
171
|
+
let failed = false;
|
|
172
|
+
let firstError;
|
|
173
|
+
const worker = async () => {
|
|
174
|
+
while (!failed) {
|
|
175
|
+
const i = next++;
|
|
176
|
+
if (i >= items.length) return;
|
|
177
|
+
try {
|
|
178
|
+
results[i] = await fn(items[i], i);
|
|
179
|
+
} catch (err) {
|
|
180
|
+
if (!failed) {
|
|
181
|
+
failed = true;
|
|
182
|
+
firstError = err;
|
|
183
|
+
}
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
};
|
|
188
|
+
const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
|
|
189
|
+
await Promise.all(workers);
|
|
190
|
+
if (failed) throw firstError;
|
|
191
|
+
return results;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// src/core/keys.ts
|
|
195
|
+
var FIELD = " ";
|
|
196
|
+
var NO_NAMESPACE = "/";
|
|
197
|
+
function segmentKey(ref) {
|
|
198
|
+
return `${ref.namespace ?? NO_NAMESPACE}${FIELD}${ref.segment}`;
|
|
199
|
+
}
|
|
200
|
+
function segmentPrefix(ref) {
|
|
201
|
+
return `${segmentKey(ref)}${FIELD}`;
|
|
202
|
+
}
|
|
203
|
+
function chunkRefKey(ref) {
|
|
204
|
+
return `${segmentPrefix(ref)}${ref.chunkKey}`;
|
|
205
|
+
}
|
|
206
|
+
function chunkGenKey(ref, generation) {
|
|
207
|
+
return `${chunkRefKey(ref)}${FIELD}${generation}`;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
// src/core/metrics.ts
|
|
211
|
+
var NOOP_METRICS = {
|
|
212
|
+
onEvent() {
|
|
213
|
+
}
|
|
214
|
+
};
|
|
215
|
+
function safeMetrics(sink) {
|
|
216
|
+
if (sink === NOOP_METRICS) return sink;
|
|
217
|
+
return {
|
|
218
|
+
onEvent(event) {
|
|
219
|
+
try {
|
|
220
|
+
sink.onEvent(event);
|
|
221
|
+
} catch {
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
var OP_NAMES = [
|
|
227
|
+
"add",
|
|
228
|
+
"remove",
|
|
229
|
+
"addMany",
|
|
230
|
+
"removeMany",
|
|
231
|
+
"has",
|
|
232
|
+
"count",
|
|
233
|
+
"intersectInto"
|
|
234
|
+
];
|
|
235
|
+
var CountingMetricsSink = class _CountingMetricsSink {
|
|
236
|
+
cold = { gets: 0, bytes: 0, totalMs: 0 };
|
|
237
|
+
warm = { reads: 0, readBytes: 0, writes: 0, writeBytes: 0 };
|
|
238
|
+
cache = { hits: 0, misses: 0 };
|
|
239
|
+
retries = { occ: 0, transient: 0 };
|
|
240
|
+
intersect = { calls: 0, fetchedChunks: 0, skippedChunks: 0 };
|
|
241
|
+
compaction = { attempts: 0, committed: 0, dirtyChunks: 0, purged: 0, totalMs: 0 };
|
|
242
|
+
ops = _CountingMetricsSink.zeroOps();
|
|
243
|
+
static zeroOps() {
|
|
244
|
+
const ops = /* @__PURE__ */ Object.create(null);
|
|
245
|
+
for (const name of OP_NAMES) ops[name] = { count: 0, totalMs: 0 };
|
|
246
|
+
return ops;
|
|
247
|
+
}
|
|
248
|
+
onEvent(event) {
|
|
249
|
+
switch (event.kind) {
|
|
250
|
+
case "cold.get":
|
|
251
|
+
this.cold.gets += 1;
|
|
252
|
+
this.cold.bytes += event.bytes;
|
|
253
|
+
this.cold.totalMs += event.ms;
|
|
254
|
+
break;
|
|
255
|
+
case "cache":
|
|
256
|
+
if (event.hit) this.cache.hits += 1;
|
|
257
|
+
else this.cache.misses += 1;
|
|
258
|
+
break;
|
|
259
|
+
case "warm.read":
|
|
260
|
+
this.warm.reads += 1;
|
|
261
|
+
this.warm.readBytes += event.bytes;
|
|
262
|
+
break;
|
|
263
|
+
case "warm.write":
|
|
264
|
+
this.warm.writes += 1;
|
|
265
|
+
this.warm.writeBytes += event.bytes;
|
|
266
|
+
break;
|
|
267
|
+
case "retry":
|
|
268
|
+
if (event.reason === "occ") this.retries.occ += 1;
|
|
269
|
+
else this.retries.transient += 1;
|
|
270
|
+
break;
|
|
271
|
+
case "intersect":
|
|
272
|
+
this.intersect.calls += 1;
|
|
273
|
+
this.intersect.fetchedChunks += event.fetchedChunks;
|
|
274
|
+
this.intersect.skippedChunks += event.skippedChunks;
|
|
275
|
+
break;
|
|
276
|
+
case "compaction":
|
|
277
|
+
this.compaction.attempts += 1;
|
|
278
|
+
if (event.compacted) this.compaction.committed += 1;
|
|
279
|
+
this.compaction.dirtyChunks += event.dirtyChunks;
|
|
280
|
+
this.compaction.purged += event.purged;
|
|
281
|
+
this.compaction.totalMs += event.ms;
|
|
282
|
+
break;
|
|
283
|
+
case "op": {
|
|
284
|
+
const op = this.ops[event.name];
|
|
285
|
+
op.count += 1;
|
|
286
|
+
op.totalMs += event.ms;
|
|
287
|
+
break;
|
|
288
|
+
}
|
|
289
|
+
default: {
|
|
290
|
+
const exhaustive = event;
|
|
291
|
+
return exhaustive;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
snapshot() {
|
|
296
|
+
const ops = /* @__PURE__ */ Object.create(null);
|
|
297
|
+
for (const name of OP_NAMES) ops[name] = { ...this.ops[name] };
|
|
298
|
+
return {
|
|
299
|
+
cold: { ...this.cold },
|
|
300
|
+
warm: { ...this.warm },
|
|
301
|
+
cache: { ...this.cache },
|
|
302
|
+
retries: { ...this.retries },
|
|
303
|
+
intersect: { ...this.intersect },
|
|
304
|
+
compaction: { ...this.compaction },
|
|
305
|
+
ops
|
|
306
|
+
};
|
|
307
|
+
}
|
|
308
|
+
reset() {
|
|
309
|
+
this.cold = { gets: 0, bytes: 0, totalMs: 0 };
|
|
310
|
+
this.warm = { reads: 0, readBytes: 0, writes: 0, writeBytes: 0 };
|
|
311
|
+
this.cache = { hits: 0, misses: 0 };
|
|
312
|
+
this.retries = { occ: 0, transient: 0 };
|
|
313
|
+
this.intersect = { calls: 0, fetchedChunks: 0, skippedChunks: 0 };
|
|
314
|
+
this.compaction = { attempts: 0, committed: 0, dirtyChunks: 0, purged: 0, totalMs: 0 };
|
|
315
|
+
this.ops = _CountingMetricsSink.zeroOps();
|
|
316
|
+
}
|
|
317
|
+
};
|
|
318
|
+
|
|
319
|
+
// src/core/ports.ts
|
|
320
|
+
var NO_ROW = /* @__PURE__ */ Symbol.for("cloud-roaring.no-row");
|
|
321
|
+
|
|
322
|
+
// src/core/retry.ts
|
|
323
|
+
var DEFAULT_RETRY_POLICY = {
|
|
324
|
+
maxAttempts: 4,
|
|
325
|
+
baseDelayMs: 50,
|
|
326
|
+
maxDelayMs: 2e3,
|
|
327
|
+
backoffFactor: 2,
|
|
328
|
+
jitter: "full"
|
|
329
|
+
};
|
|
330
|
+
var DEFAULT_OCC_BACKOFF = {
|
|
331
|
+
maxAttempts: 1,
|
|
332
|
+
// attempts are owned by the engine's loop; this policy only supplies the delay schedule
|
|
333
|
+
baseDelayMs: 5,
|
|
334
|
+
maxDelayMs: 200,
|
|
335
|
+
backoffFactor: 2,
|
|
336
|
+
jitter: "full"
|
|
337
|
+
};
|
|
338
|
+
function isTransient(err) {
|
|
339
|
+
return isTransientError(err);
|
|
340
|
+
}
|
|
341
|
+
function backoffDelayMs(policy, attempt) {
|
|
342
|
+
const raw = policy.baseDelayMs * policy.backoffFactor ** (attempt - 1);
|
|
343
|
+
return Math.min(policy.maxDelayMs, raw);
|
|
344
|
+
}
|
|
345
|
+
function applyJitter(policy, delayMs, rng) {
|
|
346
|
+
if (policy.jitter === "none") return delayMs;
|
|
347
|
+
return rng.next() * delayMs;
|
|
348
|
+
}
|
|
349
|
+
async function withRetry(op, policy, deps) {
|
|
350
|
+
const retryable = deps.isRetryable ?? isTransient;
|
|
351
|
+
const attempts = Math.max(1, policy.maxAttempts);
|
|
352
|
+
let lastErr;
|
|
353
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
354
|
+
try {
|
|
355
|
+
return await op();
|
|
356
|
+
} catch (err) {
|
|
357
|
+
lastErr = err;
|
|
358
|
+
if (attempt >= attempts || !retryable(err)) throw err;
|
|
359
|
+
const delayMs = applyJitter(policy, backoffDelayMs(policy, attempt), deps.rng);
|
|
360
|
+
deps.onRetry?.({ attempt, delayMs, err });
|
|
361
|
+
await deps.clock.sleep(delayMs);
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
throw lastErr;
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
// src/core/engine.ts
|
|
368
|
+
var DEFAULT_MAX_BITMAP_BYTES = 1 << 20;
|
|
369
|
+
var DEFAULT_MAX_RETRIES = 16;
|
|
370
|
+
var DEFAULT_INTERSECT_CONCURRENCY = 8;
|
|
371
|
+
var InstantClock = class {
|
|
372
|
+
now() {
|
|
373
|
+
return 0;
|
|
374
|
+
}
|
|
375
|
+
sleep() {
|
|
376
|
+
return Promise.resolve();
|
|
377
|
+
}
|
|
378
|
+
};
|
|
379
|
+
var ZeroRng = class {
|
|
380
|
+
next() {
|
|
381
|
+
return 0;
|
|
382
|
+
}
|
|
383
|
+
};
|
|
384
|
+
var SegmentEngine = class {
|
|
385
|
+
warm;
|
|
386
|
+
cold;
|
|
387
|
+
cache;
|
|
388
|
+
codec;
|
|
389
|
+
maxBitmapBytes;
|
|
390
|
+
maxRetries;
|
|
391
|
+
clock;
|
|
392
|
+
rng;
|
|
393
|
+
occBackoff;
|
|
394
|
+
metrics;
|
|
395
|
+
metricsOn;
|
|
396
|
+
/** Consistency for READ-path Warm fetches (precomputed; OCC RMW passes strong explicitly, see `warmGet`). */
|
|
397
|
+
readOpts;
|
|
398
|
+
writeConcurrency;
|
|
399
|
+
/** Resolved per-op budget (null = disabled); undefined deps ⇒ the generous default. See {@link checkBudget}. */
|
|
400
|
+
budget;
|
|
401
|
+
constructor(deps) {
|
|
402
|
+
this.warm = deps.warm;
|
|
403
|
+
this.cold = deps.cold;
|
|
404
|
+
this.cache = deps.cache;
|
|
405
|
+
this.codec = deps.codec;
|
|
406
|
+
this.maxBitmapBytes = deps.maxBitmapBytes ?? DEFAULT_MAX_BITMAP_BYTES;
|
|
407
|
+
this.maxRetries = deps.maxRetries ?? DEFAULT_MAX_RETRIES;
|
|
408
|
+
this.clock = deps.clock ?? new InstantClock();
|
|
409
|
+
this.rng = deps.rng ?? new ZeroRng();
|
|
410
|
+
this.occBackoff = deps.occBackoff ?? DEFAULT_OCC_BACKOFF;
|
|
411
|
+
this.readOpts = { consistent: (deps.warmReadConsistency ?? "strong") !== "eventual" };
|
|
412
|
+
this.writeConcurrency = deps.writeConcurrency ?? 1;
|
|
413
|
+
this.budget = deps.budget === void 0 ? DEFAULT_BUDGET : deps.budget;
|
|
414
|
+
this.metrics = safeMetrics(deps.metrics ?? NOOP_METRICS);
|
|
415
|
+
this.metricsOn = this.metrics !== NOOP_METRICS;
|
|
416
|
+
}
|
|
417
|
+
add(seg, id) {
|
|
418
|
+
const { chunkKey, remainder } = splitId(id);
|
|
419
|
+
return this.readModifyWrite(seg, chunkKey, (d) => applyAdd(d, remainder));
|
|
420
|
+
}
|
|
421
|
+
remove(seg, id) {
|
|
422
|
+
const { chunkKey, remainder } = splitId(id);
|
|
423
|
+
return this.readModifyWrite(seg, chunkKey, (d) => applyRemove(d, remainder));
|
|
424
|
+
}
|
|
425
|
+
addMany(seg, ids) {
|
|
426
|
+
return this.mutateMany(seg, ids, applyAdd);
|
|
427
|
+
}
|
|
428
|
+
removeMany(seg, ids) {
|
|
429
|
+
return this.mutateMany(seg, ids, applyRemove);
|
|
430
|
+
}
|
|
431
|
+
async has(seg, id, opts) {
|
|
432
|
+
const { chunkKey, remainder } = splitId(id);
|
|
433
|
+
const ref = { ...seg, chunkKey };
|
|
434
|
+
const row = await this.warmGet(ref, opts ?? this.readOpts);
|
|
435
|
+
if (row) {
|
|
436
|
+
const delta = decodeDelta(row.bytes, this.maxBitmapBytes, this.codec);
|
|
437
|
+
if (delta.removes.has(remainder)) return false;
|
|
438
|
+
if (delta.adds.has(remainder)) return true;
|
|
439
|
+
}
|
|
440
|
+
const cold = await this.coldChunk(ref, await this.currentGen(seg));
|
|
441
|
+
return cold ? cold.has(remainder) : false;
|
|
442
|
+
}
|
|
443
|
+
async count(seg) {
|
|
444
|
+
const warmRows = await this.collectWarm(seg);
|
|
445
|
+
const cardinalities = this.cold.cardinalities ? await this.cold.cardinalities(seg) : null;
|
|
446
|
+
if (!cardinalities) {
|
|
447
|
+
const chunkKeys = await this.chunkKeys(seg, warmRows);
|
|
448
|
+
checkBudget(this.budget, chunkKeys.length, "count");
|
|
449
|
+
const gen2 = await this.currentGen(seg);
|
|
450
|
+
let total2 = 0;
|
|
451
|
+
for (const chunkKey of chunkKeys) {
|
|
452
|
+
total2 += (await this.effectiveChunk(seg, chunkKey, warmRows, gen2)).size;
|
|
453
|
+
}
|
|
454
|
+
return total2;
|
|
455
|
+
}
|
|
456
|
+
checkBudget(this.budget, warmRows.size, "count");
|
|
457
|
+
const gen = await this.currentGen(seg);
|
|
458
|
+
const keys = new Set(warmRows.keys());
|
|
459
|
+
for (const k of cardinalities.keys()) keys.add(k);
|
|
460
|
+
let total = 0;
|
|
461
|
+
for (const k of keys) {
|
|
462
|
+
this.assertChunkKeyInRange(k);
|
|
463
|
+
total += warmRows.has(k) ? (await this.effectiveChunk(seg, k, warmRows, gen)).size : cardinalities.get(k) ?? 0;
|
|
464
|
+
}
|
|
465
|
+
return total;
|
|
466
|
+
}
|
|
467
|
+
/** Whether the Cold source can measure segment size (for grounded cost); false ⇒ storage isn't grounded. */
|
|
468
|
+
get supportsColdSize() {
|
|
469
|
+
return typeof this.cold.sizeOf === "function";
|
|
470
|
+
}
|
|
471
|
+
/** Grounded Cold size of a segment's current generation (cost reporting), or null if it has no generation. */
|
|
472
|
+
segmentSize(seg) {
|
|
473
|
+
return this.cold.sizeOf ? this.cold.sizeOf(seg) : Promise.resolve(null);
|
|
474
|
+
}
|
|
475
|
+
async *iterate(seg) {
|
|
476
|
+
const warmRows = await this.collectWarm(seg);
|
|
477
|
+
const chunkKeys = await this.chunkKeys(seg, warmRows);
|
|
478
|
+
checkBudget(this.budget, chunkKeys.length, "iterate");
|
|
479
|
+
const gen = await this.currentGen(seg);
|
|
480
|
+
for (const chunkKey of chunkKeys) {
|
|
481
|
+
const eff = await this.effectiveChunk(seg, chunkKey, warmRows, gen);
|
|
482
|
+
for (const remainder of eff) yield joinId(chunkKey, remainder);
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
/**
|
|
486
|
+
* Chunk-skipping intersection. Computes
|
|
487
|
+
* `seg[0] ∩ seg[1] ∩ …` and streams the result ids ascending. It aligns each segment's *effective*
|
|
488
|
+
* chunk-key set (Warm ∪ Cold), keeps only keys present in **all** operands (a key missing from any
|
|
489
|
+
* operand can't contribute → its **Cold chunks are never fetched** — the core saving), then for each
|
|
490
|
+
* surviving key merges tiers per operand (operand fetches run in parallel) and hands the buffers to the
|
|
491
|
+
* roaring engine for the AND, streaming results through a bounded in-flight window.
|
|
492
|
+
*
|
|
493
|
+
* **Memory:** the Cold payload footprint is bounded by the window (`concurrency × operands × chunk`), not
|
|
494
|
+
* by segment size — that's the Lambda-friendly property. The **Warm** side, however, is snapshotted up
|
|
495
|
+
* front (`collectWarm` per operand), so total memory also carries each operand's Warm size. Under
|
|
496
|
+
* Topology-A (read-mostly; compaction keeps Warm small) that's negligible; a streaming Warm/Cold
|
|
497
|
+
* merge-join for large Warm is Phase 4.
|
|
498
|
+
*
|
|
499
|
+
* Generation-consistent within the call (normal case): each operand's current generation is resolved **once**
|
|
500
|
+
* up front (before the fan-out) and threaded into every chunk read, so a concurrent compaction can't corrupt
|
|
501
|
+
* or tear the result — every chunk read is a whole, checksum-verified, immutable generation (finding C10).
|
|
502
|
+
* The edge a *long* call can hit: if it straddles a mid-call `coldGenTtlMs` boundary and a compaction has
|
|
503
|
+
* committed, an operand's not-yet-read chunks may re-resolve forward to the newer generation (a generation
|
|
504
|
+
* hop within one long call) — the call never crashes or returns a torn object, but may mix generations. That
|
|
505
|
+
* window is a single TTL (default 2 s), not the multi-cycle GC-sweep window it took pre-Phase-B. A shorter call
|
|
506
|
+
* is unaffected **unless the reader cache evicts an operand mid-call** (`maxOpenSegments`, gap #1): the re-read
|
|
507
|
+
* re-resolves fresh (bypassing the TTL), which can hop generations even sub-TTL — still whole/immutable per
|
|
508
|
+
* read, never torn. Full intra-op snapshot isolation is deferred (gap #4).
|
|
509
|
+
*/
|
|
510
|
+
async *intersect(segs, options) {
|
|
511
|
+
if (segs.length === 0) {
|
|
512
|
+
throw new ValidationError("intersect requires at least one segment");
|
|
513
|
+
}
|
|
514
|
+
const limit = options?.concurrency ?? DEFAULT_INTERSECT_CONCURRENCY;
|
|
515
|
+
if (!Number.isInteger(limit) || limit < 1) {
|
|
516
|
+
throw new ValidationError(
|
|
517
|
+
`concurrency must be a positive integer; got ${options?.concurrency}`
|
|
518
|
+
);
|
|
519
|
+
}
|
|
520
|
+
const budget = resolvePerOpBudget(options?.budget, this.budget);
|
|
521
|
+
const operands = await Promise.all(
|
|
522
|
+
segs.map(async (seg) => {
|
|
523
|
+
const warmRows = await this.collectWarm(seg);
|
|
524
|
+
const keys = await this.chunkKeys(seg, warmRows);
|
|
525
|
+
const gen = await this.currentGen(seg);
|
|
526
|
+
return { seg, warmRows, keys: new Set(keys), gen };
|
|
527
|
+
})
|
|
528
|
+
);
|
|
529
|
+
const pivot = operands.reduce((a, b) => a.keys.size <= b.keys.size ? a : b);
|
|
530
|
+
const common = [];
|
|
531
|
+
for (const k of pivot.keys) {
|
|
532
|
+
if (operands.every((op) => op.keys.has(k))) common.push(k);
|
|
533
|
+
}
|
|
534
|
+
common.sort((a, b) => a - b);
|
|
535
|
+
checkBudget(budget, common.length * operands.length, "intersect");
|
|
536
|
+
if (this.metricsOn) {
|
|
537
|
+
const distinctKeys = /* @__PURE__ */ new Set();
|
|
538
|
+
for (const op of operands) {
|
|
539
|
+
for (const k of op.keys) distinctKeys.add(k);
|
|
540
|
+
}
|
|
541
|
+
this.metrics.onEvent({
|
|
542
|
+
kind: "intersect",
|
|
543
|
+
operands: segs.length,
|
|
544
|
+
fetchedChunks: common.length,
|
|
545
|
+
skippedChunks: distinctKeys.size - common.length
|
|
546
|
+
});
|
|
547
|
+
}
|
|
548
|
+
const startAt = (key) => this.intersectChunk(operands, key).then(
|
|
549
|
+
(result) => ({ key, result }),
|
|
550
|
+
(error) => ({ key, result: null, error })
|
|
551
|
+
);
|
|
552
|
+
const inFlight = [];
|
|
553
|
+
let next = 0;
|
|
554
|
+
while (next < common.length && inFlight.length < limit) inFlight.push(startAt(common[next++]));
|
|
555
|
+
while (inFlight.length > 0) {
|
|
556
|
+
const slot = await inFlight.shift();
|
|
557
|
+
if (next < common.length) inFlight.push(startAt(common[next++]));
|
|
558
|
+
if (slot.error !== void 0) throw slot.error;
|
|
559
|
+
if (slot.result && !slot.result.isEmpty) {
|
|
560
|
+
for (const remainder of slot.result) yield joinId(slot.key, remainder);
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
/**
|
|
565
|
+
* The AND of one chunk key across all operands (tier-merged per operand), or `null` if empty. Operand
|
|
566
|
+
* chunks are fetched **in parallel** (the spec's parallel byte-range reads). `acc` takes ownership of the
|
|
567
|
+
* first operand's effective chunk and mutates it in place — safe because `effectiveChunk` always returns
|
|
568
|
+
* a FRESH bitmap (never a cached/shared Cold instance), so this can't poison the HOT cache.
|
|
569
|
+
*/
|
|
570
|
+
async intersectChunk(operands, chunkKey) {
|
|
571
|
+
const chunks = await Promise.all(
|
|
572
|
+
operands.map((op) => this.effectiveChunk(op.seg, chunkKey, op.warmRows, op.gen))
|
|
573
|
+
);
|
|
574
|
+
const acc = chunks[0];
|
|
575
|
+
for (let i = 1; i < chunks.length; i++) {
|
|
576
|
+
acc.andInPlace(chunks[i]);
|
|
577
|
+
if (acc.isEmpty) return null;
|
|
578
|
+
}
|
|
579
|
+
return acc.isEmpty ? null : acc;
|
|
580
|
+
}
|
|
581
|
+
/**
|
|
582
|
+
* Materialize `seg[0] ∩ seg[1] ∩ …` into `dest` via batched `addMany` (streaming, bounded memory).
|
|
583
|
+
* **Not atomic** — like `addMany`, ids land chunk-by-chunk; a failure mid-stream leaves a partial result.
|
|
584
|
+
* `dest` is added to, not replaced.
|
|
585
|
+
*/
|
|
586
|
+
async intersectInto(dest, segs, options) {
|
|
587
|
+
const batchSize = Math.max(1, options?.batchSize ?? 4096);
|
|
588
|
+
let batch = [];
|
|
589
|
+
for await (const id of this.intersect(segs, options)) {
|
|
590
|
+
batch.push(id);
|
|
591
|
+
if (batch.length >= batchSize) {
|
|
592
|
+
await this.addMany(dest, batch);
|
|
593
|
+
batch = [];
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
if (batch.length > 0) await this.addMany(dest, batch);
|
|
597
|
+
}
|
|
598
|
+
async mutateMany(seg, ids, apply) {
|
|
599
|
+
const byChunk = /* @__PURE__ */ new Map();
|
|
600
|
+
for (const id of ids) {
|
|
601
|
+
const { chunkKey, remainder } = splitId(id);
|
|
602
|
+
const rems = byChunk.get(chunkKey) ?? [];
|
|
603
|
+
rems.push(remainder);
|
|
604
|
+
byChunk.set(chunkKey, rems);
|
|
605
|
+
}
|
|
606
|
+
await mapWithConcurrency(
|
|
607
|
+
[...byChunk],
|
|
608
|
+
this.writeConcurrency,
|
|
609
|
+
([chunkKey, rems]) => this.readModifyWrite(seg, chunkKey, (d) => {
|
|
610
|
+
for (const r of rems) apply(d, r);
|
|
611
|
+
})
|
|
612
|
+
);
|
|
613
|
+
}
|
|
614
|
+
/** OCC read-modify-write on a chunk's Warm delta, retrying on conflict (bounded). */
|
|
615
|
+
async readModifyWrite(seg, chunkKey, mutate) {
|
|
616
|
+
const ref = { ...seg, chunkKey };
|
|
617
|
+
for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
|
|
618
|
+
const row = await this.warmGet(ref);
|
|
619
|
+
const delta = row ? decodeDelta(row.bytes, this.maxBitmapBytes, this.codec) : emptyDelta(this.codec);
|
|
620
|
+
mutate(delta);
|
|
621
|
+
const encoded = encodeDelta(delta);
|
|
622
|
+
try {
|
|
623
|
+
await this.warm.putConditional(ref, encoded, row ? row.token : NO_ROW);
|
|
624
|
+
if (this.metricsOn) {
|
|
625
|
+
this.metrics.onEvent({
|
|
626
|
+
kind: "warm.write",
|
|
627
|
+
namespace: ref.namespace,
|
|
628
|
+
segment: ref.segment,
|
|
629
|
+
bytes: encoded.length
|
|
630
|
+
});
|
|
631
|
+
}
|
|
632
|
+
return;
|
|
633
|
+
} catch (err) {
|
|
634
|
+
if (isWriteConflictError(err) && attempt < this.maxRetries) {
|
|
635
|
+
const delayMs = applyJitter(
|
|
636
|
+
this.occBackoff,
|
|
637
|
+
backoffDelayMs(this.occBackoff, attempt + 1),
|
|
638
|
+
this.rng
|
|
639
|
+
);
|
|
640
|
+
if (this.metricsOn) {
|
|
641
|
+
this.metrics.onEvent({ kind: "retry", reason: "occ", attempt: attempt + 1, delayMs });
|
|
642
|
+
}
|
|
643
|
+
await this.clock.sleep(delayMs);
|
|
644
|
+
continue;
|
|
645
|
+
}
|
|
646
|
+
throw err;
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
throw new WriteConflictError(`OCC retries exhausted for chunk ${chunkKey}`);
|
|
650
|
+
}
|
|
651
|
+
/** `warm.get` + a `warm.read` metric (read bytes = the row size, 0 if absent). */
|
|
652
|
+
async warmGet(ref, opts) {
|
|
653
|
+
const row = await this.warm.get(ref, opts);
|
|
654
|
+
if (this.metricsOn) {
|
|
655
|
+
this.metrics.onEvent({
|
|
656
|
+
kind: "warm.read",
|
|
657
|
+
namespace: ref.namespace,
|
|
658
|
+
segment: ref.segment,
|
|
659
|
+
bytes: row ? row.bytes.length : 0
|
|
660
|
+
});
|
|
661
|
+
}
|
|
662
|
+
return row;
|
|
663
|
+
}
|
|
664
|
+
async collectWarm(seg) {
|
|
665
|
+
const rows = /* @__PURE__ */ new Map();
|
|
666
|
+
let bytes = 0;
|
|
667
|
+
for await (const row of this.warm.listChunks(seg, this.readOpts)) {
|
|
668
|
+
rows.set(row.chunkKey, row.bytes);
|
|
669
|
+
bytes += row.bytes.length;
|
|
670
|
+
}
|
|
671
|
+
if (this.metricsOn) {
|
|
672
|
+
this.metrics.onEvent({
|
|
673
|
+
kind: "warm.read",
|
|
674
|
+
namespace: seg.namespace,
|
|
675
|
+
segment: seg.segment,
|
|
676
|
+
bytes
|
|
677
|
+
});
|
|
678
|
+
}
|
|
679
|
+
return rows;
|
|
680
|
+
}
|
|
681
|
+
async chunkKeys(seg, warmRows) {
|
|
682
|
+
const keys = new Set(warmRows.keys());
|
|
683
|
+
for (const k of await this.cold.listChunkKeys(seg)) keys.add(k);
|
|
684
|
+
for (const k of keys) this.assertChunkKeyInRange(k);
|
|
685
|
+
return [...keys].sort((a, b) => a - b);
|
|
686
|
+
}
|
|
687
|
+
/**
|
|
688
|
+
* Tier-derived keys are untrusted (invariant 5) — fail fast on a corrupt/out-of-range key rather than
|
|
689
|
+
* letting it flow into `joinId` and produce a bogus id.
|
|
690
|
+
*/
|
|
691
|
+
assertChunkKeyInRange(k) {
|
|
692
|
+
if (!Number.isInteger(k) || k < 0 || k >= CHUNK_COUNT) {
|
|
693
|
+
throw new IntegrityError(`chunk key from a tier is out of range: ${k}`);
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
async effectiveChunk(seg, chunkKey, warmRows, gen) {
|
|
697
|
+
const warmBytes = warmRows.get(chunkKey);
|
|
698
|
+
const delta = warmBytes ? decodeDelta(warmBytes, this.maxBitmapBytes, this.codec) : emptyDelta(this.codec);
|
|
699
|
+
const cold = await this.coldChunk({ ...seg, chunkKey }, gen) ?? this.codec.empty();
|
|
700
|
+
return effective(cold, delta);
|
|
701
|
+
}
|
|
702
|
+
/** The segment's current generation, resolved once per op (`undefined` ⇒ source can't report it). */
|
|
703
|
+
currentGen(seg) {
|
|
704
|
+
return this.cold.currentGeneration ? this.cold.currentGeneration(seg) : Promise.resolve(void 0);
|
|
705
|
+
}
|
|
706
|
+
/**
|
|
707
|
+
* Decode a Cold chunk. `gen` is the segment's current generation, resolved **once per op** by the caller (not
|
|
708
|
+
* per chunk — that would put a registry re-resolve on every chunk of a count/intersect). The HOT cache is
|
|
709
|
+
* keyed by it (gap #4), so a compaction that advances the generation misses the cache and re-reads the new
|
|
710
|
+
* bytes instead of serving a stale decoded chunk (an erased id can't resurrect from a cached pre-compaction
|
|
711
|
+
* chunk). `gen === null` ⇒ the source reports no current generation ⇒ no cold bytes for any chunk, so skip the
|
|
712
|
+
* fetch entirely (also avoids a redundant re-resolve on the warm-only path). `gen === undefined` ⇒ the source
|
|
713
|
+
* can't report a generation (pins one for its lifetime) ⇒ the key stays generation-free (unchanged behaviour).
|
|
714
|
+
* Superseded-generation entries age out under the LRU ceiling — no active purge.
|
|
715
|
+
*/
|
|
716
|
+
async coldChunk(ref, gen) {
|
|
717
|
+
if (gen === null) return null;
|
|
718
|
+
const cacheKey = gen === void 0 ? chunkRefKey(ref) : chunkGenKey(ref, gen);
|
|
719
|
+
if (this.cache) {
|
|
720
|
+
const cached = this.cache.get(cacheKey);
|
|
721
|
+
if (cached) {
|
|
722
|
+
if (this.metricsOn) this.metrics.onEvent({ kind: "cache", hit: true });
|
|
723
|
+
return cached;
|
|
724
|
+
}
|
|
725
|
+
if (this.metricsOn) this.metrics.onEvent({ kind: "cache", hit: false });
|
|
726
|
+
}
|
|
727
|
+
const startedAt = this.metricsOn ? this.clock.now() : 0;
|
|
728
|
+
const bytes = await this.cold.getChunk(ref);
|
|
729
|
+
if (this.metricsOn) {
|
|
730
|
+
this.metrics.onEvent({
|
|
731
|
+
kind: "cold.get",
|
|
732
|
+
namespace: ref.namespace,
|
|
733
|
+
segment: ref.segment,
|
|
734
|
+
bytes: bytes ? bytes.length : 0,
|
|
735
|
+
ms: Math.max(0, this.clock.now() - startedAt)
|
|
736
|
+
});
|
|
737
|
+
}
|
|
738
|
+
if (!bytes) return null;
|
|
739
|
+
const bitmap = this.codec.safeDeserialize(bytes, this.maxBitmapBytes);
|
|
740
|
+
this.cache?.set(cacheKey, bitmap);
|
|
741
|
+
return bitmap;
|
|
742
|
+
}
|
|
743
|
+
};
|
|
744
|
+
|
|
745
|
+
// src/core/lru.ts
|
|
746
|
+
var BoundedLru = class {
|
|
747
|
+
map = /* @__PURE__ */ new Map();
|
|
748
|
+
maxEntries;
|
|
749
|
+
maxBytes;
|
|
750
|
+
ttlMs;
|
|
751
|
+
clock;
|
|
752
|
+
totalBytes = 0;
|
|
753
|
+
constructor(options) {
|
|
754
|
+
if (!Number.isInteger(options.maxEntries) || options.maxEntries < 1) {
|
|
755
|
+
throw new ValidationError(`maxEntries must be a positive integer; got ${options.maxEntries}`);
|
|
756
|
+
}
|
|
757
|
+
if (options.maxBytes !== void 0 && (!(options.maxBytes > 0) || !Number.isFinite(options.maxBytes))) {
|
|
758
|
+
throw new ValidationError(
|
|
759
|
+
`maxBytes must be a positive finite number; got ${options.maxBytes}`
|
|
760
|
+
);
|
|
761
|
+
}
|
|
762
|
+
if (options.ttlMs !== void 0 && (!(options.ttlMs > 0) || !Number.isFinite(options.ttlMs))) {
|
|
763
|
+
throw new ValidationError(`ttlMs must be a positive finite number; got ${options.ttlMs}`);
|
|
764
|
+
}
|
|
765
|
+
this.maxEntries = options.maxEntries;
|
|
766
|
+
this.maxBytes = options.maxBytes;
|
|
767
|
+
this.ttlMs = options.ttlMs;
|
|
768
|
+
this.clock = options.clock;
|
|
769
|
+
}
|
|
770
|
+
get(key) {
|
|
771
|
+
const entry = this.map.get(key);
|
|
772
|
+
if (entry === void 0) return void 0;
|
|
773
|
+
if (entry.expiresAt <= this.clock.now()) {
|
|
774
|
+
this.drop(key, entry);
|
|
775
|
+
return void 0;
|
|
776
|
+
}
|
|
777
|
+
this.map.delete(key);
|
|
778
|
+
this.map.set(key, entry);
|
|
779
|
+
return entry.value;
|
|
780
|
+
}
|
|
781
|
+
/** Read a value **without** counting as an access (recency unchanged, no purge on expiry). */
|
|
782
|
+
peek(key) {
|
|
783
|
+
return this.map.get(key)?.value;
|
|
784
|
+
}
|
|
785
|
+
set(key, value) {
|
|
786
|
+
const prev = this.map.get(key);
|
|
787
|
+
if (prev !== void 0) this.totalBytes -= prev.bytes;
|
|
788
|
+
this.map.delete(key);
|
|
789
|
+
const expiresAt = this.ttlMs === void 0 ? Infinity : this.clock.now() + this.ttlMs;
|
|
790
|
+
this.map.set(key, { value, expiresAt, bytes: 0 });
|
|
791
|
+
while (this.map.size > this.maxEntries) {
|
|
792
|
+
const oldest = this.map.keys().next();
|
|
793
|
+
if (oldest.done === true) break;
|
|
794
|
+
this.dropKey(oldest.value);
|
|
795
|
+
}
|
|
796
|
+
this.evictToBudget(key);
|
|
797
|
+
}
|
|
798
|
+
/**
|
|
799
|
+
* Report an entry's byte weight (typically once a cached promise resolves and its size is known). Updates
|
|
800
|
+
* the running total and evicts the least-recently-used entries if the aggregate now exceeds `maxBytes`.
|
|
801
|
+
* Recency is **not** changed. The `key` being weighted is protected from this eviction pass (we just
|
|
802
|
+
* populated it) — so a single entry larger than the whole budget is kept rather than immediately dropped.
|
|
803
|
+
*/
|
|
804
|
+
setWeight(key, bytes) {
|
|
805
|
+
if (!(bytes >= 0) || !Number.isFinite(bytes)) {
|
|
806
|
+
throw new ValidationError(`weight must be a non-negative finite number; got ${bytes}`);
|
|
807
|
+
}
|
|
808
|
+
const entry = this.map.get(key);
|
|
809
|
+
if (entry === void 0) return;
|
|
810
|
+
this.totalBytes += bytes - entry.bytes;
|
|
811
|
+
entry.bytes = bytes;
|
|
812
|
+
this.evictToBudget(key);
|
|
813
|
+
}
|
|
814
|
+
evictToBudget(protectKey) {
|
|
815
|
+
if (this.maxBytes === void 0) return;
|
|
816
|
+
while (this.totalBytes > this.maxBytes && this.map.size > 1) {
|
|
817
|
+
const oldest = this.oldestExcept(protectKey);
|
|
818
|
+
if (oldest === void 0) break;
|
|
819
|
+
this.dropKey(oldest);
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
/** The least-recently-used key that is not `protectKey`, or undefined if none other exists. */
|
|
823
|
+
oldestExcept(protectKey) {
|
|
824
|
+
for (const key of this.map.keys()) {
|
|
825
|
+
if (key !== protectKey) return key;
|
|
826
|
+
}
|
|
827
|
+
return void 0;
|
|
828
|
+
}
|
|
829
|
+
dropKey(key) {
|
|
830
|
+
const entry = this.map.get(key);
|
|
831
|
+
if (entry !== void 0) this.drop(key, entry);
|
|
832
|
+
}
|
|
833
|
+
drop(key, entry) {
|
|
834
|
+
this.totalBytes -= entry.bytes;
|
|
835
|
+
this.map.delete(key);
|
|
836
|
+
}
|
|
837
|
+
/** Membership test that does NOT count as an access (recency is unchanged). Purges if expired. */
|
|
838
|
+
has(key) {
|
|
839
|
+
const entry = this.map.get(key);
|
|
840
|
+
if (entry === void 0) return false;
|
|
841
|
+
if (entry.expiresAt <= this.clock.now()) {
|
|
842
|
+
this.drop(key, entry);
|
|
843
|
+
return false;
|
|
844
|
+
}
|
|
845
|
+
return true;
|
|
846
|
+
}
|
|
847
|
+
delete(key) {
|
|
848
|
+
const entry = this.map.get(key);
|
|
849
|
+
if (entry === void 0) return false;
|
|
850
|
+
this.drop(key, entry);
|
|
851
|
+
return true;
|
|
852
|
+
}
|
|
853
|
+
clear() {
|
|
854
|
+
this.map.clear();
|
|
855
|
+
this.totalBytes = 0;
|
|
856
|
+
}
|
|
857
|
+
/** Current entry count (may include not-yet-evicted expired entries). */
|
|
858
|
+
get size() {
|
|
859
|
+
return this.map.size;
|
|
860
|
+
}
|
|
861
|
+
/** Current aggregate reported weight (bytes) across all entries. */
|
|
862
|
+
get weightBytes() {
|
|
863
|
+
return this.totalBytes;
|
|
864
|
+
}
|
|
865
|
+
};
|
|
866
|
+
|
|
867
|
+
// src/core/cost.ts
|
|
868
|
+
var AWS_US_EAST_1_ONDEMAND = {
|
|
869
|
+
name: "aws-us-east-1-ondemand",
|
|
870
|
+
cold: { getPerMillion: 0.4, putPerMillion: 5, storagePerGiBMonth: 0.023 },
|
|
871
|
+
warm: {
|
|
872
|
+
rruPerMillion: 0.125,
|
|
873
|
+
wruPerMillion: 0.625,
|
|
874
|
+
readUnitKiB: 4,
|
|
875
|
+
writeUnitKiB: 1,
|
|
876
|
+
storagePerGiBMonth: 0.25,
|
|
877
|
+
stronglyConsistent: true
|
|
878
|
+
},
|
|
879
|
+
redis: { monthlyUSD: 346 }
|
|
880
|
+
// ElastiCache HA: 1 primary + 2 replicas (cache.m7g.large); ~$115 single-node
|
|
881
|
+
};
|
|
882
|
+
var DEFAULT_PRICING = AWS_US_EAST_1_ONDEMAND;
|
|
883
|
+
var SECONDS_PER_MONTH = 730 * 3600;
|
|
884
|
+
var GIB = 1024 ** 3;
|
|
885
|
+
function requireFiniteNonNeg(n, field) {
|
|
886
|
+
if (!Number.isFinite(n) || n < 0) {
|
|
887
|
+
throw new ValidationError(`${field} must be a finite number >= 0; got ${n}`);
|
|
888
|
+
}
|
|
889
|
+
return n;
|
|
890
|
+
}
|
|
891
|
+
function requireFinitePos(n, field) {
|
|
892
|
+
if (!Number.isFinite(n) || n <= 0) {
|
|
893
|
+
throw new ValidationError(`${field} must be a finite number > 0; got ${n}`);
|
|
894
|
+
}
|
|
895
|
+
return n;
|
|
896
|
+
}
|
|
897
|
+
function clamp01(n) {
|
|
898
|
+
return n < 0 ? 0 : n > 1 ? 1 : n;
|
|
899
|
+
}
|
|
900
|
+
function sizingBytes(spec) {
|
|
901
|
+
if (spec.sizeBytes !== void 0) return requireFiniteNonNeg(spec.sizeBytes, "segment.sizeBytes");
|
|
902
|
+
if (spec.cardinality !== void 0) {
|
|
903
|
+
return requireFiniteNonNeg(spec.cardinality, "segment.cardinality") * 2;
|
|
904
|
+
}
|
|
905
|
+
return 0;
|
|
906
|
+
}
|
|
907
|
+
var BATCHABLE_WRITE_RATIO = 8;
|
|
908
|
+
var BATCHABLE_MIN_SAVINGS_USD = 1;
|
|
909
|
+
function deriveAdvisories(input) {
|
|
910
|
+
const { writes, writesUSD, maxWarmRows, warmWriteUSD } = input;
|
|
911
|
+
const out = [];
|
|
912
|
+
if (writes > 0 && writesUSD > 0 && maxWarmRows > 0) {
|
|
913
|
+
const ratio = writes / maxWarmRows;
|
|
914
|
+
const batchedFloorUSD = maxWarmRows * warmWriteUSD;
|
|
915
|
+
const savings = writesUSD - batchedFloorUSD;
|
|
916
|
+
if (ratio >= BATCHABLE_WRITE_RATIO && savings >= BATCHABLE_MIN_SAVINGS_USD) {
|
|
917
|
+
out.push({
|
|
918
|
+
code: "batchable-writes",
|
|
919
|
+
message: `${Math.round(writes).toLocaleString("en-US")} writes/mo against data occupying at most ${maxWarmRows.toLocaleString("en-US")} Warm rows \u2014 each row rewritten ~${Math.round(ratio).toLocaleString("en-US")}\xD7 ($${writesUSD.toFixed(2)}/mo). IF these ids ever arrive together, addMany() collapses them to one write per chunk and bulkLoadCrbmGeneration writes Cold directly, taking the write term toward $${batchedFloorUSD.toFixed(2)}/mo. If they genuinely arrive one at a time (real-time qualification), add() is the correct path and this is simply what it costs.`,
|
|
920
|
+
currentUSD: writesUSD,
|
|
921
|
+
batchedFloorUSD
|
|
922
|
+
});
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
return out;
|
|
926
|
+
}
|
|
927
|
+
function buildReport(input) {
|
|
928
|
+
const { topology, pricing, grounded } = input;
|
|
929
|
+
const { cold, warm, redis } = pricing;
|
|
930
|
+
const S = SECONDS_PER_MONTH;
|
|
931
|
+
requireFiniteNonNeg(cold.getPerMillion, "pricing.cold.getPerMillion");
|
|
932
|
+
requireFiniteNonNeg(cold.putPerMillion, "pricing.cold.putPerMillion");
|
|
933
|
+
requireFiniteNonNeg(cold.storagePerGiBMonth, "pricing.cold.storagePerGiBMonth");
|
|
934
|
+
requireFiniteNonNeg(warm.rruPerMillion, "pricing.warm.rruPerMillion");
|
|
935
|
+
requireFiniteNonNeg(warm.wruPerMillion, "pricing.warm.wruPerMillion");
|
|
936
|
+
requireFinitePos(warm.readUnitKiB, "pricing.warm.readUnitKiB");
|
|
937
|
+
requireFinitePos(warm.writeUnitKiB, "pricing.warm.writeUnitKiB");
|
|
938
|
+
requireFiniteNonNeg(redis.monthlyUSD, "pricing.redis.monthlyUSD");
|
|
939
|
+
const coldBytes = requireFiniteNonNeg(input.coldBytes, "coldBytes");
|
|
940
|
+
const cacheHitRate = clamp01(
|
|
941
|
+
requireFiniteNonNeg(input.workload.cacheHitRate ?? 0, "cacheHitRate")
|
|
942
|
+
);
|
|
943
|
+
const readsPerSec = requireFiniteNonNeg(input.workload.readsPerSec ?? 0, "readsPerSec");
|
|
944
|
+
const writesPerSec = requireFiniteNonNeg(input.workload.writesPerSec ?? 0, "writesPerSec");
|
|
945
|
+
const intersectsPerSec = requireFiniteNonNeg(
|
|
946
|
+
input.workload.intersectsPerSec ?? 0,
|
|
947
|
+
"intersectsPerSec"
|
|
948
|
+
);
|
|
949
|
+
const avgItemKiB = requireFiniteNonNeg(input.workload.avgItemKiB ?? 8, "avgItemKiB");
|
|
950
|
+
const chunksPerIntersect = requireFiniteNonNeg(
|
|
951
|
+
input.workload.chunksPerIntersect ?? 1,
|
|
952
|
+
"chunksPerIntersect"
|
|
953
|
+
);
|
|
954
|
+
const compactionsPerMonth = requireFiniteNonNeg(
|
|
955
|
+
input.workload.compactionsPerMonth ?? 0,
|
|
956
|
+
"compactionsPerMonth"
|
|
957
|
+
);
|
|
958
|
+
const avgItemBytes = avgItemKiB * 1024;
|
|
959
|
+
const derivedChunks = avgItemBytes > 0 ? Math.ceil(coldBytes / avgItemBytes) : 0;
|
|
960
|
+
const chunksPerCompaction = requireFiniteNonNeg(
|
|
961
|
+
input.workload.chunksPerCompaction ?? derivedChunks,
|
|
962
|
+
"chunksPerCompaction"
|
|
963
|
+
);
|
|
964
|
+
const dirtyChunksPerCompaction = requireFiniteNonNeg(
|
|
965
|
+
input.workload.dirtyChunksPerCompaction ?? 0,
|
|
966
|
+
"dirtyChunksPerCompaction"
|
|
967
|
+
);
|
|
968
|
+
const rruMult = warm.stronglyConsistent ? 1 : 0.5;
|
|
969
|
+
const warmReadUSD = Math.max(1, Math.ceil(avgItemKiB / warm.readUnitKiB)) * rruMult * (warm.rruPerMillion / 1e6);
|
|
970
|
+
const warmWriteUSD = Math.max(1, Math.ceil(avgItemKiB / warm.writeUnitKiB)) * (warm.wruPerMillion / 1e6);
|
|
971
|
+
const coldGetUSD = cold.getPerMillion / 1e6;
|
|
972
|
+
const missFraction = 1 - cacheHitRate;
|
|
973
|
+
const readMisses = readsPerSec * S * missFraction;
|
|
974
|
+
const writes = writesPerSec * S;
|
|
975
|
+
const intersects = intersectsPerSec * S;
|
|
976
|
+
const storageUSD = coldBytes / GIB * cold.storagePerGiBMonth;
|
|
977
|
+
const readsUSD = topology === "A" ? readMisses * coldGetUSD : readMisses * warmReadUSD;
|
|
978
|
+
const writesUSD = topology === "B" ? writes * warmWriteUSD : 0;
|
|
979
|
+
const intersectsUSD = intersects * chunksPerIntersect * coldGetUSD;
|
|
980
|
+
const putUSD = cold.putPerMillion / 1e6;
|
|
981
|
+
const compactionColdUSD = compactionsPerMonth * (chunksPerCompaction * coldGetUSD + putUSD);
|
|
982
|
+
const compactionWarmUSD = compactionsPerMonth * dirtyChunksPerCompaction * (warmReadUSD + warmWriteUSD);
|
|
983
|
+
const compactionUSD = compactionColdUSD + compactionWarmUSD;
|
|
984
|
+
const total = readsUSD + writesUSD + intersectsUSD + storageUSD + compactionUSD;
|
|
985
|
+
const warmTierUSD = (topology === "B" ? readsUSD + writesUSD : 0) + compactionWarmUSD;
|
|
986
|
+
const coldTierUSD = storageUSD + (topology === "A" ? readsUSD : 0) + intersectsUSD + compactionColdUSD;
|
|
987
|
+
const headroom = Math.max(0, redis.monthlyUSD - storageUSD);
|
|
988
|
+
const perReadUSD = topology === "A" ? coldGetUSD : warmReadUSD;
|
|
989
|
+
const writesCross = topology === "B" && warmWriteUSD > 0 ? headroom / (S * warmWriteUSD) : Infinity;
|
|
990
|
+
const readsCross = perReadUSD > 0 && missFraction > 0 ? headroom / (S * perReadUSD * missFraction) : Infinity;
|
|
991
|
+
const verdict = total <= redis.monthlyUSD * 0.1 ? "win-big" : total < redis.monthlyUSD ? "win" : "lose-zone";
|
|
992
|
+
const dominant = Math.max(readsUSD, writesUSD, intersectsUSD, storageUSD, compactionUSD);
|
|
993
|
+
let driver = "storage";
|
|
994
|
+
if (dominant === writesUSD && writesUSD > 0)
|
|
995
|
+
driver = "warm writes (the per-1KiB WRU cost driver)";
|
|
996
|
+
else if (dominant === readsUSD && readsUSD > 0)
|
|
997
|
+
driver = topology === "A" ? "cold reads (S3 GETs)" : "warm reads";
|
|
998
|
+
else if (dominant === intersectsUSD && intersectsUSD > 0) driver = "intersection cold fetches";
|
|
999
|
+
else if (dominant === compactionUSD && compactionUSD > 0)
|
|
1000
|
+
driver = "compaction (whole-generation re-read)";
|
|
1001
|
+
const rationale = verdict === "lose-zone" ? `pay-per-use total $${total.toFixed(2)}/mo exceeds the $${redis.monthlyUSD}/mo flat baseline; dominated by ${driver}` : verdict === "win-big" ? `$${total.toFixed(2)}/mo \u2014 \u226410% of the $${redis.monthlyUSD}/mo baseline; dominated by ${driver}` : `$${total.toFixed(2)}/mo, under the $${redis.monthlyUSD}/mo baseline; dominated by ${driver}`;
|
|
1002
|
+
const notes = [
|
|
1003
|
+
"S3\u2192same-region egress treated as free; internet egress is not modeled.",
|
|
1004
|
+
"Warm delta storage treated as negligible (only dirty chunks).",
|
|
1005
|
+
"Request cost is from the supplied workload rates (live-metrics-derived request cost is a later phase).",
|
|
1006
|
+
compactionsPerMonth > 0 ? `Compaction modeled: ${compactionsPerMonth}/mo \xD7 (${chunksPerCompaction} Cold GET + 1 PUT${dirtyChunksPerCompaction > 0 ? ` + ${dirtyChunksPerCompaction}-row Warm purge` : ""}); the whole-generation re-read dominates. GC delete of the superseded generation is free (not modeled).` : "Compaction is NOT modeled \u2014 set workload.compactionsPerMonth (+ chunksPerCompaction) to include the whole-generation re-read that dominates operational cost.",
|
|
1007
|
+
...input.extraNotes ?? []
|
|
1008
|
+
];
|
|
1009
|
+
return {
|
|
1010
|
+
monthlyUSD: {
|
|
1011
|
+
byTier: { hot: 0, warm: warmTierUSD, cold: coldTierUSD },
|
|
1012
|
+
byOp: {
|
|
1013
|
+
reads: readsUSD,
|
|
1014
|
+
writes: writesUSD,
|
|
1015
|
+
intersects: intersectsUSD,
|
|
1016
|
+
storage: storageUSD,
|
|
1017
|
+
compaction: compactionUSD
|
|
1018
|
+
},
|
|
1019
|
+
total
|
|
1020
|
+
},
|
|
1021
|
+
redisCrossover: { writesPerSec: writesCross, readsPerSec: readsCross },
|
|
1022
|
+
verdict,
|
|
1023
|
+
rationale,
|
|
1024
|
+
// Topology-A has no per-op writes (they arrive via bulk-load), so `writesUSD` is 0 there and the advisory
|
|
1025
|
+
// cannot fire — correctly, since bulk-load is already the batched path.
|
|
1026
|
+
advisories: deriveAdvisories({
|
|
1027
|
+
writes,
|
|
1028
|
+
writesUSD,
|
|
1029
|
+
maxWarmRows: requireFiniteNonNeg(input.maxWarmRows, "maxWarmRows"),
|
|
1030
|
+
warmWriteUSD
|
|
1031
|
+
}),
|
|
1032
|
+
assumptions: { cacheHitRate, pricingName: pricing.name, grounded, topology, notes }
|
|
1033
|
+
};
|
|
1034
|
+
}
|
|
1035
|
+
function estimateCost(input) {
|
|
1036
|
+
const topology = input.topology ?? "A";
|
|
1037
|
+
const pricing = input.pricing ?? DEFAULT_PRICING;
|
|
1038
|
+
const workload = input.workload ?? {};
|
|
1039
|
+
let coldBytes = 0;
|
|
1040
|
+
let maxWarmRows = 0;
|
|
1041
|
+
for (const spec of input.segments) {
|
|
1042
|
+
const count = Math.floor(requireFiniteNonNeg(spec.count ?? 1, "segment.count"));
|
|
1043
|
+
coldBytes += sizingBytes(spec) * count;
|
|
1044
|
+
const perSegment = spec.cardinality !== void 0 ? Math.min(
|
|
1045
|
+
Math.floor(requireFiniteNonNeg(spec.cardinality, "segment.cardinality")),
|
|
1046
|
+
CHUNK_COUNT
|
|
1047
|
+
) : CHUNK_COUNT;
|
|
1048
|
+
maxWarmRows += perSegment * count;
|
|
1049
|
+
}
|
|
1050
|
+
return buildReport({ coldBytes, workload, topology, pricing, grounded: false, maxWarmRows });
|
|
1051
|
+
}
|
|
1052
|
+
function groundedReport(input) {
|
|
1053
|
+
return buildReport({
|
|
1054
|
+
coldBytes: input.coldBytes,
|
|
1055
|
+
workload: input.workload ?? {},
|
|
1056
|
+
topology: input.topology ?? "A",
|
|
1057
|
+
pricing: input.pricing ?? DEFAULT_PRICING,
|
|
1058
|
+
grounded: input.grounded ?? true,
|
|
1059
|
+
extraNotes: input.extraNotes,
|
|
1060
|
+
maxWarmRows: input.maxWarmRows ?? CHUNK_COUNT
|
|
1061
|
+
});
|
|
1062
|
+
}
|
|
1063
|
+
|
|
1064
|
+
// src/core/audit.ts
|
|
1065
|
+
var NOOP_AUDIT = {
|
|
1066
|
+
onEvent() {
|
|
1067
|
+
}
|
|
1068
|
+
};
|
|
1069
|
+
function safeAudit(sink) {
|
|
1070
|
+
if (sink === NOOP_AUDIT) return sink;
|
|
1071
|
+
return {
|
|
1072
|
+
onEvent(event) {
|
|
1073
|
+
try {
|
|
1074
|
+
sink.onEvent(event);
|
|
1075
|
+
} catch {
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
};
|
|
1079
|
+
}
|
|
1080
|
+
var RecordingAuditSink = class {
|
|
1081
|
+
recorded = [];
|
|
1082
|
+
onEvent(event) {
|
|
1083
|
+
this.recorded.push(event);
|
|
1084
|
+
}
|
|
1085
|
+
/** An independent copy of the recorded events, in emission order. */
|
|
1086
|
+
snapshot() {
|
|
1087
|
+
return this.recorded.map((e) => ({ ...e }));
|
|
1088
|
+
}
|
|
1089
|
+
reset() {
|
|
1090
|
+
this.recorded.length = 0;
|
|
1091
|
+
}
|
|
1092
|
+
};
|
|
1093
|
+
|
|
1094
|
+
// src/core/codec.ts
|
|
1095
|
+
function requireCodec(codec, api) {
|
|
1096
|
+
if (codec === void 0) {
|
|
1097
|
+
throw new ValidationError(
|
|
1098
|
+
`${api} needs a bitmap codec: pass \`codec\` (e.g. \`roaringCodec\` from @cloudbitmaps/roaring). @cloudbitmaps/core is codec-agnostic and has no default \u2014 install a flavor package, whose equivalents bind the codec for you.`
|
|
1099
|
+
);
|
|
1100
|
+
}
|
|
1101
|
+
return codec;
|
|
1102
|
+
}
|
|
1103
|
+
|
|
1104
|
+
// src/core/crypto.ts
|
|
1105
|
+
var AAD_VERSION = 1;
|
|
1106
|
+
var SCOPE_CHUNK = 0;
|
|
1107
|
+
var SCOPE_INDEX = 1;
|
|
1108
|
+
function aadFor(ref, generation, scope) {
|
|
1109
|
+
const enc = new TextEncoder();
|
|
1110
|
+
const ns = enc.encode(ref.namespace ?? "");
|
|
1111
|
+
const seg = enc.encode(ref.segment);
|
|
1112
|
+
const out = new Uint8Array(1 + 2 + ns.length + 2 + seg.length + 8 + 1 + 4);
|
|
1113
|
+
const view = new DataView(out.buffer);
|
|
1114
|
+
let o = 0;
|
|
1115
|
+
out[o] = AAD_VERSION;
|
|
1116
|
+
o += 1;
|
|
1117
|
+
view.setUint16(o, ns.length, true);
|
|
1118
|
+
o += 2;
|
|
1119
|
+
out.set(ns, o);
|
|
1120
|
+
o += ns.length;
|
|
1121
|
+
view.setUint16(o, seg.length, true);
|
|
1122
|
+
o += 2;
|
|
1123
|
+
out.set(seg, o);
|
|
1124
|
+
o += seg.length;
|
|
1125
|
+
view.setBigUint64(o, BigInt(generation), true);
|
|
1126
|
+
o += 8;
|
|
1127
|
+
out[o] = scope === "index" ? SCOPE_INDEX : SCOPE_CHUNK;
|
|
1128
|
+
o += 1;
|
|
1129
|
+
view.setUint32(o, scope === "index" ? 0 : scope, true);
|
|
1130
|
+
return out;
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
// src/core/validate.ts
|
|
1134
|
+
var CHUNK_KEY_MAX = 65535;
|
|
1135
|
+
var NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,255}$/;
|
|
1136
|
+
function validatePart(value, field) {
|
|
1137
|
+
if (typeof value !== "string" || !NAME.test(value) || value.includes("..")) {
|
|
1138
|
+
throw new ValidationError(
|
|
1139
|
+
`${field} must match ${String(NAME)} and contain no "..": got ${JSON.stringify(value)}`
|
|
1140
|
+
);
|
|
1141
|
+
}
|
|
1142
|
+
}
|
|
1143
|
+
function validateSegmentRef(ref) {
|
|
1144
|
+
validatePart(ref.segment, "segment");
|
|
1145
|
+
if (ref.namespace !== void 0) validatePart(ref.namespace, "namespace");
|
|
1146
|
+
}
|
|
1147
|
+
function validateChunkRef(ref) {
|
|
1148
|
+
validateSegmentRef(ref);
|
|
1149
|
+
if (!Number.isInteger(ref.chunkKey) || ref.chunkKey < 0 || ref.chunkKey > CHUNK_KEY_MAX) {
|
|
1150
|
+
throw new ValidationError(
|
|
1151
|
+
`chunkKey must be an integer in [0, ${CHUNK_KEY_MAX}]; got ${ref.chunkKey}`
|
|
1152
|
+
);
|
|
1153
|
+
}
|
|
1154
|
+
}
|
|
1155
|
+
|
|
1156
|
+
// src/core/crbm/crc32c.ts
|
|
1157
|
+
var TABLE = (() => {
|
|
1158
|
+
const POLY = 2197175160;
|
|
1159
|
+
const table = new Uint32Array(256);
|
|
1160
|
+
for (let n = 0; n < 256; n++) {
|
|
1161
|
+
let c = n;
|
|
1162
|
+
for (let k = 0; k < 8; k++) {
|
|
1163
|
+
c = c & 1 ? POLY ^ c >>> 1 : c >>> 1;
|
|
1164
|
+
}
|
|
1165
|
+
table[n] = c >>> 0;
|
|
1166
|
+
}
|
|
1167
|
+
return table;
|
|
1168
|
+
})();
|
|
1169
|
+
function crc32c(bytes, seed = 0) {
|
|
1170
|
+
let crc = (seed ^ 4294967295) >>> 0;
|
|
1171
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
1172
|
+
crc = (TABLE[(crc ^ bytes[i]) & 255] ^ crc >>> 8) >>> 0;
|
|
1173
|
+
}
|
|
1174
|
+
return (crc ^ 4294967295) >>> 0;
|
|
1175
|
+
}
|
|
1176
|
+
|
|
1177
|
+
// src/core/crbm/varint.ts
|
|
1178
|
+
var U32_CEIL = 4294967296;
|
|
1179
|
+
function writeVarint(out, value) {
|
|
1180
|
+
if (!Number.isInteger(value) || value < 0 || value >= U32_CEIL) {
|
|
1181
|
+
throw new RangeError(`varint value must be a u32; got ${value}`);
|
|
1182
|
+
}
|
|
1183
|
+
let v = value;
|
|
1184
|
+
while (v > 127) {
|
|
1185
|
+
out.push(v & 127 | 128);
|
|
1186
|
+
v >>>= 7;
|
|
1187
|
+
}
|
|
1188
|
+
out.push(v);
|
|
1189
|
+
}
|
|
1190
|
+
function readVarint(bytes, offset) {
|
|
1191
|
+
let result = 0;
|
|
1192
|
+
let shift = 0;
|
|
1193
|
+
let pos = offset;
|
|
1194
|
+
for (let i = 0; i < 5; i++) {
|
|
1195
|
+
if (pos >= bytes.length) {
|
|
1196
|
+
throw new IntegrityError("varint truncated past end of buffer");
|
|
1197
|
+
}
|
|
1198
|
+
const byte = bytes[pos];
|
|
1199
|
+
pos += 1;
|
|
1200
|
+
if (i === 4 && byte > 15) {
|
|
1201
|
+
throw new IntegrityError("varint exceeds u32 range");
|
|
1202
|
+
}
|
|
1203
|
+
result += (byte & 127) * 2 ** shift;
|
|
1204
|
+
if ((byte & 128) === 0) {
|
|
1205
|
+
return { value: result, next: pos };
|
|
1206
|
+
}
|
|
1207
|
+
shift += 7;
|
|
1208
|
+
}
|
|
1209
|
+
throw new IntegrityError("varint exceeds u32 range");
|
|
1210
|
+
}
|
|
1211
|
+
|
|
1212
|
+
// src/core/crbm/format.ts
|
|
1213
|
+
var MAGIC = Uint8Array.of(67, 82, 66, 77);
|
|
1214
|
+
var VERSION_MAJOR = 1;
|
|
1215
|
+
var VERSION_MINOR = 0;
|
|
1216
|
+
var PREAMBLE_BYTES = 8;
|
|
1217
|
+
var PAYLOAD_START = PREAMBLE_BYTES;
|
|
1218
|
+
var FOOTER_BYTES = 104;
|
|
1219
|
+
var FOOTER = {
|
|
1220
|
+
indexOffset: 0,
|
|
1221
|
+
// u64
|
|
1222
|
+
indexLength: 8,
|
|
1223
|
+
// u64
|
|
1224
|
+
indexCrc32c: 16,
|
|
1225
|
+
// u32
|
|
1226
|
+
flags: 20,
|
|
1227
|
+
// u32
|
|
1228
|
+
roaringSerializationId: 24,
|
|
1229
|
+
// u16
|
|
1230
|
+
elementWidth: 26,
|
|
1231
|
+
// u8
|
|
1232
|
+
containerCodec: 27,
|
|
1233
|
+
// u8
|
|
1234
|
+
versionMajor: 28,
|
|
1235
|
+
// u8
|
|
1236
|
+
versionMinor: 29,
|
|
1237
|
+
// u16
|
|
1238
|
+
generation: 32,
|
|
1239
|
+
// u64
|
|
1240
|
+
indexNonce: 40,
|
|
1241
|
+
// 12 B
|
|
1242
|
+
indexTag: 52,
|
|
1243
|
+
// 16 B
|
|
1244
|
+
chunkCount: 84,
|
|
1245
|
+
// u32
|
|
1246
|
+
totalCardinality: 88,
|
|
1247
|
+
// u64
|
|
1248
|
+
footerCrc32c: 96,
|
|
1249
|
+
// u32 — covers footer bytes [0, 96)
|
|
1250
|
+
endMagic: 100
|
|
1251
|
+
// char[4]
|
|
1252
|
+
};
|
|
1253
|
+
var FOOTER_CRC_COVERAGE = FOOTER.footerCrc32c;
|
|
1254
|
+
var FLAG_ENCRYPTED = 1 << 0;
|
|
1255
|
+
var FLAG_LITTLE_ENDIAN = 1 << 2;
|
|
1256
|
+
var ELEMENT_WIDTH_32 = 32;
|
|
1257
|
+
var ROARING_PORTABLE_ID = 1;
|
|
1258
|
+
var CONTAINER_CODEC_NONE = 0;
|
|
1259
|
+
var DEFAULT_TAIL_BYTES = 256 * 1024;
|
|
1260
|
+
var DEFAULT_MAX_INDEX_BYTES = 8 * 1024 * 1024;
|
|
1261
|
+
var DEFAULT_MAX_PAYLOAD_BYTES = 16 * 1024 * 1024;
|
|
1262
|
+
var MAX_CHUNK_CARDINALITY = 65536;
|
|
1263
|
+
var CRC32C_BYTES = 4;
|
|
1264
|
+
var AEAD_NONCE_BYTES = 12;
|
|
1265
|
+
var AEAD_TAG_BYTES = 16;
|
|
1266
|
+
|
|
1267
|
+
// src/core/crbm/reader.ts
|
|
1268
|
+
var RETAINED_BYTES_PER_INDEX_ENTRY = 160;
|
|
1269
|
+
function magicMatches(bytes, offset) {
|
|
1270
|
+
return bytes[offset] === MAGIC[0] && bytes[offset + 1] === MAGIC[1] && bytes[offset + 2] === MAGIC[2] && bytes[offset + 3] === MAGIC[3];
|
|
1271
|
+
}
|
|
1272
|
+
function readU64(view, offset, field) {
|
|
1273
|
+
const big = view.getBigUint64(offset, true);
|
|
1274
|
+
if (big > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
1275
|
+
throw new IntegrityError(`.crbm ${field} ${big} exceeds safe-integer range`);
|
|
1276
|
+
}
|
|
1277
|
+
return Number(big);
|
|
1278
|
+
}
|
|
1279
|
+
var CrbmReader = class _CrbmReader {
|
|
1280
|
+
constructor(blob, objectSize, generation, totalCardinality, entries, orderedKeys, servedFromTail, crypto) {
|
|
1281
|
+
this.blob = blob;
|
|
1282
|
+
this.objectSize = objectSize;
|
|
1283
|
+
this.generation = generation;
|
|
1284
|
+
this.totalCardinality = totalCardinality;
|
|
1285
|
+
this.entries = entries;
|
|
1286
|
+
this.orderedKeys = orderedKeys;
|
|
1287
|
+
this.servedFromTail = servedFromTail;
|
|
1288
|
+
this.crypto = crypto;
|
|
1289
|
+
}
|
|
1290
|
+
blob;
|
|
1291
|
+
objectSize;
|
|
1292
|
+
generation;
|
|
1293
|
+
totalCardinality;
|
|
1294
|
+
entries;
|
|
1295
|
+
orderedKeys;
|
|
1296
|
+
servedFromTail;
|
|
1297
|
+
crypto;
|
|
1298
|
+
/** Total object bytes (from the one-GET tail read) — for grounded storage cost (Phase 5b). */
|
|
1299
|
+
get sizeBytes() {
|
|
1300
|
+
return this.objectSize;
|
|
1301
|
+
}
|
|
1302
|
+
/**
|
|
1303
|
+
* Estimated retained JS heap of this reader's parsed index — the weight the cold reader cache bounds on
|
|
1304
|
+
* (gap #1: a wide segment's index dominates the reader's footprint). `entries.size` scales with the number
|
|
1305
|
+
* of resident chunks (≤ 65536), so this is `entries.size × {@link RETAINED_BYTES_PER_INDEX_ENTRY}`.
|
|
1306
|
+
*/
|
|
1307
|
+
get retainedIndexBytes() {
|
|
1308
|
+
return this.entries.size * RETAINED_BYTES_PER_INDEX_ENTRY;
|
|
1309
|
+
}
|
|
1310
|
+
/** Per-chunk cardinality (`chunkKey → count`) from the parsed index — no payload reads (Phase 5c cheap count). */
|
|
1311
|
+
cardinalities() {
|
|
1312
|
+
const out = /* @__PURE__ */ new Map();
|
|
1313
|
+
for (const [chunkKey, entry] of this.entries) out.set(chunkKey, entry.cardinality);
|
|
1314
|
+
return out;
|
|
1315
|
+
}
|
|
1316
|
+
static async open(blob, options = {}) {
|
|
1317
|
+
const tailBytes = Math.max(options.tailBytes ?? DEFAULT_TAIL_BYTES, FOOTER_BYTES);
|
|
1318
|
+
const maxPayloadBytes = options.maxPayloadBytes ?? DEFAULT_MAX_PAYLOAD_BYTES;
|
|
1319
|
+
const maxIndexBytes = options.maxIndexBytes ?? DEFAULT_MAX_INDEX_BYTES;
|
|
1320
|
+
const { bytes: tail, size } = await blob.getTail(tailBytes);
|
|
1321
|
+
if (size < PREAMBLE_BYTES + FOOTER_BYTES || tail.length < FOOTER_BYTES) {
|
|
1322
|
+
throw new IntegrityError(`.crbm too small: ${size}B`);
|
|
1323
|
+
}
|
|
1324
|
+
const footer = tail.subarray(tail.length - FOOTER_BYTES);
|
|
1325
|
+
if (!magicMatches(footer, FOOTER.endMagic)) {
|
|
1326
|
+
throw new IntegrityError(".crbm end magic mismatch");
|
|
1327
|
+
}
|
|
1328
|
+
const fview = new DataView(footer.buffer, footer.byteOffset, footer.byteLength);
|
|
1329
|
+
const storedFooterCrc = fview.getUint32(FOOTER.footerCrc32c, true);
|
|
1330
|
+
if (crc32c(footer.subarray(0, FOOTER_CRC_COVERAGE)) !== storedFooterCrc) {
|
|
1331
|
+
throw new IntegrityError(".crbm footer CRC mismatch");
|
|
1332
|
+
}
|
|
1333
|
+
const versionMajor = footer[FOOTER.versionMajor];
|
|
1334
|
+
if (versionMajor !== VERSION_MAJOR) {
|
|
1335
|
+
throw new UnsupportedError(`.crbm major version ${versionMajor} not supported`);
|
|
1336
|
+
}
|
|
1337
|
+
const flags = fview.getUint32(FOOTER.flags, true);
|
|
1338
|
+
const encrypted = (flags & FLAG_ENCRYPTED) !== 0;
|
|
1339
|
+
if (encrypted && options.crypto === void 0) {
|
|
1340
|
+
throw new ValidationError(".crbm is encrypted but no decryption key (crypto) was provided");
|
|
1341
|
+
}
|
|
1342
|
+
if ((flags & FLAG_LITTLE_ENDIAN) === 0) {
|
|
1343
|
+
throw new UnsupportedError(".crbm big-endian layout not supported (v1 is little-endian)");
|
|
1344
|
+
}
|
|
1345
|
+
if ((flags & -6) !== 0) {
|
|
1346
|
+
throw new UnsupportedError(`.crbm has unknown flag bits set: 0x${flags.toString(16)}`);
|
|
1347
|
+
}
|
|
1348
|
+
const elementWidth = footer[FOOTER.elementWidth];
|
|
1349
|
+
if (elementWidth !== ELEMENT_WIDTH_32) {
|
|
1350
|
+
throw new UnsupportedError(
|
|
1351
|
+
`.crbm element_width ${elementWidth} not supported (v1 reads 32-bit ids; 64-bit is a future major version)`
|
|
1352
|
+
);
|
|
1353
|
+
}
|
|
1354
|
+
const serializationId = fview.getUint16(FOOTER.roaringSerializationId, true);
|
|
1355
|
+
if (serializationId !== ROARING_PORTABLE_ID) {
|
|
1356
|
+
throw new UnsupportedError(
|
|
1357
|
+
`.crbm roaring_serialization_id ${serializationId} not supported (v1 is portable=${ROARING_PORTABLE_ID})`
|
|
1358
|
+
);
|
|
1359
|
+
}
|
|
1360
|
+
const containerCodec = footer[FOOTER.containerCodec];
|
|
1361
|
+
if (containerCodec !== CONTAINER_CODEC_NONE) {
|
|
1362
|
+
throw new UnsupportedError(
|
|
1363
|
+
`.crbm container_codec ${containerCodec} not supported (v1 defines only ${CONTAINER_CODEC_NONE}=none)`
|
|
1364
|
+
);
|
|
1365
|
+
}
|
|
1366
|
+
const indexOffset = readU64(fview, FOOTER.indexOffset, "index_offset");
|
|
1367
|
+
const indexLength = readU64(fview, FOOTER.indexLength, "index_length");
|
|
1368
|
+
const indexCrc = fview.getUint32(FOOTER.indexCrc32c, true);
|
|
1369
|
+
const chunkCount = fview.getUint32(FOOTER.chunkCount, true);
|
|
1370
|
+
const totalCardinality = readU64(fview, FOOTER.totalCardinality, "total_cardinality");
|
|
1371
|
+
const generation = readU64(fview, FOOTER.generation, "generation");
|
|
1372
|
+
if (indexOffset < PAYLOAD_START || indexOffset + indexLength > size - FOOTER_BYTES) {
|
|
1373
|
+
throw new IntegrityError(
|
|
1374
|
+
`.crbm index region [${indexOffset}, +${indexLength}) out of bounds`
|
|
1375
|
+
);
|
|
1376
|
+
}
|
|
1377
|
+
if (indexLength > maxIndexBytes) {
|
|
1378
|
+
throw new IntegrityError(`.crbm index ${indexLength}B exceeds cap ${maxIndexBytes}B`);
|
|
1379
|
+
}
|
|
1380
|
+
const tailCoversFrom = size - tail.length;
|
|
1381
|
+
if (tailCoversFrom === 0) {
|
|
1382
|
+
if (!magicMatches(tail, 0) || tail[4] !== versionMajor) {
|
|
1383
|
+
throw new IntegrityError(".crbm preamble magic/version mismatch with footer");
|
|
1384
|
+
}
|
|
1385
|
+
}
|
|
1386
|
+
let indexBytes;
|
|
1387
|
+
let servedFromTail;
|
|
1388
|
+
if (indexOffset >= tailCoversFrom) {
|
|
1389
|
+
const start = indexOffset - tailCoversFrom;
|
|
1390
|
+
indexBytes = tail.subarray(start, start + indexLength);
|
|
1391
|
+
servedFromTail = true;
|
|
1392
|
+
} else {
|
|
1393
|
+
indexBytes = await blob.getRange(indexOffset, indexLength);
|
|
1394
|
+
servedFromTail = false;
|
|
1395
|
+
}
|
|
1396
|
+
if (crc32c(indexBytes) !== indexCrc) {
|
|
1397
|
+
throw new IntegrityError(".crbm index CRC mismatch");
|
|
1398
|
+
}
|
|
1399
|
+
const indexForParse = encrypted ? options.crypto.aead.open(
|
|
1400
|
+
{
|
|
1401
|
+
nonce: footer.subarray(FOOTER.indexNonce, FOOTER.indexNonce + AEAD_NONCE_BYTES),
|
|
1402
|
+
ciphertext: indexBytes,
|
|
1403
|
+
tag: footer.subarray(FOOTER.indexTag, FOOTER.indexTag + AEAD_TAG_BYTES)
|
|
1404
|
+
},
|
|
1405
|
+
options.crypto.aadFor("index")
|
|
1406
|
+
) : indexBytes;
|
|
1407
|
+
const { entries, orderedKeys, cardinalitySum } = parseIndex(
|
|
1408
|
+
indexForParse,
|
|
1409
|
+
size,
|
|
1410
|
+
maxPayloadBytes
|
|
1411
|
+
);
|
|
1412
|
+
if (encrypted) {
|
|
1413
|
+
if (chunkCount !== 0 || totalCardinality !== 0) {
|
|
1414
|
+
throw new IntegrityError(
|
|
1415
|
+
".crbm encrypted footer must zero chunk_count + total_cardinality"
|
|
1416
|
+
);
|
|
1417
|
+
}
|
|
1418
|
+
} else {
|
|
1419
|
+
if (entries.size !== chunkCount) {
|
|
1420
|
+
throw new IntegrityError(
|
|
1421
|
+
`.crbm chunk_count ${chunkCount} != ${entries.size} index entries`
|
|
1422
|
+
);
|
|
1423
|
+
}
|
|
1424
|
+
if (cardinalitySum !== totalCardinality) {
|
|
1425
|
+
throw new IntegrityError(
|
|
1426
|
+
`.crbm total_cardinality ${totalCardinality} != \u03A3 index cardinality ${cardinalitySum}`
|
|
1427
|
+
);
|
|
1428
|
+
}
|
|
1429
|
+
}
|
|
1430
|
+
return new _CrbmReader(
|
|
1431
|
+
blob,
|
|
1432
|
+
size,
|
|
1433
|
+
generation,
|
|
1434
|
+
cardinalitySum,
|
|
1435
|
+
entries,
|
|
1436
|
+
orderedKeys,
|
|
1437
|
+
servedFromTail,
|
|
1438
|
+
options.crypto
|
|
1439
|
+
);
|
|
1440
|
+
}
|
|
1441
|
+
/** Chunk keys present in this generation, ascending. */
|
|
1442
|
+
chunkKeys() {
|
|
1443
|
+
return this.orderedKeys.slice();
|
|
1444
|
+
}
|
|
1445
|
+
/** Segment cardinality from the footer (no payload reads) — the cheap `count()` path. */
|
|
1446
|
+
count() {
|
|
1447
|
+
return this.totalCardinality;
|
|
1448
|
+
}
|
|
1449
|
+
has(chunkKey) {
|
|
1450
|
+
return this.entries.has(chunkKey);
|
|
1451
|
+
}
|
|
1452
|
+
/**
|
|
1453
|
+
* Range-GET one chunk's payload, verifying its CRC32C before returning. Returns `null` if the chunk is
|
|
1454
|
+
* absent. Throws `IntegrityError` on a CRC mismatch (the bytes must never reach the native
|
|
1455
|
+
* deserializer). The returned buffer is a **read-only view** owned by the reader/driver — callers must
|
|
1456
|
+
* not mutate it.
|
|
1457
|
+
*/
|
|
1458
|
+
async getChunk(chunkKey) {
|
|
1459
|
+
const e = this.entries.get(chunkKey);
|
|
1460
|
+
if (e === void 0) return null;
|
|
1461
|
+
if (e.offset < PAYLOAD_START || e.offset + e.length > this.objectSize - FOOTER_BYTES) {
|
|
1462
|
+
throw new IntegrityError(`chunk ${chunkKey} payload out of bounds`);
|
|
1463
|
+
}
|
|
1464
|
+
const bytes = await this.blob.getRange(e.offset, e.length);
|
|
1465
|
+
if (crc32c(bytes) !== e.crc32c) {
|
|
1466
|
+
throw new IntegrityError(`chunk ${chunkKey} payload CRC mismatch`);
|
|
1467
|
+
}
|
|
1468
|
+
if (this.crypto === void 0) return bytes;
|
|
1469
|
+
if (bytes.length < AEAD_NONCE_BYTES + AEAD_TAG_BYTES) {
|
|
1470
|
+
throw new IntegrityError(`chunk ${chunkKey} encrypted payload too short`);
|
|
1471
|
+
}
|
|
1472
|
+
return this.crypto.aead.open(
|
|
1473
|
+
{
|
|
1474
|
+
nonce: bytes.subarray(0, AEAD_NONCE_BYTES),
|
|
1475
|
+
ciphertext: bytes.subarray(AEAD_NONCE_BYTES, bytes.length - AEAD_TAG_BYTES),
|
|
1476
|
+
tag: bytes.subarray(bytes.length - AEAD_TAG_BYTES)
|
|
1477
|
+
},
|
|
1478
|
+
this.crypto.aadFor(chunkKey)
|
|
1479
|
+
);
|
|
1480
|
+
}
|
|
1481
|
+
};
|
|
1482
|
+
function parseIndex(indexBytes, objectSize, maxPayloadBytes) {
|
|
1483
|
+
const entries = /* @__PURE__ */ new Map();
|
|
1484
|
+
const orderedKeys = [];
|
|
1485
|
+
let pos = 0;
|
|
1486
|
+
let prevKey = 0;
|
|
1487
|
+
let prevEnd = PAYLOAD_START;
|
|
1488
|
+
let cardinalitySum = 0;
|
|
1489
|
+
for (let i = 0; pos < indexBytes.length; i++) {
|
|
1490
|
+
const keyDelta = readVarint(indexBytes, pos);
|
|
1491
|
+
const offDelta = readVarint(indexBytes, keyDelta.next);
|
|
1492
|
+
const len = readVarint(indexBytes, offDelta.next);
|
|
1493
|
+
const card = readVarint(indexBytes, len.next);
|
|
1494
|
+
pos = card.next;
|
|
1495
|
+
if (pos + CRC32C_BYTES > indexBytes.length) {
|
|
1496
|
+
throw new IntegrityError(".crbm index truncated (missing payload CRC)");
|
|
1497
|
+
}
|
|
1498
|
+
const crc = (indexBytes[pos] | indexBytes[pos + 1] << 8 | indexBytes[pos + 2] << 16 | indexBytes[pos + 3] << 24) >>> 0;
|
|
1499
|
+
pos += CRC32C_BYTES;
|
|
1500
|
+
const chunkKey = prevKey + keyDelta.value;
|
|
1501
|
+
const offset = prevEnd + offDelta.value;
|
|
1502
|
+
if (i > 0 && keyDelta.value === 0) {
|
|
1503
|
+
throw new IntegrityError(`.crbm index has a duplicate chunkKey ${chunkKey}`);
|
|
1504
|
+
}
|
|
1505
|
+
if (chunkKey > 65535) throw new IntegrityError(`.crbm chunkKey ${chunkKey} out of range`);
|
|
1506
|
+
if (len.value === 0 || len.value > maxPayloadBytes) {
|
|
1507
|
+
throw new IntegrityError(`.crbm chunk ${chunkKey} length ${len.value} invalid`);
|
|
1508
|
+
}
|
|
1509
|
+
if (card.value < 1 || card.value > MAX_CHUNK_CARDINALITY) {
|
|
1510
|
+
throw new IntegrityError(`.crbm chunk ${chunkKey} cardinality ${card.value} invalid`);
|
|
1511
|
+
}
|
|
1512
|
+
if (offset < PAYLOAD_START || offset + len.value > objectSize - FOOTER_BYTES) {
|
|
1513
|
+
throw new IntegrityError(`.crbm chunk ${chunkKey} payload out of bounds`);
|
|
1514
|
+
}
|
|
1515
|
+
entries.set(chunkKey, {
|
|
1516
|
+
chunkKey,
|
|
1517
|
+
offset,
|
|
1518
|
+
length: len.value,
|
|
1519
|
+
cardinality: card.value,
|
|
1520
|
+
crc32c: crc
|
|
1521
|
+
});
|
|
1522
|
+
orderedKeys.push(chunkKey);
|
|
1523
|
+
cardinalitySum += card.value;
|
|
1524
|
+
prevKey = chunkKey;
|
|
1525
|
+
prevEnd = offset + len.value;
|
|
1526
|
+
}
|
|
1527
|
+
return { entries, orderedKeys, cardinalitySum };
|
|
1528
|
+
}
|
|
1529
|
+
|
|
1530
|
+
// src/core/crbm/writer.ts
|
|
1531
|
+
var CrbmWriter = class {
|
|
1532
|
+
constructor(sink, options) {
|
|
1533
|
+
this.sink = sink;
|
|
1534
|
+
this.options = options;
|
|
1535
|
+
if (!Number.isInteger(options.generation) || options.generation < 0) {
|
|
1536
|
+
throw new ValidationError(
|
|
1537
|
+
`generation must be a non-negative integer; got ${options.generation}`
|
|
1538
|
+
);
|
|
1539
|
+
}
|
|
1540
|
+
}
|
|
1541
|
+
sink;
|
|
1542
|
+
options;
|
|
1543
|
+
entries = [];
|
|
1544
|
+
offset = PAYLOAD_START;
|
|
1545
|
+
lastKey = -1;
|
|
1546
|
+
totalCardinality = 0;
|
|
1547
|
+
preambleWritten = false;
|
|
1548
|
+
finished = false;
|
|
1549
|
+
/**
|
|
1550
|
+
* Append one chunk's payload (the roaring-serialized bytes) with its cardinality. `chunkKey` must be
|
|
1551
|
+
* strictly greater than the previous one (ascending, no duplicates); `cardinality` is `[1, 65536]`.
|
|
1552
|
+
*/
|
|
1553
|
+
async addChunk(chunkKey, payload, cardinality) {
|
|
1554
|
+
if (this.finished) throw new ValidationError("CrbmWriter already finished");
|
|
1555
|
+
if (!Number.isInteger(chunkKey) || chunkKey < 0 || chunkKey > 65535) {
|
|
1556
|
+
throw new ValidationError(`chunkKey must be a u16; got ${chunkKey}`);
|
|
1557
|
+
}
|
|
1558
|
+
if (chunkKey <= this.lastKey) {
|
|
1559
|
+
throw new ValidationError(
|
|
1560
|
+
`chunkKey ${chunkKey} not strictly ascending (last ${this.lastKey})`
|
|
1561
|
+
);
|
|
1562
|
+
}
|
|
1563
|
+
if (!Number.isInteger(cardinality) || cardinality < 1 || cardinality > MAX_CHUNK_CARDINALITY) {
|
|
1564
|
+
throw new ValidationError(
|
|
1565
|
+
`cardinality must be in [1, ${MAX_CHUNK_CARDINALITY}]; got ${cardinality}`
|
|
1566
|
+
);
|
|
1567
|
+
}
|
|
1568
|
+
if (payload.length === 0) throw new ValidationError("chunk payload must be non-empty");
|
|
1569
|
+
await this.ensurePreamble();
|
|
1570
|
+
const stored = this.options.crypto ? this.sealChunk(chunkKey, payload) : payload;
|
|
1571
|
+
await this.sink.write(stored);
|
|
1572
|
+
this.entries.push({
|
|
1573
|
+
chunkKey,
|
|
1574
|
+
offset: this.offset,
|
|
1575
|
+
length: stored.length,
|
|
1576
|
+
cardinality,
|
|
1577
|
+
crc32c: crc32c(stored)
|
|
1578
|
+
});
|
|
1579
|
+
this.offset += stored.length;
|
|
1580
|
+
this.totalCardinality += cardinality;
|
|
1581
|
+
this.lastKey = chunkKey;
|
|
1582
|
+
}
|
|
1583
|
+
/** Seal one chunk payload into the on-disk frame `nonce ‖ ciphertext ‖ tag`, bound to its chunkKey. */
|
|
1584
|
+
sealChunk(chunkKey, payload) {
|
|
1585
|
+
const crypto = this.options.crypto;
|
|
1586
|
+
const sealed = crypto.aead.seal(payload, crypto.aadFor(chunkKey));
|
|
1587
|
+
const out = new Uint8Array(sealed.nonce.length + sealed.ciphertext.length + sealed.tag.length);
|
|
1588
|
+
out.set(sealed.nonce, 0);
|
|
1589
|
+
out.set(sealed.ciphertext, sealed.nonce.length);
|
|
1590
|
+
out.set(sealed.tag, sealed.nonce.length + sealed.ciphertext.length);
|
|
1591
|
+
return out;
|
|
1592
|
+
}
|
|
1593
|
+
/** Write the index region + the fixed footer. After this the object is complete and immutable. */
|
|
1594
|
+
async finish() {
|
|
1595
|
+
if (this.finished) throw new ValidationError("CrbmWriter already finished");
|
|
1596
|
+
await this.ensurePreamble();
|
|
1597
|
+
const indexPlain = this.encodeIndex();
|
|
1598
|
+
let indexRegion = indexPlain;
|
|
1599
|
+
let indexNonce;
|
|
1600
|
+
let indexTag;
|
|
1601
|
+
if (this.options.crypto !== void 0) {
|
|
1602
|
+
const sealed = this.options.crypto.aead.seal(indexPlain, this.options.crypto.aadFor("index"));
|
|
1603
|
+
indexRegion = sealed.ciphertext;
|
|
1604
|
+
indexNonce = sealed.nonce;
|
|
1605
|
+
indexTag = sealed.tag;
|
|
1606
|
+
}
|
|
1607
|
+
const indexOffset = this.offset;
|
|
1608
|
+
await this.sink.write(indexRegion);
|
|
1609
|
+
this.offset += indexRegion.length;
|
|
1610
|
+
const footer = this.buildFooter(indexOffset, indexRegion, indexNonce, indexTag);
|
|
1611
|
+
await this.sink.write(footer);
|
|
1612
|
+
this.finished = true;
|
|
1613
|
+
}
|
|
1614
|
+
async ensurePreamble() {
|
|
1615
|
+
if (this.preambleWritten) return;
|
|
1616
|
+
const preamble = new Uint8Array(PREAMBLE_BYTES);
|
|
1617
|
+
preamble.set(MAGIC, 0);
|
|
1618
|
+
preamble[4] = VERSION_MAJOR;
|
|
1619
|
+
preamble[5] = VERSION_MINOR;
|
|
1620
|
+
await this.sink.write(preamble);
|
|
1621
|
+
this.preambleWritten = true;
|
|
1622
|
+
}
|
|
1623
|
+
encodeIndex() {
|
|
1624
|
+
const out = [];
|
|
1625
|
+
let prevKey = 0;
|
|
1626
|
+
let prevEnd = PAYLOAD_START;
|
|
1627
|
+
for (const e of this.entries) {
|
|
1628
|
+
writeVarint(out, e.chunkKey - prevKey);
|
|
1629
|
+
writeVarint(out, e.offset - prevEnd);
|
|
1630
|
+
writeVarint(out, e.length);
|
|
1631
|
+
writeVarint(out, e.cardinality);
|
|
1632
|
+
out.push(
|
|
1633
|
+
e.crc32c & 255,
|
|
1634
|
+
e.crc32c >>> 8 & 255,
|
|
1635
|
+
e.crc32c >>> 16 & 255,
|
|
1636
|
+
e.crc32c >>> 24 & 255
|
|
1637
|
+
);
|
|
1638
|
+
prevKey = e.chunkKey;
|
|
1639
|
+
prevEnd = e.offset + e.length;
|
|
1640
|
+
}
|
|
1641
|
+
return Uint8Array.from(out);
|
|
1642
|
+
}
|
|
1643
|
+
buildFooter(indexOffset, indexRegion, indexNonce, indexTag) {
|
|
1644
|
+
const encrypted = this.options.crypto !== void 0;
|
|
1645
|
+
const footer = new Uint8Array(FOOTER_BYTES);
|
|
1646
|
+
const view = new DataView(footer.buffer);
|
|
1647
|
+
view.setBigUint64(FOOTER.indexOffset, BigInt(indexOffset), true);
|
|
1648
|
+
view.setBigUint64(FOOTER.indexLength, BigInt(indexRegion.length), true);
|
|
1649
|
+
view.setUint32(FOOTER.indexCrc32c, crc32c(indexRegion), true);
|
|
1650
|
+
view.setUint32(FOOTER.flags, FLAG_LITTLE_ENDIAN | (encrypted ? FLAG_ENCRYPTED : 0), true);
|
|
1651
|
+
view.setUint16(
|
|
1652
|
+
FOOTER.roaringSerializationId,
|
|
1653
|
+
this.options.roaringSerializationId ?? ROARING_PORTABLE_ID,
|
|
1654
|
+
true
|
|
1655
|
+
);
|
|
1656
|
+
footer[FOOTER.elementWidth] = this.options.elementWidth ?? ELEMENT_WIDTH_32;
|
|
1657
|
+
footer[FOOTER.containerCodec] = CONTAINER_CODEC_NONE;
|
|
1658
|
+
footer[FOOTER.versionMajor] = VERSION_MAJOR;
|
|
1659
|
+
footer[FOOTER.versionMinor] = VERSION_MINOR;
|
|
1660
|
+
if (indexNonce !== void 0) footer.set(indexNonce, FOOTER.indexNonce);
|
|
1661
|
+
if (indexTag !== void 0) footer.set(indexTag, FOOTER.indexTag);
|
|
1662
|
+
view.setBigUint64(FOOTER.generation, BigInt(this.options.generation), true);
|
|
1663
|
+
view.setUint32(FOOTER.chunkCount, encrypted ? 0 : this.entries.length, true);
|
|
1664
|
+
view.setBigUint64(FOOTER.totalCardinality, BigInt(encrypted ? 0 : this.totalCardinality), true);
|
|
1665
|
+
view.setUint32(FOOTER.footerCrc32c, crc32c(footer.subarray(0, FOOTER_CRC_COVERAGE)), true);
|
|
1666
|
+
footer.set(MAGIC, FOOTER.endMagic);
|
|
1667
|
+
return footer;
|
|
1668
|
+
}
|
|
1669
|
+
};
|
|
1670
|
+
|
|
1671
|
+
// src/core/crbm-cold-source.ts
|
|
1672
|
+
function coldBlobReader(driver, key) {
|
|
1673
|
+
return {
|
|
1674
|
+
getRange(offset, length) {
|
|
1675
|
+
return driver.getRange(key, offset, length);
|
|
1676
|
+
},
|
|
1677
|
+
getTail(maxBytes) {
|
|
1678
|
+
return driver.getTail(key, maxBytes);
|
|
1679
|
+
}
|
|
1680
|
+
};
|
|
1681
|
+
}
|
|
1682
|
+
var DEFAULT_CURRENT_GEN_TTL_MS = 2e3;
|
|
1683
|
+
var DEFAULT_MAX_OPEN_SEGMENTS = 1024;
|
|
1684
|
+
var DEFAULT_MAX_OPEN_INDEX_BYTES = 64 * 1024 * 1024;
|
|
1685
|
+
var CrbmColdChunkSource = class {
|
|
1686
|
+
constructor(driver, options = {}) {
|
|
1687
|
+
this.driver = driver;
|
|
1688
|
+
if (!driver.capabilities().rangeRead) {
|
|
1689
|
+
throw new CapabilityError("Cold driver must support range reads (rangeRead)");
|
|
1690
|
+
}
|
|
1691
|
+
const {
|
|
1692
|
+
registry,
|
|
1693
|
+
keystore,
|
|
1694
|
+
requireEncryption,
|
|
1695
|
+
clock,
|
|
1696
|
+
currentGenTtlMs,
|
|
1697
|
+
maxOpenSegments,
|
|
1698
|
+
maxOpenIndexBytes,
|
|
1699
|
+
...readerOptions
|
|
1700
|
+
} = options;
|
|
1701
|
+
if (keystore !== void 0 && registry === void 0) {
|
|
1702
|
+
throw new CapabilityError(
|
|
1703
|
+
"a keystore needs a registry (that is where wrapped DEKs are stored)"
|
|
1704
|
+
);
|
|
1705
|
+
}
|
|
1706
|
+
if (requireEncryption === true && registry === void 0) {
|
|
1707
|
+
throw new CapabilityError("requireEncryption needs a registry");
|
|
1708
|
+
}
|
|
1709
|
+
this.registry = registry;
|
|
1710
|
+
this.keystore = keystore;
|
|
1711
|
+
this.requireEncryption = requireEncryption ?? false;
|
|
1712
|
+
this.readerOptions = readerOptions;
|
|
1713
|
+
this.clock = clock;
|
|
1714
|
+
this.currentGenTtlMs = currentGenTtlMs ?? DEFAULT_CURRENT_GEN_TTL_MS;
|
|
1715
|
+
this.snapshots = new BoundedLru({
|
|
1716
|
+
maxEntries: maxOpenSegments ?? DEFAULT_MAX_OPEN_SEGMENTS,
|
|
1717
|
+
maxBytes: maxOpenIndexBytes ?? DEFAULT_MAX_OPEN_INDEX_BYTES,
|
|
1718
|
+
clock: clock ?? { now: () => 0 }
|
|
1719
|
+
});
|
|
1720
|
+
}
|
|
1721
|
+
driver;
|
|
1722
|
+
/**
|
|
1723
|
+
* One resolved reader per segment, re-resolved on a short TTL ({@link CrbmColdChunkSourceOptions.currentGenTtlMs},
|
|
1724
|
+
* needs a clock). Within the TTL a segment's Cold bytes are treated as an immutable snapshot; when the TTL
|
|
1725
|
+
* elapses the next read cheaply re-resolves `currentGen` and, only if it advanced (a compaction committed),
|
|
1726
|
+
* opens the new generation — so a long-lived source observes new generations within the TTL rather than
|
|
1727
|
+
* pinning one forever (gap #4). The engine pairs this with a **generation-keyed** HOT cache so a bump never
|
|
1728
|
+
* serves a stale decoded chunk. Without a clock or a registry the source pins the first generation for its
|
|
1729
|
+
* lifetime (pre-Phase-B behaviour). A segment with no generation yet is not memoized, so it's re-checked until
|
|
1730
|
+
* one exists. **Bounded** by a {@link BoundedLru} ({@link CrbmColdChunkSourceOptions.maxOpenSegments}, default
|
|
1731
|
+
* 1024): past the ceiling the least-recently-used segment's reader (and its parsed index) is evicted — the
|
|
1732
|
+
* steady-state memory bound (gap #1); re-opening an evicted segment is one cheap tail GET.
|
|
1733
|
+
*/
|
|
1734
|
+
snapshots;
|
|
1735
|
+
registry;
|
|
1736
|
+
keystore;
|
|
1737
|
+
requireEncryption;
|
|
1738
|
+
readerOptions;
|
|
1739
|
+
clock;
|
|
1740
|
+
currentGenTtlMs;
|
|
1741
|
+
/** The current resolved reader for a segment, refreshing on the TTL (gap #4). Cheap within the TTL window. */
|
|
1742
|
+
resolvedReader(ref) {
|
|
1743
|
+
const key = segmentKey(ref);
|
|
1744
|
+
const existing = this.snapshots.get(key);
|
|
1745
|
+
if (existing === void 0) return this.install(key, this.resolveLatest(ref)).reader;
|
|
1746
|
+
if (!this.expired(existing.installedAtMs)) return existing.reader;
|
|
1747
|
+
return this.install(key, this.computeRefreshedReader(ref, existing)).reader;
|
|
1748
|
+
}
|
|
1749
|
+
/** TTL elapsed: cheaply re-resolve `currentGen`; reopen only if it actually changed, else reuse the prior reader. */
|
|
1750
|
+
async computeRefreshedReader(ref, existing) {
|
|
1751
|
+
const current = await existing.reader.catch(() => null);
|
|
1752
|
+
let target;
|
|
1753
|
+
try {
|
|
1754
|
+
target = await this.resolveTarget(ref);
|
|
1755
|
+
} catch {
|
|
1756
|
+
target = void 0;
|
|
1757
|
+
}
|
|
1758
|
+
if (target === void 0) return current ?? this.resolveLatest(ref);
|
|
1759
|
+
if (target === null) return null;
|
|
1760
|
+
if (current !== null && target.generation === current.generation) return current;
|
|
1761
|
+
return this.openForTarget(ref, target);
|
|
1762
|
+
}
|
|
1763
|
+
/**
|
|
1764
|
+
* Memoize a reader-promise as the segment's snapshot at `now()`, forgetting it later if it resolves to
|
|
1765
|
+
* null / throws (identity-guarded, so a stale cleanup never clobbers a fresher snapshot). Installing the
|
|
1766
|
+
* (pending) promise **synchronously** is what lets concurrent callers coalesce onto one in-flight resolve.
|
|
1767
|
+
*/
|
|
1768
|
+
install(key, reader) {
|
|
1769
|
+
const snap = { reader, installedAtMs: this.now() };
|
|
1770
|
+
this.snapshots.set(key, snap);
|
|
1771
|
+
const forgetIfStale = () => {
|
|
1772
|
+
if (this.snapshots.get(key) === snap) this.snapshots.delete(key);
|
|
1773
|
+
};
|
|
1774
|
+
reader.then((r) => {
|
|
1775
|
+
if (r === null) {
|
|
1776
|
+
forgetIfStale();
|
|
1777
|
+
return;
|
|
1778
|
+
}
|
|
1779
|
+
if (this.snapshots.peek(key) === snap) this.snapshots.setWeight(key, r.retainedIndexBytes);
|
|
1780
|
+
}, forgetIfStale);
|
|
1781
|
+
return snap;
|
|
1782
|
+
}
|
|
1783
|
+
now() {
|
|
1784
|
+
return this.clock ? this.clock.now() : 0;
|
|
1785
|
+
}
|
|
1786
|
+
expired(installedAtMs) {
|
|
1787
|
+
return this.clock !== void 0 && this.registry !== void 0 && this.currentGenTtlMs > 0 && this.clock.now() - installedAtMs >= this.currentGenTtlMs;
|
|
1788
|
+
}
|
|
1789
|
+
/**
|
|
1790
|
+
* The segment's current generation number (gap #4) — the engine keys its HOT chunk cache by this so a
|
|
1791
|
+
* compaction bump is observed instead of serving a stale decoded chunk. Served from the (TTL-refreshed)
|
|
1792
|
+
* snapshot, so no extra backend read within the TTL window. `null` if the segment has no committed generation.
|
|
1793
|
+
*/
|
|
1794
|
+
async currentGeneration(ref) {
|
|
1795
|
+
const reader = await this.resolvedReader(ref);
|
|
1796
|
+
return reader?.generation ?? null;
|
|
1797
|
+
}
|
|
1798
|
+
async resolveLatest(ref) {
|
|
1799
|
+
const target = await this.resolveTarget(ref);
|
|
1800
|
+
if (target === null) return null;
|
|
1801
|
+
return this.openForTarget(ref, target);
|
|
1802
|
+
}
|
|
1803
|
+
/** Open a {@link CrbmReader} for an already-resolved generation target (decrypting if the segment is encrypted). */
|
|
1804
|
+
async openForTarget(ref, target) {
|
|
1805
|
+
const genKey = {
|
|
1806
|
+
namespace: ref.namespace,
|
|
1807
|
+
segment: ref.segment,
|
|
1808
|
+
generation: target.generation
|
|
1809
|
+
};
|
|
1810
|
+
const crypto = await this.cryptoForRead(ref, target.generation, target.wrappedDeks);
|
|
1811
|
+
return CrbmReader.open(coldBlobReader(this.driver, genKey), { ...this.readerOptions, crypto });
|
|
1812
|
+
}
|
|
1813
|
+
/**
|
|
1814
|
+
* The current generation + its DEK wrappings: the registry's authoritative record (one strong read), or a
|
|
1815
|
+
* `list` scan for the max generation when there's no registry (a registry-less source can only read cleartext
|
|
1816
|
+
* — there's nowhere a wrapped DEK could live).
|
|
1817
|
+
*/
|
|
1818
|
+
async resolveTarget(ref) {
|
|
1819
|
+
if (this.registry !== void 0) {
|
|
1820
|
+
const record = await this.registry.get(ref);
|
|
1821
|
+
if (record === null) return null;
|
|
1822
|
+
if (record.status === "destroyed") return null;
|
|
1823
|
+
return { generation: record.currentGen, wrappedDeks: record.wrappedDeks };
|
|
1824
|
+
}
|
|
1825
|
+
let maxGen = -1;
|
|
1826
|
+
for await (const key of this.driver.list(ref)) {
|
|
1827
|
+
if (key.generation > maxGen) maxGen = key.generation;
|
|
1828
|
+
}
|
|
1829
|
+
return maxGen < 0 ? null : { generation: maxGen };
|
|
1830
|
+
}
|
|
1831
|
+
/** Build the per-generation decryption context for an encrypted segment (undefined for cleartext). */
|
|
1832
|
+
async cryptoForRead(ref, generation, wrappedDeks) {
|
|
1833
|
+
if (wrappedDeks === void 0 || wrappedDeks.length === 0) {
|
|
1834
|
+
if (this.requireEncryption) {
|
|
1835
|
+
throw new KeyUnavailableError(
|
|
1836
|
+
`requireEncryption: segment "${ref.segment}" is cleartext but encryption is required`
|
|
1837
|
+
);
|
|
1838
|
+
}
|
|
1839
|
+
return void 0;
|
|
1840
|
+
}
|
|
1841
|
+
if (this.keystore === void 0) {
|
|
1842
|
+
throw new KeyUnavailableError(
|
|
1843
|
+
`segment "${ref.segment}" is encrypted but this CrbmColdChunkSource has no keystore`
|
|
1844
|
+
);
|
|
1845
|
+
}
|
|
1846
|
+
const aead = await this.keystore.openDek(wrappedDeks);
|
|
1847
|
+
return { aead, aadFor: (scope) => aadFor(ref, generation, scope) };
|
|
1848
|
+
}
|
|
1849
|
+
async getChunk(ref) {
|
|
1850
|
+
validateChunkRef(ref);
|
|
1851
|
+
return this.withFreshSnapshot(ref, (reader) => reader.getChunk(ref.chunkKey), null);
|
|
1852
|
+
}
|
|
1853
|
+
async listChunkKeys(ref) {
|
|
1854
|
+
validateSegmentRef(ref);
|
|
1855
|
+
return this.withFreshSnapshot(ref, (reader) => reader.chunkKeys(), []);
|
|
1856
|
+
}
|
|
1857
|
+
async sizeOf(ref) {
|
|
1858
|
+
validateSegmentRef(ref);
|
|
1859
|
+
return this.withFreshSnapshot(
|
|
1860
|
+
ref,
|
|
1861
|
+
(reader) => ({ sizeBytes: reader.sizeBytes }),
|
|
1862
|
+
null
|
|
1863
|
+
);
|
|
1864
|
+
}
|
|
1865
|
+
async cardinalities(ref) {
|
|
1866
|
+
validateSegmentRef(ref);
|
|
1867
|
+
return this.withFreshSnapshot(
|
|
1868
|
+
ref,
|
|
1869
|
+
(reader) => reader.cardinalities(),
|
|
1870
|
+
null
|
|
1871
|
+
);
|
|
1872
|
+
}
|
|
1873
|
+
/**
|
|
1874
|
+
* Run `read` against the pinned snapshot, healing the one torn-read window compaction's GC can open: if the
|
|
1875
|
+
* generation we pinned was superseded *and* swept (the grace window elapsed) mid-read, the Cold driver throws
|
|
1876
|
+
* {@link NotFoundError}. Rather than surface that as a query failure (**I5**), we drop the stale snapshot,
|
|
1877
|
+
* re-resolve `currentGen`, and retry once — the read then serves the newer (committed, immutable) generation,
|
|
1878
|
+
* a monotonic move forward. A *second* NotFound is pathological (GC outrunning resolution) and propagates
|
|
1879
|
+
* rather than fabricate an absent answer — never return a wrong result. `ifGone` is returned only when the
|
|
1880
|
+
* segment legitimately has no committed generation at all (cold is empty). The single retry is bounded.
|
|
1881
|
+
*/
|
|
1882
|
+
async withFreshSnapshot(ref, read, ifGone) {
|
|
1883
|
+
const key = segmentKey(ref);
|
|
1884
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
1885
|
+
const pending = this.resolvedReader(ref);
|
|
1886
|
+
const reader = await pending;
|
|
1887
|
+
if (reader === null) return ifGone;
|
|
1888
|
+
try {
|
|
1889
|
+
return await read(reader);
|
|
1890
|
+
} catch (err) {
|
|
1891
|
+
if (!isNotFoundError(err) || attempt === 1) throw err;
|
|
1892
|
+
const cur = this.snapshots.get(key);
|
|
1893
|
+
if (cur !== void 0 && cur.reader === pending) this.snapshots.delete(key);
|
|
1894
|
+
}
|
|
1895
|
+
}
|
|
1896
|
+
return ifGone;
|
|
1897
|
+
}
|
|
1898
|
+
};
|
|
1899
|
+
function writeCrbmGeneration(driver, key, chunks, options = {}) {
|
|
1900
|
+
const sorted = [...chunks].sort((a, b) => a.chunkKey - b.chunkKey);
|
|
1901
|
+
return driver.putImmutable(key, async (sink) => {
|
|
1902
|
+
const writer = new CrbmWriter(sink, { generation: key.generation, crypto: options.crypto });
|
|
1903
|
+
for (const { chunkKey, bitmap } of sorted) {
|
|
1904
|
+
if (bitmap.isEmpty) continue;
|
|
1905
|
+
await writer.addChunk(chunkKey, bitmap.serialize(), bitmap.size);
|
|
1906
|
+
}
|
|
1907
|
+
await writer.finish();
|
|
1908
|
+
});
|
|
1909
|
+
}
|
|
1910
|
+
async function writeCrbmGenerationStream(driver, key, chunks, options = {}) {
|
|
1911
|
+
const chunkKeys = [];
|
|
1912
|
+
let cardinality = 0;
|
|
1913
|
+
const { size, sha256 } = await driver.putImmutable(key, async (sink) => {
|
|
1914
|
+
const writer = new CrbmWriter(sink, { generation: key.generation, crypto: options.crypto });
|
|
1915
|
+
for await (const { chunkKey, bitmap } of chunks) {
|
|
1916
|
+
if (bitmap.isEmpty) continue;
|
|
1917
|
+
await writer.addChunk(chunkKey, bitmap.serialize(), bitmap.size);
|
|
1918
|
+
chunkKeys.push(chunkKey);
|
|
1919
|
+
cardinality += bitmap.size;
|
|
1920
|
+
}
|
|
1921
|
+
await writer.finish();
|
|
1922
|
+
});
|
|
1923
|
+
return { size, sha256, chunkKeys, cardinality };
|
|
1924
|
+
}
|
|
1925
|
+
async function publishGeneration(registry, key, options = {}) {
|
|
1926
|
+
for (let attempt = 0; attempt < 5; attempt++) {
|
|
1927
|
+
const record = await registry.get(key);
|
|
1928
|
+
try {
|
|
1929
|
+
if (record === null) {
|
|
1930
|
+
await registry.create(key, {
|
|
1931
|
+
currentGen: key.generation,
|
|
1932
|
+
wrappedDeks: options.wrappedDeks
|
|
1933
|
+
});
|
|
1934
|
+
} else if (record.currentGen > key.generation) {
|
|
1935
|
+
return false;
|
|
1936
|
+
} else if (record.currentGen === key.generation) {
|
|
1937
|
+
return true;
|
|
1938
|
+
} else {
|
|
1939
|
+
await registry.compareAndSwap(key, record.token, { currentGen: key.generation });
|
|
1940
|
+
}
|
|
1941
|
+
return true;
|
|
1942
|
+
} catch (err) {
|
|
1943
|
+
if (isWriteConflictError(err)) continue;
|
|
1944
|
+
throw err;
|
|
1945
|
+
}
|
|
1946
|
+
}
|
|
1947
|
+
const final = await registry.get(key);
|
|
1948
|
+
if (final !== null && final.currentGen >= key.generation)
|
|
1949
|
+
return final.currentGen === key.generation;
|
|
1950
|
+
throw new WriteConflictError(
|
|
1951
|
+
`publishGeneration: contention setting currentGen for ${key.segment}`
|
|
1952
|
+
);
|
|
1953
|
+
}
|
|
1954
|
+
async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
|
|
1955
|
+
if (options.keystore === void 0 && options.requireEncryption === true) {
|
|
1956
|
+
throw new ValidationError("requireEncryption: bulk-load needs a keystore to write encrypted");
|
|
1957
|
+
}
|
|
1958
|
+
const codec = requireCodec(options.codec, "bulkLoadCrbmGeneration");
|
|
1959
|
+
const byChunk = /* @__PURE__ */ new Map();
|
|
1960
|
+
for await (const id of toAsyncIterable(ids)) {
|
|
1961
|
+
const { chunkKey, remainder } = splitId(id);
|
|
1962
|
+
let bitmap = byChunk.get(chunkKey);
|
|
1963
|
+
if (bitmap === void 0) {
|
|
1964
|
+
bitmap = codec.empty();
|
|
1965
|
+
byChunk.set(chunkKey, bitmap);
|
|
1966
|
+
}
|
|
1967
|
+
bitmap.add(remainder);
|
|
1968
|
+
}
|
|
1969
|
+
let cardinality = 0;
|
|
1970
|
+
const chunks = [];
|
|
1971
|
+
for (const [chunkKey, bitmap] of byChunk) {
|
|
1972
|
+
cardinality += bitmap.size;
|
|
1973
|
+
chunks.push({ chunkKey, bitmap });
|
|
1974
|
+
}
|
|
1975
|
+
if (options.keystore !== void 0 && options.registry === void 0) {
|
|
1976
|
+
throw new ValidationError("bulk-load encryption requires a registry to store the wrapped DEK");
|
|
1977
|
+
}
|
|
1978
|
+
const existing = options.registry !== void 0 ? await options.registry.get(key) : null;
|
|
1979
|
+
if (existing?.status === "destroyed") {
|
|
1980
|
+
throw new ValidationError(
|
|
1981
|
+
`segment "${key.segment}" is destroyed (crypto-shredded) \u2014 refusing to write; use a new segment`
|
|
1982
|
+
);
|
|
1983
|
+
}
|
|
1984
|
+
let crypto;
|
|
1985
|
+
let newWrapped;
|
|
1986
|
+
if (options.keystore !== void 0) {
|
|
1987
|
+
if (existing?.wrappedDeks !== void 0 && existing.wrappedDeks.length > 0) {
|
|
1988
|
+
const aead = await options.keystore.openDek(existing.wrappedDeks);
|
|
1989
|
+
crypto = { aead, aadFor: (scope) => aadFor(key, key.generation, scope) };
|
|
1990
|
+
} else {
|
|
1991
|
+
const minted = await options.keystore.createDek();
|
|
1992
|
+
newWrapped = minted.wrapped;
|
|
1993
|
+
crypto = { aead: minted.aead, aadFor: (scope) => aadFor(key, key.generation, scope) };
|
|
1994
|
+
}
|
|
1995
|
+
}
|
|
1996
|
+
const { size, sha256 } = await writeCrbmGeneration(driver, key, chunks, { crypto });
|
|
1997
|
+
if (options.registry !== void 0) {
|
|
1998
|
+
const becameCurrent = await publishGeneration(options.registry, key, {
|
|
1999
|
+
wrappedDeks: newWrapped
|
|
2000
|
+
});
|
|
2001
|
+
if (becameCurrent) {
|
|
2002
|
+
safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
|
|
2003
|
+
kind: "segment.publish",
|
|
2004
|
+
namespace: key.namespace,
|
|
2005
|
+
segment: key.segment,
|
|
2006
|
+
generation: key.generation
|
|
2007
|
+
});
|
|
2008
|
+
}
|
|
2009
|
+
}
|
|
2010
|
+
return { size, sha256, chunkCount: chunks.length, cardinality };
|
|
2011
|
+
}
|
|
2012
|
+
async function* toAsyncIterable(src) {
|
|
2013
|
+
if (Symbol.asyncIterator in src) {
|
|
2014
|
+
yield* src;
|
|
2015
|
+
} else {
|
|
2016
|
+
yield* src;
|
|
2017
|
+
}
|
|
2018
|
+
}
|
|
2019
|
+
|
|
2020
|
+
// src/core/compaction.ts
|
|
2021
|
+
var DEFAULT_MAX_BITMAP_BYTES2 = 1 << 20;
|
|
2022
|
+
var DEFAULT_LEASE_MS = 6e4;
|
|
2023
|
+
var DEFAULT_QUARANTINE_THRESHOLD = 5;
|
|
2024
|
+
var DEFAULT_QUARANTINE_COOLDOWN_MS = 5 * 6e4;
|
|
2025
|
+
function validateCompactionOptions(options) {
|
|
2026
|
+
if (typeof options.owner !== "string" || options.owner.length === 0) {
|
|
2027
|
+
throw new ValidationError(
|
|
2028
|
+
"compaction: options.owner must be a non-empty string (the lease owner id)"
|
|
2029
|
+
);
|
|
2030
|
+
}
|
|
2031
|
+
if (options.leaseMs !== void 0 && (!Number.isFinite(options.leaseMs) || options.leaseMs <= 0)) {
|
|
2032
|
+
throw new ValidationError(
|
|
2033
|
+
`compaction: options.leaseMs must be a positive finite number; got ${options.leaseMs}`
|
|
2034
|
+
);
|
|
2035
|
+
}
|
|
2036
|
+
}
|
|
2037
|
+
async function compactSegment(ref, deps, options) {
|
|
2038
|
+
validateCompactionOptions(options);
|
|
2039
|
+
const startedAt = deps.clock.now();
|
|
2040
|
+
const emit = (r) => {
|
|
2041
|
+
if (deps.metrics === void 0) return;
|
|
2042
|
+
try {
|
|
2043
|
+
deps.metrics.onEvent({
|
|
2044
|
+
kind: "compaction",
|
|
2045
|
+
namespace: ref.namespace,
|
|
2046
|
+
segment: ref.segment,
|
|
2047
|
+
compacted: r.compacted,
|
|
2048
|
+
reason: r.reason,
|
|
2049
|
+
dirtyChunks: r.dirtyChunks,
|
|
2050
|
+
purged: r.purged,
|
|
2051
|
+
ms: Math.max(0, deps.clock.now() - startedAt)
|
|
2052
|
+
});
|
|
2053
|
+
} catch {
|
|
2054
|
+
}
|
|
2055
|
+
};
|
|
2056
|
+
try {
|
|
2057
|
+
const result = await compactSegmentInner(ref, deps, options);
|
|
2058
|
+
emit(result);
|
|
2059
|
+
return result;
|
|
2060
|
+
} catch (err) {
|
|
2061
|
+
emit({ compacted: false, reason: "error", dirtyChunks: 0, purged: 0 });
|
|
2062
|
+
throw err;
|
|
2063
|
+
}
|
|
2064
|
+
}
|
|
2065
|
+
async function compactSegmentInner(ref, deps, options) {
|
|
2066
|
+
const now = () => deps.clock.now();
|
|
2067
|
+
const maxBytes = deps.maxBitmapBytes ?? DEFAULT_MAX_BITMAP_BYTES2;
|
|
2068
|
+
const leaseMs = options.leaseMs ?? DEFAULT_LEASE_MS;
|
|
2069
|
+
const base = { segment: ref.segment, namespace: ref.namespace };
|
|
2070
|
+
const audit = safeAudit(options.audit ?? NOOP_AUDIT);
|
|
2071
|
+
const auditCompacted = (generation) => audit.onEvent({
|
|
2072
|
+
kind: "segment.compact",
|
|
2073
|
+
namespace: ref.namespace,
|
|
2074
|
+
segment: ref.segment,
|
|
2075
|
+
generation
|
|
2076
|
+
});
|
|
2077
|
+
const cryptoAt = (aead, generation) => aead === void 0 ? void 0 : { aead, aadFor: (scope) => aadFor(ref, generation, scope) };
|
|
2078
|
+
const record = await deps.registry.get(ref);
|
|
2079
|
+
if (record !== null && record.status === "destroyed") {
|
|
2080
|
+
return {
|
|
2081
|
+
...base,
|
|
2082
|
+
compacted: false,
|
|
2083
|
+
reason: "destroyed",
|
|
2084
|
+
fromGen: record.currentGen,
|
|
2085
|
+
dirtyChunks: 0,
|
|
2086
|
+
purged: 0,
|
|
2087
|
+
survived: 0
|
|
2088
|
+
};
|
|
2089
|
+
}
|
|
2090
|
+
if (record === null) {
|
|
2091
|
+
const pinned = await collectDirty(
|
|
2092
|
+
deps.warm,
|
|
2093
|
+
ref,
|
|
2094
|
+
maxBytes,
|
|
2095
|
+
requireCodec(deps.codec, "compactSegment")
|
|
2096
|
+
);
|
|
2097
|
+
if (pinned.length === 0)
|
|
2098
|
+
return {
|
|
2099
|
+
...base,
|
|
2100
|
+
compacted: false,
|
|
2101
|
+
reason: "clean",
|
|
2102
|
+
fromGen: null,
|
|
2103
|
+
dirtyChunks: 0,
|
|
2104
|
+
purged: 0,
|
|
2105
|
+
survived: 0
|
|
2106
|
+
};
|
|
2107
|
+
let aead;
|
|
2108
|
+
let wrappedDeks;
|
|
2109
|
+
if (deps.keystore !== void 0) {
|
|
2110
|
+
const minted = await deps.keystore.createDek();
|
|
2111
|
+
aead = minted.aead;
|
|
2112
|
+
wrappedDeks = minted.wrapped;
|
|
2113
|
+
} else if (deps.requireEncryption === true) {
|
|
2114
|
+
throw new ValidationError(
|
|
2115
|
+
`requireEncryption: segment "${ref.segment}" needs a keystore to bootstrap encrypted`
|
|
2116
|
+
);
|
|
2117
|
+
}
|
|
2118
|
+
const gen0 = { namespace: ref.namespace, segment: ref.segment, generation: 0 };
|
|
2119
|
+
const stream = mergeChunksStream(ref, null, pinned, deps, maxBytes, void 0);
|
|
2120
|
+
let wrote = true;
|
|
2121
|
+
let tally;
|
|
2122
|
+
try {
|
|
2123
|
+
tally = await writeCrbmGenerationStream(deps.cold, gen0, stream, {
|
|
2124
|
+
crypto: cryptoAt(aead, 0)
|
|
2125
|
+
});
|
|
2126
|
+
} catch (err) {
|
|
2127
|
+
if (!isWriteConflictError(err)) throw err;
|
|
2128
|
+
wrote = false;
|
|
2129
|
+
}
|
|
2130
|
+
if (wrote && tally !== void 0) {
|
|
2131
|
+
await verifyGeneration(deps.cold, gen0, tally, cryptoAt(aead, 0));
|
|
2132
|
+
}
|
|
2133
|
+
try {
|
|
2134
|
+
await deps.registry.create(ref, {
|
|
2135
|
+
currentGen: 0,
|
|
2136
|
+
dirtyChunkCount: 0,
|
|
2137
|
+
status: "active",
|
|
2138
|
+
wrappedDeks
|
|
2139
|
+
});
|
|
2140
|
+
} catch (err) {
|
|
2141
|
+
if (!isWriteConflictError(err)) throw err;
|
|
2142
|
+
}
|
|
2143
|
+
if (!wrote)
|
|
2144
|
+
return {
|
|
2145
|
+
...base,
|
|
2146
|
+
compacted: false,
|
|
2147
|
+
reason: "bootstrap-raced",
|
|
2148
|
+
fromGen: null,
|
|
2149
|
+
dirtyChunks: pinned.length,
|
|
2150
|
+
purged: 0,
|
|
2151
|
+
survived: pinned.length
|
|
2152
|
+
};
|
|
2153
|
+
auditCompacted(0);
|
|
2154
|
+
const { purged, survived } = await purgeAfterCommit(deps.warm, ref, pinned);
|
|
2155
|
+
return {
|
|
2156
|
+
...base,
|
|
2157
|
+
compacted: true,
|
|
2158
|
+
fromGen: null,
|
|
2159
|
+
toGen: 0,
|
|
2160
|
+
dirtyChunks: pinned.length,
|
|
2161
|
+
purged,
|
|
2162
|
+
survived
|
|
2163
|
+
};
|
|
2164
|
+
}
|
|
2165
|
+
const expired = record.leaseExpiresAt === void 0 || record.leaseExpiresAt <= now();
|
|
2166
|
+
const ownedByOther = record.status === "compacting" && !expired && record.leaseOwner !== options.owner;
|
|
2167
|
+
if (ownedByOther) {
|
|
2168
|
+
return {
|
|
2169
|
+
...base,
|
|
2170
|
+
compacted: false,
|
|
2171
|
+
reason: "leased-by-other",
|
|
2172
|
+
fromGen: record.currentGen,
|
|
2173
|
+
dirtyChunks: 0,
|
|
2174
|
+
purged: 0,
|
|
2175
|
+
survived: 0
|
|
2176
|
+
};
|
|
2177
|
+
}
|
|
2178
|
+
let leaseToken;
|
|
2179
|
+
try {
|
|
2180
|
+
const res = await deps.registry.compareAndSwap(ref, record.token, {
|
|
2181
|
+
status: "compacting",
|
|
2182
|
+
leaseOwner: options.owner,
|
|
2183
|
+
leaseExpiresAt: now() + leaseMs
|
|
2184
|
+
});
|
|
2185
|
+
leaseToken = res.token;
|
|
2186
|
+
} catch (err) {
|
|
2187
|
+
if (isWriteConflictError(err)) {
|
|
2188
|
+
return {
|
|
2189
|
+
...base,
|
|
2190
|
+
compacted: false,
|
|
2191
|
+
reason: "leased-by-other",
|
|
2192
|
+
fromGen: record.currentGen,
|
|
2193
|
+
dirtyChunks: 0,
|
|
2194
|
+
purged: 0,
|
|
2195
|
+
survived: 0
|
|
2196
|
+
};
|
|
2197
|
+
}
|
|
2198
|
+
throw err;
|
|
2199
|
+
}
|
|
2200
|
+
const g = record.currentGen;
|
|
2201
|
+
try {
|
|
2202
|
+
let aead;
|
|
2203
|
+
if (record.wrappedDeks !== void 0 && record.wrappedDeks.length > 0) {
|
|
2204
|
+
if (deps.keystore === void 0) {
|
|
2205
|
+
throw new KeyUnavailableError(
|
|
2206
|
+
`segment "${ref.segment}" is encrypted but compaction has no keystore`
|
|
2207
|
+
);
|
|
2208
|
+
}
|
|
2209
|
+
aead = await deps.keystore.openDek(record.wrappedDeks);
|
|
2210
|
+
} else if (deps.requireEncryption === true) {
|
|
2211
|
+
throw new ValidationError(`requireEncryption: segment "${ref.segment}" is cleartext`);
|
|
2212
|
+
}
|
|
2213
|
+
const live = await deps.registry.get(ref);
|
|
2214
|
+
if (live === null || live.currentGen !== g) {
|
|
2215
|
+
await releaseLease(deps, ref, leaseToken);
|
|
2216
|
+
return {
|
|
2217
|
+
...base,
|
|
2218
|
+
compacted: false,
|
|
2219
|
+
reason: "superseded",
|
|
2220
|
+
fromGen: g,
|
|
2221
|
+
dirtyChunks: 0,
|
|
2222
|
+
purged: 0,
|
|
2223
|
+
survived: 0
|
|
2224
|
+
};
|
|
2225
|
+
}
|
|
2226
|
+
await deleteGenerationsAbove(deps.cold, ref, g);
|
|
2227
|
+
const pinned = await collectDirty(
|
|
2228
|
+
deps.warm,
|
|
2229
|
+
ref,
|
|
2230
|
+
maxBytes,
|
|
2231
|
+
requireCodec(deps.codec, "compactSegment")
|
|
2232
|
+
);
|
|
2233
|
+
if (pinned.length === 0) {
|
|
2234
|
+
await releaseLease(deps, ref, leaseToken);
|
|
2235
|
+
return {
|
|
2236
|
+
...base,
|
|
2237
|
+
compacted: false,
|
|
2238
|
+
reason: "clean",
|
|
2239
|
+
fromGen: g,
|
|
2240
|
+
dirtyChunks: 0,
|
|
2241
|
+
purged: 0,
|
|
2242
|
+
survived: 0
|
|
2243
|
+
};
|
|
2244
|
+
}
|
|
2245
|
+
const newGen = g + 1;
|
|
2246
|
+
const stream = mergeChunksStream(ref, g, pinned, deps, maxBytes, cryptoAt(aead, g));
|
|
2247
|
+
await stageGeneration(
|
|
2248
|
+
deps.cold,
|
|
2249
|
+
{ namespace: ref.namespace, segment: ref.segment, generation: newGen },
|
|
2250
|
+
stream,
|
|
2251
|
+
cryptoAt(aead, newGen)
|
|
2252
|
+
);
|
|
2253
|
+
try {
|
|
2254
|
+
await deps.registry.compareAndSwap(ref, leaseToken, {
|
|
2255
|
+
currentGen: newGen,
|
|
2256
|
+
status: "active",
|
|
2257
|
+
leaseOwner: void 0,
|
|
2258
|
+
leaseExpiresAt: void 0,
|
|
2259
|
+
dirtyChunkCount: 0,
|
|
2260
|
+
lastCompactedAt: now(),
|
|
2261
|
+
// dead-man's-switch (gap #2): a stale value = a stuck segment
|
|
2262
|
+
consecutiveFailures: 0
|
|
2263
|
+
// a successful commit clears any prior poison-quarantine streak
|
|
2264
|
+
});
|
|
2265
|
+
} catch (err) {
|
|
2266
|
+
if (isWriteConflictError(err)) {
|
|
2267
|
+
return {
|
|
2268
|
+
...base,
|
|
2269
|
+
compacted: false,
|
|
2270
|
+
reason: "lease-lost",
|
|
2271
|
+
fromGen: g,
|
|
2272
|
+
dirtyChunks: pinned.length,
|
|
2273
|
+
purged: 0,
|
|
2274
|
+
survived: 0
|
|
2275
|
+
};
|
|
2276
|
+
}
|
|
2277
|
+
throw err;
|
|
2278
|
+
}
|
|
2279
|
+
auditCompacted(newGen);
|
|
2280
|
+
const { purged, survived } = await purgeAfterCommit(deps.warm, ref, pinned);
|
|
2281
|
+
return {
|
|
2282
|
+
...base,
|
|
2283
|
+
compacted: true,
|
|
2284
|
+
fromGen: g,
|
|
2285
|
+
toGen: newGen,
|
|
2286
|
+
dirtyChunks: pinned.length,
|
|
2287
|
+
purged,
|
|
2288
|
+
survived
|
|
2289
|
+
};
|
|
2290
|
+
} catch (err) {
|
|
2291
|
+
await releaseLease(deps, ref, leaseToken).catch(() => void 0);
|
|
2292
|
+
if (isWriteConflictError(err)) {
|
|
2293
|
+
return {
|
|
2294
|
+
...base,
|
|
2295
|
+
compacted: false,
|
|
2296
|
+
reason: "stage-conflict",
|
|
2297
|
+
fromGen: g,
|
|
2298
|
+
dirtyChunks: 0,
|
|
2299
|
+
purged: 0,
|
|
2300
|
+
survived: 0
|
|
2301
|
+
};
|
|
2302
|
+
}
|
|
2303
|
+
throw err;
|
|
2304
|
+
}
|
|
2305
|
+
}
|
|
2306
|
+
async function gcOrphanGenerations(ref, deps, options = {}) {
|
|
2307
|
+
const keep = Math.max(0, options.keep ?? 1);
|
|
2308
|
+
const record = await deps.registry.get(ref);
|
|
2309
|
+
if (record === null) return [];
|
|
2310
|
+
const current = record.currentGen;
|
|
2311
|
+
const gens = [];
|
|
2312
|
+
for await (const key of deps.cold.list(ref)) gens.push(key.generation);
|
|
2313
|
+
const below = gens.filter((g) => g < current).sort((a, b) => b - a);
|
|
2314
|
+
const toDelete = below.slice(keep);
|
|
2315
|
+
for (const generation of toDelete) {
|
|
2316
|
+
await deps.cold.delete({ namespace: ref.namespace, segment: ref.segment, generation });
|
|
2317
|
+
}
|
|
2318
|
+
return toDelete;
|
|
2319
|
+
}
|
|
2320
|
+
async function collectDirty(warm, ref, maxBytes, codec) {
|
|
2321
|
+
const out = [];
|
|
2322
|
+
for await (const row of warm.listChunks(ref)) {
|
|
2323
|
+
out.push({
|
|
2324
|
+
chunkKey: row.chunkKey,
|
|
2325
|
+
token: row.token,
|
|
2326
|
+
delta: decodeDelta(row.bytes, maxBytes, codec)
|
|
2327
|
+
});
|
|
2328
|
+
}
|
|
2329
|
+
return out;
|
|
2330
|
+
}
|
|
2331
|
+
async function* mergeChunksStream(ref, g, pinned, deps, maxBytes, readCrypto) {
|
|
2332
|
+
const codec = requireCodec(deps.codec, "compactSegment");
|
|
2333
|
+
const dirty = new Map(pinned.map((p) => [p.chunkKey, p.delta]));
|
|
2334
|
+
const reader = g === null ? null : await openReader(
|
|
2335
|
+
deps.cold,
|
|
2336
|
+
{ namespace: ref.namespace, segment: ref.segment, generation: g },
|
|
2337
|
+
readCrypto
|
|
2338
|
+
);
|
|
2339
|
+
const keys = new Set(dirty.keys());
|
|
2340
|
+
if (reader !== null) for (const k of reader.chunkKeys()) keys.add(k);
|
|
2341
|
+
for (const chunkKey of [...keys].sort((a, b) => a - b)) {
|
|
2342
|
+
const coldBytes = reader === null ? null : await reader.getChunk(chunkKey);
|
|
2343
|
+
const cold = coldBytes === null ? codec.empty() : codec.safeDeserialize(coldBytes, maxBytes);
|
|
2344
|
+
const merged = effective(cold, dirty.get(chunkKey) ?? emptyDelta(codec));
|
|
2345
|
+
if (!merged.isEmpty) yield { chunkKey, bitmap: merged };
|
|
2346
|
+
}
|
|
2347
|
+
}
|
|
2348
|
+
async function stageGeneration(cold, key, chunks, crypto) {
|
|
2349
|
+
const tally = await writeCrbmGenerationStream(cold, key, chunks, { crypto });
|
|
2350
|
+
await verifyGeneration(cold, key, tally, crypto);
|
|
2351
|
+
}
|
|
2352
|
+
async function verifyGeneration(cold, key, expected, crypto) {
|
|
2353
|
+
const expectedCount = expected.cardinality;
|
|
2354
|
+
const expectedKeys = [...expected.chunkKeys].sort((a, b) => a - b);
|
|
2355
|
+
const reader = await openReader(cold, key, crypto);
|
|
2356
|
+
const actualKeys = [...reader.chunkKeys()].sort((a, b) => a - b);
|
|
2357
|
+
const keysMatch = actualKeys.length === expectedKeys.length && actualKeys.every((k, i) => k === expectedKeys[i]);
|
|
2358
|
+
if (!keysMatch) {
|
|
2359
|
+
throw new WriteConflictError(
|
|
2360
|
+
`compaction verify failed for ${key.segment}.${key.generation}: chunk-key set mismatch (${actualKeys.length} vs ${expectedKeys.length})`
|
|
2361
|
+
);
|
|
2362
|
+
}
|
|
2363
|
+
if (reader.count() !== expectedCount) {
|
|
2364
|
+
throw new WriteConflictError(
|
|
2365
|
+
`compaction verify failed for ${key.segment}.${key.generation}: cardinality ${reader.count()} != ${expectedCount}`
|
|
2366
|
+
);
|
|
2367
|
+
}
|
|
2368
|
+
}
|
|
2369
|
+
async function purge(warm, ref, pinned) {
|
|
2370
|
+
let purged = 0;
|
|
2371
|
+
let survived = 0;
|
|
2372
|
+
for (const { chunkKey, token } of pinned) {
|
|
2373
|
+
const chunkRef = { namespace: ref.namespace, segment: ref.segment, chunkKey };
|
|
2374
|
+
try {
|
|
2375
|
+
await warm.deleteConditional(chunkRef, token);
|
|
2376
|
+
purged += 1;
|
|
2377
|
+
} catch (err) {
|
|
2378
|
+
if (!isWriteConflictError(err)) throw err;
|
|
2379
|
+
survived += 1;
|
|
2380
|
+
}
|
|
2381
|
+
}
|
|
2382
|
+
return { purged, survived };
|
|
2383
|
+
}
|
|
2384
|
+
async function purgeAfterCommit(warm, ref, pinned) {
|
|
2385
|
+
return purge(warm, ref, pinned).catch(() => ({ purged: 0, survived: pinned.length }));
|
|
2386
|
+
}
|
|
2387
|
+
async function deleteGenerationsAbove(cold, ref, g) {
|
|
2388
|
+
for await (const key of cold.list(ref)) {
|
|
2389
|
+
if (key.generation > g) await cold.delete(key);
|
|
2390
|
+
}
|
|
2391
|
+
}
|
|
2392
|
+
async function releaseLease(deps, ref, leaseToken) {
|
|
2393
|
+
try {
|
|
2394
|
+
await deps.registry.compareAndSwap(ref, leaseToken, {
|
|
2395
|
+
status: "active",
|
|
2396
|
+
leaseOwner: void 0,
|
|
2397
|
+
leaseExpiresAt: void 0
|
|
2398
|
+
});
|
|
2399
|
+
} catch (err) {
|
|
2400
|
+
if (!isWriteConflictError(err)) throw err;
|
|
2401
|
+
}
|
|
2402
|
+
}
|
|
2403
|
+
function openReader(cold, key, crypto) {
|
|
2404
|
+
const blob = {
|
|
2405
|
+
getRange: (offset, length) => cold.getRange(key, offset, length),
|
|
2406
|
+
getTail: (maxBytes) => cold.getTail(key, maxBytes)
|
|
2407
|
+
};
|
|
2408
|
+
return CrbmReader.open(blob, { crypto });
|
|
2409
|
+
}
|
|
2410
|
+
function validateShard(shard, totalShards) {
|
|
2411
|
+
if (!Number.isInteger(totalShards) || totalShards < 1) {
|
|
2412
|
+
throw new ValidationError(`totalShards must be a positive integer; got ${totalShards}`);
|
|
2413
|
+
}
|
|
2414
|
+
if (!Number.isInteger(shard) || shard < 0 || shard >= totalShards) {
|
|
2415
|
+
throw new ValidationError(`shard must be an integer in [0, ${totalShards}); got ${shard}`);
|
|
2416
|
+
}
|
|
2417
|
+
}
|
|
2418
|
+
function shardOf(key, totalShards) {
|
|
2419
|
+
let h = 2166136261;
|
|
2420
|
+
for (let i = 0; i < key.length; i++) {
|
|
2421
|
+
h ^= key.charCodeAt(i);
|
|
2422
|
+
h = Math.imul(h, 16777619);
|
|
2423
|
+
}
|
|
2424
|
+
return (h >>> 0) % totalShards;
|
|
2425
|
+
}
|
|
2426
|
+
async function findCompactable(deps, options = {}) {
|
|
2427
|
+
const threshold = Math.max(1, options.threshold ?? 1);
|
|
2428
|
+
const totalShards = options.totalShards ?? 1;
|
|
2429
|
+
const shard = options.shard ?? 0;
|
|
2430
|
+
validateShard(shard, totalShards);
|
|
2431
|
+
const quarantineThreshold = options.quarantineThreshold ?? DEFAULT_QUARANTINE_THRESHOLD;
|
|
2432
|
+
const cooldownMs = options.quarantineCooldownMs ?? DEFAULT_QUARANTINE_COOLDOWN_MS;
|
|
2433
|
+
const now = deps.clock.now();
|
|
2434
|
+
const known = /* @__PURE__ */ new Map();
|
|
2435
|
+
for await (const rec of deps.registry.list(options.namespace)) {
|
|
2436
|
+
known.set(segmentKey(rec), {
|
|
2437
|
+
ref: { namespace: rec.namespace, segment: rec.segment },
|
|
2438
|
+
currentGen: rec.currentGen,
|
|
2439
|
+
token: rec.token,
|
|
2440
|
+
dirtyHint: rec.dirtyChunkCount,
|
|
2441
|
+
consecutiveFailures: rec.consecutiveFailures ?? 0,
|
|
2442
|
+
updatedAt: rec.updatedAt,
|
|
2443
|
+
lastCompactedAt: rec.lastCompactedAt
|
|
2444
|
+
});
|
|
2445
|
+
}
|
|
2446
|
+
for (const ref of options.candidates ?? []) {
|
|
2447
|
+
const k = segmentKey(ref);
|
|
2448
|
+
if (!known.has(k)) {
|
|
2449
|
+
known.set(k, { ref, currentGen: null, dirtyHint: 0, consecutiveFailures: 0, updatedAt: 0 });
|
|
2450
|
+
}
|
|
2451
|
+
}
|
|
2452
|
+
const out = [];
|
|
2453
|
+
for (const [key, seg] of known) {
|
|
2454
|
+
if (totalShards > 1 && shardOf(key, totalShards) !== shard) continue;
|
|
2455
|
+
if (seg.consecutiveFailures >= quarantineThreshold && now - seg.updatedAt < cooldownMs)
|
|
2456
|
+
continue;
|
|
2457
|
+
let dirty = 0;
|
|
2458
|
+
for await (const row of deps.warm.listChunks(seg.ref)) {
|
|
2459
|
+
dirty += 1;
|
|
2460
|
+
}
|
|
2461
|
+
if (seg.token !== void 0 && dirty !== seg.dirtyHint) {
|
|
2462
|
+
try {
|
|
2463
|
+
await deps.registry.compareAndSwap(seg.ref, seg.token, { dirtyChunkCount: dirty });
|
|
2464
|
+
} catch (err) {
|
|
2465
|
+
if (!isWriteConflictError(err)) throw err;
|
|
2466
|
+
}
|
|
2467
|
+
}
|
|
2468
|
+
if (dirty >= threshold) {
|
|
2469
|
+
out.push({
|
|
2470
|
+
ref: seg.ref,
|
|
2471
|
+
dirtyChunks: dirty,
|
|
2472
|
+
currentGen: seg.currentGen,
|
|
2473
|
+
lastCompactedAt: seg.lastCompactedAt
|
|
2474
|
+
});
|
|
2475
|
+
}
|
|
2476
|
+
}
|
|
2477
|
+
return out;
|
|
2478
|
+
}
|
|
2479
|
+
async function runCompactionCycle(deps, options) {
|
|
2480
|
+
validateCompactionOptions(options);
|
|
2481
|
+
if (options.maxSegments !== void 0 && (!Number.isInteger(options.maxSegments) || options.maxSegments < 1)) {
|
|
2482
|
+
throw new ValidationError(`maxSegments must be a positive integer; got ${options.maxSegments}`);
|
|
2483
|
+
}
|
|
2484
|
+
const found = await findCompactable(deps, options);
|
|
2485
|
+
found.sort(
|
|
2486
|
+
(a, b) => b.dirtyChunks - a.dirtyChunks || (a.lastCompactedAt ?? 0) - (b.lastCompactedAt ?? 0)
|
|
2487
|
+
);
|
|
2488
|
+
const budget = options.maxSegments;
|
|
2489
|
+
const picked = budget !== void 0 ? found.slice(0, budget) : found;
|
|
2490
|
+
const results = [];
|
|
2491
|
+
let compacted = 0;
|
|
2492
|
+
for (const { ref } of picked) {
|
|
2493
|
+
try {
|
|
2494
|
+
const result = await compactSegment(ref, deps, options);
|
|
2495
|
+
results.push(result);
|
|
2496
|
+
if (result.compacted) {
|
|
2497
|
+
compacted += 1;
|
|
2498
|
+
await gcOrphanGenerations(ref, deps, { keep: options.keep }).catch(() => void 0);
|
|
2499
|
+
}
|
|
2500
|
+
} catch (err) {
|
|
2501
|
+
await bumpFailure(deps, ref).catch(() => void 0);
|
|
2502
|
+
results.push({
|
|
2503
|
+
segment: ref.segment,
|
|
2504
|
+
namespace: ref.namespace,
|
|
2505
|
+
compacted: false,
|
|
2506
|
+
reason: "error",
|
|
2507
|
+
error: err instanceof Error ? err.message : String(err),
|
|
2508
|
+
fromGen: null,
|
|
2509
|
+
dirtyChunks: 0,
|
|
2510
|
+
purged: 0,
|
|
2511
|
+
survived: 0
|
|
2512
|
+
});
|
|
2513
|
+
}
|
|
2514
|
+
}
|
|
2515
|
+
return { candidates: found.length, compacted, deferred: found.length - picked.length, results };
|
|
2516
|
+
}
|
|
2517
|
+
async function bumpFailure(deps, ref) {
|
|
2518
|
+
const rec = await deps.registry.get(ref);
|
|
2519
|
+
if (rec === null) return;
|
|
2520
|
+
try {
|
|
2521
|
+
await deps.registry.compareAndSwap(ref, rec.token, {
|
|
2522
|
+
consecutiveFailures: (rec.consecutiveFailures ?? 0) + 1
|
|
2523
|
+
});
|
|
2524
|
+
} catch (err) {
|
|
2525
|
+
if (!isWriteConflictError(err)) throw err;
|
|
2526
|
+
}
|
|
2527
|
+
}
|
|
2528
|
+
|
|
2529
|
+
// src/export/index.ts
|
|
2530
|
+
var DEFAULT_NDJSON_BATCH_BYTES = 64 * 1024;
|
|
2531
|
+
var errMessage = (err) => err instanceof Error ? err.message : String(err);
|
|
2532
|
+
var refKey = (ref) => `${ref.namespace ?? ""} ${ref.segment}`;
|
|
2533
|
+
async function runExport(reader, registry, sink, options = {}) {
|
|
2534
|
+
const format = options.format ?? "roaring";
|
|
2535
|
+
const ext = format === "roaring" ? ".roaring" : ".ndjson";
|
|
2536
|
+
const batchCap = options.ndjsonBatchBytes ?? DEFAULT_NDJSON_BATCH_BYTES;
|
|
2537
|
+
const candidates = options.candidates ?? [];
|
|
2538
|
+
const segments = [];
|
|
2539
|
+
const failed = [];
|
|
2540
|
+
let totalIds = 0;
|
|
2541
|
+
const seen = candidates.length > 0 ? /* @__PURE__ */ new Set() : null;
|
|
2542
|
+
const exportRef = async (ref) => {
|
|
2543
|
+
let writer;
|
|
2544
|
+
let count = 0;
|
|
2545
|
+
let bytes = 0;
|
|
2546
|
+
try {
|
|
2547
|
+
validateSegmentRef(ref);
|
|
2548
|
+
const ids = reader.segment(ref.segment, { namespace: ref.namespace }).iterate();
|
|
2549
|
+
writer = await sink.open(ref, ext);
|
|
2550
|
+
if (format === "roaring") {
|
|
2551
|
+
const bm = requireCodec(options.codec, "runExport({ format: 'roaring' })").empty();
|
|
2552
|
+
for await (const id of ids) {
|
|
2553
|
+
bm.add(id);
|
|
2554
|
+
count += 1;
|
|
2555
|
+
}
|
|
2556
|
+
const out = bm.serialize();
|
|
2557
|
+
await writer.write(out);
|
|
2558
|
+
bytes = out.length;
|
|
2559
|
+
} else {
|
|
2560
|
+
let buf = "";
|
|
2561
|
+
for await (const id of ids) {
|
|
2562
|
+
buf += `${id}
|
|
2563
|
+
`;
|
|
2564
|
+
count += 1;
|
|
2565
|
+
if (buf.length >= batchCap) {
|
|
2566
|
+
const b = Buffer.from(buf, "utf8");
|
|
2567
|
+
await writer.write(b);
|
|
2568
|
+
bytes += b.length;
|
|
2569
|
+
buf = "";
|
|
2570
|
+
}
|
|
2571
|
+
}
|
|
2572
|
+
if (buf.length > 0) {
|
|
2573
|
+
const b = Buffer.from(buf, "utf8");
|
|
2574
|
+
await writer.write(b);
|
|
2575
|
+
bytes += b.length;
|
|
2576
|
+
}
|
|
2577
|
+
}
|
|
2578
|
+
} catch (err) {
|
|
2579
|
+
if (writer !== void 0) {
|
|
2580
|
+
try {
|
|
2581
|
+
await writer.abort?.();
|
|
2582
|
+
} catch {
|
|
2583
|
+
}
|
|
2584
|
+
}
|
|
2585
|
+
failed.push({ segment: ref.segment, namespace: ref.namespace, error: errMessage(err) });
|
|
2586
|
+
return;
|
|
2587
|
+
}
|
|
2588
|
+
try {
|
|
2589
|
+
await writer.close();
|
|
2590
|
+
} catch (err) {
|
|
2591
|
+
failed.push({ segment: ref.segment, namespace: ref.namespace, error: errMessage(err) });
|
|
2592
|
+
return;
|
|
2593
|
+
}
|
|
2594
|
+
segments.push({ segment: ref.segment, namespace: ref.namespace, count, bytes });
|
|
2595
|
+
totalIds += count;
|
|
2596
|
+
};
|
|
2597
|
+
for await (const rec of registry.list(options.namespace)) {
|
|
2598
|
+
if (rec.status === "destroyed") continue;
|
|
2599
|
+
if (seen !== null) seen.add(refKey(rec));
|
|
2600
|
+
await exportRef({ segment: rec.segment, namespace: rec.namespace });
|
|
2601
|
+
}
|
|
2602
|
+
for (const ref of candidates) {
|
|
2603
|
+
if (options.namespace !== void 0 && ref.namespace !== options.namespace) continue;
|
|
2604
|
+
const key = refKey(ref);
|
|
2605
|
+
if (seen !== null && seen.has(key)) continue;
|
|
2606
|
+
seen?.add(key);
|
|
2607
|
+
await exportRef(ref);
|
|
2608
|
+
}
|
|
2609
|
+
return { version: 1, format, totalSegments: segments.length, totalIds, segments, failed };
|
|
2610
|
+
}
|
|
2611
|
+
|
|
2612
|
+
// src/core/blob.ts
|
|
2613
|
+
var BufferSink = class {
|
|
2614
|
+
chunks = [];
|
|
2615
|
+
length = 0;
|
|
2616
|
+
write(bytes) {
|
|
2617
|
+
this.chunks.push(bytes.slice());
|
|
2618
|
+
this.length += bytes.length;
|
|
2619
|
+
return Promise.resolve();
|
|
2620
|
+
}
|
|
2621
|
+
bytes() {
|
|
2622
|
+
const out = new Uint8Array(this.length);
|
|
2623
|
+
let offset = 0;
|
|
2624
|
+
for (const chunk of this.chunks) {
|
|
2625
|
+
out.set(chunk, offset);
|
|
2626
|
+
offset += chunk.length;
|
|
2627
|
+
}
|
|
2628
|
+
return out;
|
|
2629
|
+
}
|
|
2630
|
+
};
|
|
2631
|
+
var BufferReader = class {
|
|
2632
|
+
constructor(buffer) {
|
|
2633
|
+
this.buffer = buffer;
|
|
2634
|
+
}
|
|
2635
|
+
buffer;
|
|
2636
|
+
getRange(offset, length) {
|
|
2637
|
+
if (!Number.isInteger(offset) || !Number.isInteger(length) || offset < 0 || length < 0 || offset + length > this.buffer.length) {
|
|
2638
|
+
throw new ValidationError(
|
|
2639
|
+
`range [${offset}, ${offset + length}) out of bounds for ${this.buffer.length}-byte blob`
|
|
2640
|
+
);
|
|
2641
|
+
}
|
|
2642
|
+
return Promise.resolve(this.buffer.subarray(offset, offset + length));
|
|
2643
|
+
}
|
|
2644
|
+
getTail(maxBytes) {
|
|
2645
|
+
const size = this.buffer.length;
|
|
2646
|
+
const take = Math.min(Math.max(maxBytes, 0), size);
|
|
2647
|
+
return Promise.resolve({ bytes: this.buffer.subarray(size - take), size });
|
|
2648
|
+
}
|
|
2649
|
+
};
|
|
2650
|
+
|
|
2651
|
+
// src/drivers/_shared/registry.ts
|
|
2652
|
+
var STATUSES = ["active", "compacting", "erasing", "destroyed"];
|
|
2653
|
+
var MAX_GOVERNANCE_BYTES = 64 * 1024;
|
|
2654
|
+
var MAX_WRAPPED_DEKS = 8;
|
|
2655
|
+
var MAX_WRAPPED_DEK_BYTES = 4 * 1024;
|
|
2656
|
+
function validateGeneration(gen) {
|
|
2657
|
+
if (!Number.isInteger(gen) || gen < 0) {
|
|
2658
|
+
throw new ValidationError(`currentGen must be a non-negative integer; got ${gen}`);
|
|
2659
|
+
}
|
|
2660
|
+
}
|
|
2661
|
+
function validateCount(count, field = "dirtyChunkCount") {
|
|
2662
|
+
if (!Number.isInteger(count) || count < 0) {
|
|
2663
|
+
throw new ValidationError(`${field} must be a non-negative integer; got ${count}`);
|
|
2664
|
+
}
|
|
2665
|
+
}
|
|
2666
|
+
function validateTimestamp(ms, field) {
|
|
2667
|
+
if (!Number.isFinite(ms) || ms < 0) {
|
|
2668
|
+
throw new ValidationError(`${field} must be a non-negative finite number; got ${ms}`);
|
|
2669
|
+
}
|
|
2670
|
+
}
|
|
2671
|
+
function validateStatus(status) {
|
|
2672
|
+
if (!STATUSES.includes(status)) {
|
|
2673
|
+
throw new ValidationError(`status must be one of ${STATUSES.join("/")}; got ${status}`);
|
|
2674
|
+
}
|
|
2675
|
+
}
|
|
2676
|
+
function validateGovernance(meta, field) {
|
|
2677
|
+
if (meta === void 0) return;
|
|
2678
|
+
let json;
|
|
2679
|
+
try {
|
|
2680
|
+
json = JSON.stringify(meta);
|
|
2681
|
+
} catch {
|
|
2682
|
+
throw new ValidationError(`${field} must be JSON-serializable`);
|
|
2683
|
+
}
|
|
2684
|
+
if (json.length > MAX_GOVERNANCE_BYTES) {
|
|
2685
|
+
throw new ValidationError(`${field} is ${json.length}B, exceeds cap ${MAX_GOVERNANCE_BYTES}B`);
|
|
2686
|
+
}
|
|
2687
|
+
}
|
|
2688
|
+
function validateWrappedDeks(value, isStored) {
|
|
2689
|
+
if (value === void 0) return;
|
|
2690
|
+
const fail = (msg) => {
|
|
2691
|
+
throw isStored ? new IntegrityError(msg) : new ValidationError(msg);
|
|
2692
|
+
};
|
|
2693
|
+
if (!Array.isArray(value)) fail("wrappedDeks must be an array");
|
|
2694
|
+
const list = value;
|
|
2695
|
+
if (list.length === 0) fail("wrappedDeks must be non-empty when present");
|
|
2696
|
+
if (list.length > MAX_WRAPPED_DEKS)
|
|
2697
|
+
fail(`wrappedDeks has ${list.length} entries, cap ${MAX_WRAPPED_DEKS}`);
|
|
2698
|
+
for (const w of list) {
|
|
2699
|
+
if (w === null || typeof w !== "object") fail("wrappedDeks entry must be an object");
|
|
2700
|
+
const { keyId, wrapped } = w;
|
|
2701
|
+
if (typeof keyId !== "string" || keyId.length === 0)
|
|
2702
|
+
fail("wrappedDeks entry needs a non-empty keyId");
|
|
2703
|
+
if (typeof wrapped !== "string" || wrapped.length === 0)
|
|
2704
|
+
fail("wrappedDeks entry needs a non-empty wrapped blob");
|
|
2705
|
+
if (wrapped.length > MAX_WRAPPED_DEK_BYTES)
|
|
2706
|
+
fail(`wrappedDeks entry is ${wrapped.length}B, cap ${MAX_WRAPPED_DEK_BYTES}B`);
|
|
2707
|
+
}
|
|
2708
|
+
}
|
|
2709
|
+
function validateNewRegistryRecord(rec) {
|
|
2710
|
+
validateGeneration(rec.currentGen);
|
|
2711
|
+
if (rec.dirtyChunkCount !== void 0) validateCount(rec.dirtyChunkCount);
|
|
2712
|
+
if (rec.status !== void 0) validateStatus(rec.status);
|
|
2713
|
+
validateWrappedDeks(rec.wrappedDeks, false);
|
|
2714
|
+
validateGovernance(rec.retention, "retention");
|
|
2715
|
+
validateGovernance(rec.residency, "residency");
|
|
2716
|
+
}
|
|
2717
|
+
function validateRegistryPatch(patch) {
|
|
2718
|
+
if (patch.currentGen !== void 0) validateGeneration(patch.currentGen);
|
|
2719
|
+
if (patch.dirtyChunkCount !== void 0) validateCount(patch.dirtyChunkCount);
|
|
2720
|
+
if (patch.consecutiveFailures !== void 0)
|
|
2721
|
+
validateCount(patch.consecutiveFailures, "consecutiveFailures");
|
|
2722
|
+
if (patch.lastCompactedAt !== void 0)
|
|
2723
|
+
validateTimestamp(patch.lastCompactedAt, "lastCompactedAt");
|
|
2724
|
+
if (patch.status !== void 0) validateStatus(patch.status);
|
|
2725
|
+
if ("wrappedDeks" in patch) validateWrappedDeks(patch.wrappedDeks, false);
|
|
2726
|
+
if ("retention" in patch) validateGovernance(patch.retention, "retention");
|
|
2727
|
+
if ("residency" in patch) validateGovernance(patch.residency, "residency");
|
|
2728
|
+
}
|
|
2729
|
+
var REGISTRY_SCHEMA_VERSION = 1;
|
|
2730
|
+
function assertRegistrySchemaVersion(raw, ctx) {
|
|
2731
|
+
if (raw === void 0) return;
|
|
2732
|
+
if (typeof raw !== "number" || !Number.isInteger(raw) || raw < 1) {
|
|
2733
|
+
throw new IntegrityError(`registry row has a malformed schemaVersion (${String(raw)}): ${ctx}`);
|
|
2734
|
+
}
|
|
2735
|
+
if (raw > REGISTRY_SCHEMA_VERSION) {
|
|
2736
|
+
throw new UnsupportedError(
|
|
2737
|
+
`registry row schemaVersion ${raw} is newer than this build reads (v${REGISTRY_SCHEMA_VERSION}): ${ctx}`
|
|
2738
|
+
);
|
|
2739
|
+
}
|
|
2740
|
+
}
|
|
2741
|
+
function serializeRegistryEnvelope(env) {
|
|
2742
|
+
return JSON.stringify({
|
|
2743
|
+
schemaVersion: REGISTRY_SCHEMA_VERSION,
|
|
2744
|
+
deleted: env.deleted,
|
|
2745
|
+
record: env.record
|
|
2746
|
+
});
|
|
2747
|
+
}
|
|
2748
|
+
function registryCounterOf(record) {
|
|
2749
|
+
if (!/^(0|[1-9]\d*)$/.test(record.token)) {
|
|
2750
|
+
throw new IntegrityError(`registry row token is not a canonical counter: ${record.token}`);
|
|
2751
|
+
}
|
|
2752
|
+
const n = Number(record.token);
|
|
2753
|
+
if (!Number.isSafeInteger(n)) {
|
|
2754
|
+
throw new IntegrityError(`registry row token is out of safe-integer range: ${record.token}`);
|
|
2755
|
+
}
|
|
2756
|
+
return n;
|
|
2757
|
+
}
|
|
2758
|
+
function parseRegistryEnvelope(text, ctx) {
|
|
2759
|
+
let parsed;
|
|
2760
|
+
try {
|
|
2761
|
+
parsed = JSON.parse(text);
|
|
2762
|
+
} catch {
|
|
2763
|
+
throw new IntegrityError(`registry row is not valid JSON: ${ctx}`);
|
|
2764
|
+
}
|
|
2765
|
+
if (parsed === null || typeof parsed !== "object") {
|
|
2766
|
+
throw new IntegrityError(`registry row has a malformed envelope: ${ctx}`);
|
|
2767
|
+
}
|
|
2768
|
+
const env = parsed;
|
|
2769
|
+
assertRegistrySchemaVersion(env.schemaVersion, ctx);
|
|
2770
|
+
if (typeof env.deleted !== "boolean" || env.record === null || typeof env.record !== "object") {
|
|
2771
|
+
throw new IntegrityError(`registry row has a malformed envelope: ${ctx}`);
|
|
2772
|
+
}
|
|
2773
|
+
const r = env.record;
|
|
2774
|
+
assertStoredRecordShape(r, ctx);
|
|
2775
|
+
if (typeof r.token !== "string") {
|
|
2776
|
+
throw new IntegrityError(`registry row is missing its token: ${ctx}`);
|
|
2777
|
+
}
|
|
2778
|
+
return { deleted: env.deleted, record: env.record };
|
|
2779
|
+
}
|
|
2780
|
+
function recordFromNew(ref, rec, now, token) {
|
|
2781
|
+
return {
|
|
2782
|
+
namespace: ref.namespace,
|
|
2783
|
+
segment: ref.segment,
|
|
2784
|
+
currentGen: rec.currentGen,
|
|
2785
|
+
wrappedDeks: rec.wrappedDeks,
|
|
2786
|
+
keyId: rec.keyId,
|
|
2787
|
+
dirtyChunkCount: rec.dirtyChunkCount ?? 0,
|
|
2788
|
+
status: rec.status ?? "active",
|
|
2789
|
+
consecutiveFailures: 0,
|
|
2790
|
+
// daemon health (Phase D); lastCompactedAt stays absent until first compaction
|
|
2791
|
+
retention: rec.retention,
|
|
2792
|
+
residency: rec.residency,
|
|
2793
|
+
createdAt: now,
|
|
2794
|
+
updatedAt: now,
|
|
2795
|
+
token
|
|
2796
|
+
};
|
|
2797
|
+
}
|
|
2798
|
+
function assertStoredRecordShape(r, ctx) {
|
|
2799
|
+
if (typeof r.segment !== "string" || r.namespace !== void 0 && typeof r.namespace !== "string" || typeof r.currentGen !== "number" || typeof r.dirtyChunkCount !== "number" || typeof r.status !== "string" || typeof r.createdAt !== "number" || typeof r.updatedAt !== "number") {
|
|
2800
|
+
throw new IntegrityError(`registry record is missing required fields: ${ctx}`);
|
|
2801
|
+
}
|
|
2802
|
+
if (!Number.isInteger(r.currentGen) || r.currentGen < 0) {
|
|
2803
|
+
throw new IntegrityError(`registry record has an invalid currentGen (${r.currentGen}): ${ctx}`);
|
|
2804
|
+
}
|
|
2805
|
+
if (!Number.isInteger(r.dirtyChunkCount) || r.dirtyChunkCount < 0) {
|
|
2806
|
+
throw new IntegrityError(`registry record has an invalid dirtyChunkCount: ${ctx}`);
|
|
2807
|
+
}
|
|
2808
|
+
if (!STATUSES.includes(r.status)) {
|
|
2809
|
+
throw new IntegrityError(`registry record has an unknown status (${r.status}): ${ctx}`);
|
|
2810
|
+
}
|
|
2811
|
+
if (r.leaseOwner !== void 0 && typeof r.leaseOwner !== "string") {
|
|
2812
|
+
throw new IntegrityError(`registry record has an invalid leaseOwner: ${ctx}`);
|
|
2813
|
+
}
|
|
2814
|
+
if (r.leaseExpiresAt !== void 0 && typeof r.leaseExpiresAt !== "number") {
|
|
2815
|
+
throw new IntegrityError(`registry record has an invalid leaseExpiresAt: ${ctx}`);
|
|
2816
|
+
}
|
|
2817
|
+
if (r.consecutiveFailures !== void 0 && (!Number.isInteger(r.consecutiveFailures) || r.consecutiveFailures < 0)) {
|
|
2818
|
+
throw new IntegrityError(`registry record has an invalid consecutiveFailures: ${ctx}`);
|
|
2819
|
+
}
|
|
2820
|
+
if (r.lastCompactedAt !== void 0 && (typeof r.lastCompactedAt !== "number" || !Number.isFinite(r.lastCompactedAt) || r.lastCompactedAt < 0)) {
|
|
2821
|
+
throw new IntegrityError(`registry record has an invalid lastCompactedAt: ${ctx}`);
|
|
2822
|
+
}
|
|
2823
|
+
if (r.keyId !== void 0 && typeof r.keyId !== "string") {
|
|
2824
|
+
throw new IntegrityError(`registry record has an invalid keyId: ${ctx}`);
|
|
2825
|
+
}
|
|
2826
|
+
validateWrappedDeks(r.wrappedDeks, true);
|
|
2827
|
+
}
|
|
2828
|
+
function applyRegistryPatch(prev, patch, now, token) {
|
|
2829
|
+
return {
|
|
2830
|
+
namespace: prev.namespace,
|
|
2831
|
+
segment: prev.segment,
|
|
2832
|
+
currentGen: patch.currentGen ?? prev.currentGen,
|
|
2833
|
+
wrappedDeks: "wrappedDeks" in patch ? patch.wrappedDeks : prev.wrappedDeks,
|
|
2834
|
+
keyId: "keyId" in patch ? patch.keyId : prev.keyId,
|
|
2835
|
+
dirtyChunkCount: patch.dirtyChunkCount ?? prev.dirtyChunkCount,
|
|
2836
|
+
status: patch.status ?? prev.status,
|
|
2837
|
+
leaseOwner: "leaseOwner" in patch ? patch.leaseOwner : prev.leaseOwner,
|
|
2838
|
+
leaseExpiresAt: "leaseExpiresAt" in patch ? patch.leaseExpiresAt : prev.leaseExpiresAt,
|
|
2839
|
+
lastCompactedAt: "lastCompactedAt" in patch ? patch.lastCompactedAt : prev.lastCompactedAt,
|
|
2840
|
+
consecutiveFailures: patch.consecutiveFailures ?? prev.consecutiveFailures ?? 0,
|
|
2841
|
+
retention: "retention" in patch ? patch.retention : prev.retention,
|
|
2842
|
+
residency: "residency" in patch ? patch.residency : prev.residency,
|
|
2843
|
+
createdAt: prev.createdAt,
|
|
2844
|
+
updatedAt: now,
|
|
2845
|
+
token
|
|
2846
|
+
};
|
|
2847
|
+
}
|
|
2848
|
+
|
|
2849
|
+
// src/drivers/memory.ts
|
|
2850
|
+
var MemoryWarmDriver = class {
|
|
2851
|
+
rows = /* @__PURE__ */ new Map();
|
|
2852
|
+
seq = 0;
|
|
2853
|
+
// Monotonic, unique-per-write token → equality-safe and ABA-free.
|
|
2854
|
+
nextToken() {
|
|
2855
|
+
this.seq += 1;
|
|
2856
|
+
return String(this.seq);
|
|
2857
|
+
}
|
|
2858
|
+
// In-memory reads are always strongly consistent — the `WarmReadOptions` hint would be a no-op, so the
|
|
2859
|
+
// structurally-optional param is simply omitted (still satisfies IWarmDriver).
|
|
2860
|
+
async get(ref) {
|
|
2861
|
+
validateChunkRef(ref);
|
|
2862
|
+
const row = this.rows.get(chunkRefKey(ref));
|
|
2863
|
+
return row ? { token: row.token, bytes: row.bytes } : null;
|
|
2864
|
+
}
|
|
2865
|
+
async putConditional(ref, bytes, expected) {
|
|
2866
|
+
validateChunkRef(ref);
|
|
2867
|
+
const key = chunkRefKey(ref);
|
|
2868
|
+
const existing = this.rows.get(key);
|
|
2869
|
+
if (expected === NO_ROW) {
|
|
2870
|
+
if (existing) throw new WriteConflictError(`row already exists for chunk ${ref.chunkKey}`);
|
|
2871
|
+
} else if (!existing || existing.token !== expected) {
|
|
2872
|
+
throw new WriteConflictError(`OCC token mismatch for chunk ${ref.chunkKey}`);
|
|
2873
|
+
}
|
|
2874
|
+
const token = this.nextToken();
|
|
2875
|
+
this.rows.set(key, { token, bytes, chunkKey: ref.chunkKey, prefix: segmentPrefix(ref) });
|
|
2876
|
+
return { token };
|
|
2877
|
+
}
|
|
2878
|
+
async deleteConditional(ref, expected) {
|
|
2879
|
+
validateChunkRef(ref);
|
|
2880
|
+
const key = chunkRefKey(ref);
|
|
2881
|
+
const existing = this.rows.get(key);
|
|
2882
|
+
if (!existing || existing.token !== expected) {
|
|
2883
|
+
throw new WriteConflictError(`OCC token mismatch for chunk ${ref.chunkKey}`);
|
|
2884
|
+
}
|
|
2885
|
+
this.rows.delete(key);
|
|
2886
|
+
}
|
|
2887
|
+
async *listChunks(ref) {
|
|
2888
|
+
validateSegmentRef(ref);
|
|
2889
|
+
const prefix = segmentPrefix(ref);
|
|
2890
|
+
const rows = [...this.rows.values()].filter((r) => r.prefix === prefix);
|
|
2891
|
+
rows.sort((a, b) => a.chunkKey - b.chunkKey);
|
|
2892
|
+
for (const row of rows) {
|
|
2893
|
+
yield { chunkKey: row.chunkKey, token: row.token, bytes: row.bytes };
|
|
2894
|
+
}
|
|
2895
|
+
}
|
|
2896
|
+
};
|
|
2897
|
+
var MemoryColdChunkSource = class {
|
|
2898
|
+
chunks = /* @__PURE__ */ new Map();
|
|
2899
|
+
async getChunk(ref) {
|
|
2900
|
+
validateChunkRef(ref);
|
|
2901
|
+
return this.chunks.get(chunkRefKey(ref)) ?? null;
|
|
2902
|
+
}
|
|
2903
|
+
async listChunkKeys(ref) {
|
|
2904
|
+
validateSegmentRef(ref);
|
|
2905
|
+
const prefix = segmentPrefix(ref);
|
|
2906
|
+
const keys = [];
|
|
2907
|
+
for (const key of this.chunks.keys()) {
|
|
2908
|
+
if (key.startsWith(prefix)) keys.push(Number(key.slice(prefix.length)));
|
|
2909
|
+
}
|
|
2910
|
+
return keys;
|
|
2911
|
+
}
|
|
2912
|
+
async sizeOf(ref) {
|
|
2913
|
+
validateSegmentRef(ref);
|
|
2914
|
+
const prefix = segmentPrefix(ref);
|
|
2915
|
+
let sizeBytes = 0;
|
|
2916
|
+
let found = false;
|
|
2917
|
+
for (const [key, bytes] of this.chunks) {
|
|
2918
|
+
if (!key.startsWith(prefix)) continue;
|
|
2919
|
+
found = true;
|
|
2920
|
+
sizeBytes += bytes.length;
|
|
2921
|
+
}
|
|
2922
|
+
return found ? { sizeBytes } : null;
|
|
2923
|
+
}
|
|
2924
|
+
/** Test/seed helper — populate immutable Cold bytes for a chunk (no bulk-load path until Phase 3). */
|
|
2925
|
+
seed(ref, bytes) {
|
|
2926
|
+
validateChunkRef(ref);
|
|
2927
|
+
this.chunks.set(chunkRefKey(ref), bytes);
|
|
2928
|
+
}
|
|
2929
|
+
};
|
|
2930
|
+
var MemoryRegistryDriver = class {
|
|
2931
|
+
rows = /* @__PURE__ */ new Map();
|
|
2932
|
+
now;
|
|
2933
|
+
seq = 0;
|
|
2934
|
+
constructor(options = {}) {
|
|
2935
|
+
this.now = options.now ?? (() => Date.now());
|
|
2936
|
+
}
|
|
2937
|
+
nextToken() {
|
|
2938
|
+
this.seq += 1;
|
|
2939
|
+
return String(this.seq);
|
|
2940
|
+
}
|
|
2941
|
+
capabilities() {
|
|
2942
|
+
return { strongRead: true };
|
|
2943
|
+
}
|
|
2944
|
+
async get(ref) {
|
|
2945
|
+
validateSegmentRef(ref);
|
|
2946
|
+
const row = this.rows.get(segmentKey(ref));
|
|
2947
|
+
return row ? structuredClone(row) : null;
|
|
2948
|
+
}
|
|
2949
|
+
async create(ref, record) {
|
|
2950
|
+
validateSegmentRef(ref);
|
|
2951
|
+
validateNewRegistryRecord(record);
|
|
2952
|
+
const key = segmentKey(ref);
|
|
2953
|
+
if (this.rows.has(key)) {
|
|
2954
|
+
throw new WriteConflictError(`registry row already exists for segment ${ref.segment}`);
|
|
2955
|
+
}
|
|
2956
|
+
const token = this.nextToken();
|
|
2957
|
+
this.rows.set(key, structuredClone(recordFromNew(ref, record, this.now(), token)));
|
|
2958
|
+
return { token };
|
|
2959
|
+
}
|
|
2960
|
+
async compareAndSwap(ref, expected, patch) {
|
|
2961
|
+
validateSegmentRef(ref);
|
|
2962
|
+
validateRegistryPatch(patch);
|
|
2963
|
+
const key = segmentKey(ref);
|
|
2964
|
+
const existing = this.rows.get(key);
|
|
2965
|
+
if (!existing || existing.token !== expected) {
|
|
2966
|
+
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
2967
|
+
}
|
|
2968
|
+
const token = this.nextToken();
|
|
2969
|
+
this.rows.set(key, structuredClone(applyRegistryPatch(existing, patch, this.now(), token)));
|
|
2970
|
+
return { token };
|
|
2971
|
+
}
|
|
2972
|
+
async *list(namespace) {
|
|
2973
|
+
for (const row of this.rows.values()) {
|
|
2974
|
+
if (namespace === void 0 || row.namespace === namespace) yield structuredClone(row);
|
|
2975
|
+
}
|
|
2976
|
+
}
|
|
2977
|
+
async delete(ref) {
|
|
2978
|
+
validateSegmentRef(ref);
|
|
2979
|
+
this.rows.delete(segmentKey(ref));
|
|
2980
|
+
}
|
|
2981
|
+
};
|
|
2982
|
+
var MemoryColdDriver = class {
|
|
2983
|
+
objects = /* @__PURE__ */ new Map();
|
|
2984
|
+
capabilities() {
|
|
2985
|
+
return { rangeRead: true, maxObjectBytes: Number.MAX_SAFE_INTEGER, conditionalPut: true };
|
|
2986
|
+
}
|
|
2987
|
+
async putImmutable(key, write) {
|
|
2988
|
+
const k = genObjectKey(key);
|
|
2989
|
+
const sink = new BufferSink();
|
|
2990
|
+
await write(sink);
|
|
2991
|
+
const body = sink.bytes();
|
|
2992
|
+
if (this.objects.has(k)) {
|
|
2993
|
+
throw new WriteConflictError(
|
|
2994
|
+
`generation already exists (write-once): ${key.segment}.${key.generation}`
|
|
2995
|
+
);
|
|
2996
|
+
}
|
|
2997
|
+
this.objects.set(k, body);
|
|
2998
|
+
return { size: body.length, sha256: crypto.createHash("sha256").update(body).digest("hex") };
|
|
2999
|
+
}
|
|
3000
|
+
async getRange(key, offset, length) {
|
|
3001
|
+
if (!Number.isInteger(offset) || !Number.isInteger(length) || offset < 0 || length < 0) {
|
|
3002
|
+
throw new ValidationError(`invalid range offset=${offset} length=${length}`);
|
|
3003
|
+
}
|
|
3004
|
+
const body = this.require(key);
|
|
3005
|
+
if (offset + length > body.length) {
|
|
3006
|
+
throw new ValidationError(
|
|
3007
|
+
`range [${offset}, ${offset + length}) out of bounds for ${body.length}B`
|
|
3008
|
+
);
|
|
3009
|
+
}
|
|
3010
|
+
return body.slice(offset, offset + length);
|
|
3011
|
+
}
|
|
3012
|
+
async getTail(key, maxBytes) {
|
|
3013
|
+
const body = this.require(key);
|
|
3014
|
+
const take = Math.min(Math.max(maxBytes, 0), body.length);
|
|
3015
|
+
return { bytes: body.slice(body.length - take), size: body.length };
|
|
3016
|
+
}
|
|
3017
|
+
async delete(key) {
|
|
3018
|
+
this.objects.delete(genObjectKey(key));
|
|
3019
|
+
}
|
|
3020
|
+
async *list(ref) {
|
|
3021
|
+
validateSegmentRef(ref);
|
|
3022
|
+
const prefix = `${segmentKey(ref)} `;
|
|
3023
|
+
for (const k of this.objects.keys()) {
|
|
3024
|
+
if (k.startsWith(prefix)) {
|
|
3025
|
+
yield {
|
|
3026
|
+
namespace: ref.namespace,
|
|
3027
|
+
segment: ref.segment,
|
|
3028
|
+
generation: Number(k.slice(prefix.length))
|
|
3029
|
+
};
|
|
3030
|
+
}
|
|
3031
|
+
}
|
|
3032
|
+
}
|
|
3033
|
+
require(key) {
|
|
3034
|
+
const body = this.objects.get(genObjectKey(key));
|
|
3035
|
+
if (body === void 0) {
|
|
3036
|
+
throw new NotFoundError(`no such generation: ${key.segment}.${key.generation}`);
|
|
3037
|
+
}
|
|
3038
|
+
return body;
|
|
3039
|
+
}
|
|
3040
|
+
};
|
|
3041
|
+
function genObjectKey(key) {
|
|
3042
|
+
validateSegmentRef(key);
|
|
3043
|
+
if (!Number.isInteger(key.generation) || key.generation < 0) {
|
|
3044
|
+
throw new ValidationError(`generation must be a non-negative integer; got ${key.generation}`);
|
|
3045
|
+
}
|
|
3046
|
+
return `${segmentKey(key)} ${key.generation}`;
|
|
3047
|
+
}
|
|
3048
|
+
|
|
3049
|
+
// src/drivers/_shared/keys.ts
|
|
3050
|
+
var DEFAULT_NAMESPACE = "_default";
|
|
3051
|
+
function namespacePart(namespace) {
|
|
3052
|
+
return namespace ?? DEFAULT_NAMESPACE;
|
|
3053
|
+
}
|
|
3054
|
+
|
|
3055
|
+
// src/drivers/localfs/paths.ts
|
|
3056
|
+
function segmentsDir(root, ref) {
|
|
3057
|
+
validateSegmentRef(ref);
|
|
3058
|
+
return path.join(root, namespacePart(ref.namespace), "segments");
|
|
3059
|
+
}
|
|
3060
|
+
function coldObjectPath(root, key) {
|
|
3061
|
+
validateSegmentRef(key);
|
|
3062
|
+
if (!Number.isInteger(key.generation) || key.generation < 0) {
|
|
3063
|
+
throw new ValidationError(`generation must be a non-negative integer; got ${key.generation}`);
|
|
3064
|
+
}
|
|
3065
|
+
return path.join(segmentsDir(root, key), `${key.segment}.${key.generation}.crbm`);
|
|
3066
|
+
}
|
|
3067
|
+
function parseGeneration(segment, filename) {
|
|
3068
|
+
const prefix = `${segment}.`;
|
|
3069
|
+
const suffix = ".crbm";
|
|
3070
|
+
if (!filename.startsWith(prefix) || !filename.endsWith(suffix)) return null;
|
|
3071
|
+
const middle = filename.slice(prefix.length, filename.length - suffix.length);
|
|
3072
|
+
if (!/^(0|[1-9]\d*)$/.test(middle)) return null;
|
|
3073
|
+
const gen = Number(middle);
|
|
3074
|
+
return Number.isSafeInteger(gen) ? gen : null;
|
|
3075
|
+
}
|
|
3076
|
+
var WARM_ROW_SUFFIX = ".row";
|
|
3077
|
+
var CHUNK_KEY_MAX2 = 65535;
|
|
3078
|
+
function warmSegmentDir(root, ref) {
|
|
3079
|
+
validateSegmentRef(ref);
|
|
3080
|
+
return path.join(root, namespacePart(ref.namespace), "warm", ref.segment);
|
|
3081
|
+
}
|
|
3082
|
+
function warmRowPath(root, ref) {
|
|
3083
|
+
validateChunkRef(ref);
|
|
3084
|
+
return path.join(warmSegmentDir(root, ref), `${ref.chunkKey}${WARM_ROW_SUFFIX}`);
|
|
3085
|
+
}
|
|
3086
|
+
var REGISTRY_SUFFIX = ".reg";
|
|
3087
|
+
function registryDir(root, namespace) {
|
|
3088
|
+
return path.join(root, namespacePart(namespace), "registry");
|
|
3089
|
+
}
|
|
3090
|
+
function registryRowPath(root, ref) {
|
|
3091
|
+
validateSegmentRef(ref);
|
|
3092
|
+
return path.join(registryDir(root, ref.namespace), `${ref.segment}${REGISTRY_SUFFIX}`);
|
|
3093
|
+
}
|
|
3094
|
+
function parseRegistryRow(filename) {
|
|
3095
|
+
if (!filename.endsWith(REGISTRY_SUFFIX)) return null;
|
|
3096
|
+
const segment = filename.slice(0, filename.length - REGISTRY_SUFFIX.length);
|
|
3097
|
+
try {
|
|
3098
|
+
validateSegmentRef({ segment });
|
|
3099
|
+
} catch {
|
|
3100
|
+
return null;
|
|
3101
|
+
}
|
|
3102
|
+
return segment;
|
|
3103
|
+
}
|
|
3104
|
+
function parseChunkRow(filename) {
|
|
3105
|
+
if (!filename.endsWith(WARM_ROW_SUFFIX)) return null;
|
|
3106
|
+
const middle = filename.slice(0, filename.length - WARM_ROW_SUFFIX.length);
|
|
3107
|
+
if (!/^(0|[1-9]\d*)$/.test(middle)) return null;
|
|
3108
|
+
const chunkKey = Number(middle);
|
|
3109
|
+
return chunkKey <= CHUNK_KEY_MAX2 ? chunkKey : null;
|
|
3110
|
+
}
|
|
3111
|
+
var isCode = (err, code) => err instanceof Error && err.code === code;
|
|
3112
|
+
var TRANSIENT_FS_CODES = /* @__PURE__ */ new Set(["EBUSY", "EAGAIN", "EMFILE", "ENFILE", "ETIMEDOUT"]);
|
|
3113
|
+
var isFsTransient = (err) => err instanceof Error && TRANSIENT_FS_CODES.has(err.code ?? "");
|
|
3114
|
+
var mapFsError = (err) => isFsTransient(err) ? new TransientError(`transient filesystem fault: ${err.code}`, { cause: err }) : err;
|
|
3115
|
+
var O_NOFOLLOW = fs.constants.O_NOFOLLOW ?? 0;
|
|
3116
|
+
async function fsyncDir(dir) {
|
|
3117
|
+
try {
|
|
3118
|
+
const handle = await promises.open(dir, "r");
|
|
3119
|
+
try {
|
|
3120
|
+
await handle.sync();
|
|
3121
|
+
} finally {
|
|
3122
|
+
await handle.close();
|
|
3123
|
+
}
|
|
3124
|
+
} catch {
|
|
3125
|
+
}
|
|
3126
|
+
}
|
|
3127
|
+
|
|
3128
|
+
// src/drivers/localfs/cold.ts
|
|
3129
|
+
var LocalFsColdDriver = class {
|
|
3130
|
+
constructor(root) {
|
|
3131
|
+
this.root = root;
|
|
3132
|
+
}
|
|
3133
|
+
root;
|
|
3134
|
+
capabilities() {
|
|
3135
|
+
return { rangeRead: true, maxObjectBytes: Number.MAX_SAFE_INTEGER, conditionalPut: false };
|
|
3136
|
+
}
|
|
3137
|
+
async putImmutable(key, write) {
|
|
3138
|
+
const finalPath = coldObjectPath(this.root, key);
|
|
3139
|
+
await promises.mkdir(path.dirname(finalPath), { recursive: true });
|
|
3140
|
+
const tmpPath = `${finalPath}.${crypto.randomUUID()}.tmp`;
|
|
3141
|
+
const handle = await promises.open(tmpPath, fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY | O_NOFOLLOW);
|
|
3142
|
+
const hash = crypto.createHash("sha256");
|
|
3143
|
+
let size = 0;
|
|
3144
|
+
const sink = {
|
|
3145
|
+
async write(bytes) {
|
|
3146
|
+
await handle.write(bytes);
|
|
3147
|
+
hash.update(bytes);
|
|
3148
|
+
size += bytes.length;
|
|
3149
|
+
}
|
|
3150
|
+
};
|
|
3151
|
+
try {
|
|
3152
|
+
await write(sink);
|
|
3153
|
+
await handle.sync();
|
|
3154
|
+
} catch (err) {
|
|
3155
|
+
await handle.close().catch(() => {
|
|
3156
|
+
});
|
|
3157
|
+
await promises.unlink(tmpPath).catch(() => {
|
|
3158
|
+
});
|
|
3159
|
+
throw mapFsError(err);
|
|
3160
|
+
}
|
|
3161
|
+
await handle.close();
|
|
3162
|
+
try {
|
|
3163
|
+
await promises.link(tmpPath, finalPath);
|
|
3164
|
+
} catch (err) {
|
|
3165
|
+
await promises.unlink(tmpPath).catch(() => {
|
|
3166
|
+
});
|
|
3167
|
+
if (isCode(err, "EEXIST")) {
|
|
3168
|
+
throw new WriteConflictError(
|
|
3169
|
+
`generation already exists (write-once): ${key.segment}.${key.generation}`
|
|
3170
|
+
);
|
|
3171
|
+
}
|
|
3172
|
+
throw mapFsError(err);
|
|
3173
|
+
}
|
|
3174
|
+
await promises.unlink(tmpPath).catch(() => {
|
|
3175
|
+
});
|
|
3176
|
+
await fsyncDir(path.dirname(finalPath));
|
|
3177
|
+
return { size, sha256: hash.digest("hex") };
|
|
3178
|
+
}
|
|
3179
|
+
async getRange(key, offset, length) {
|
|
3180
|
+
if (!Number.isInteger(offset) || !Number.isInteger(length) || offset < 0 || length < 0) {
|
|
3181
|
+
throw new ValidationError(`invalid range offset=${offset} length=${length}`);
|
|
3182
|
+
}
|
|
3183
|
+
const handle = await this.openRead(key);
|
|
3184
|
+
try {
|
|
3185
|
+
const { size } = await handle.stat();
|
|
3186
|
+
if (offset + length > size) {
|
|
3187
|
+
throw new ValidationError(
|
|
3188
|
+
`range [${offset}, ${offset + length}) out of bounds for ${size}B`
|
|
3189
|
+
);
|
|
3190
|
+
}
|
|
3191
|
+
const buf = Buffer.alloc(length);
|
|
3192
|
+
if (length > 0) await handle.read(buf, 0, length, offset);
|
|
3193
|
+
return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
3194
|
+
} finally {
|
|
3195
|
+
await handle.close();
|
|
3196
|
+
}
|
|
3197
|
+
}
|
|
3198
|
+
async getTail(key, maxBytes) {
|
|
3199
|
+
const handle = await this.openRead(key);
|
|
3200
|
+
try {
|
|
3201
|
+
const { size } = await handle.stat();
|
|
3202
|
+
const take = Math.min(Math.max(maxBytes, 0), size);
|
|
3203
|
+
const buf = Buffer.alloc(take);
|
|
3204
|
+
if (take > 0) await handle.read(buf, 0, take, size - take);
|
|
3205
|
+
return { bytes: new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength), size };
|
|
3206
|
+
} finally {
|
|
3207
|
+
await handle.close();
|
|
3208
|
+
}
|
|
3209
|
+
}
|
|
3210
|
+
async delete(key) {
|
|
3211
|
+
await promises.unlink(coldObjectPath(this.root, key)).catch((err) => {
|
|
3212
|
+
if (!isCode(err, "ENOENT")) throw mapFsError(err);
|
|
3213
|
+
});
|
|
3214
|
+
}
|
|
3215
|
+
async *list(ref) {
|
|
3216
|
+
const dir = segmentsDir(this.root, ref);
|
|
3217
|
+
let names;
|
|
3218
|
+
try {
|
|
3219
|
+
names = await promises.readdir(dir);
|
|
3220
|
+
} catch (err) {
|
|
3221
|
+
if (isCode(err, "ENOENT")) return;
|
|
3222
|
+
throw mapFsError(err);
|
|
3223
|
+
}
|
|
3224
|
+
for (const name of names) {
|
|
3225
|
+
const generation = parseGeneration(ref.segment, name);
|
|
3226
|
+
if (generation !== null) {
|
|
3227
|
+
yield { namespace: ref.namespace, segment: ref.segment, generation };
|
|
3228
|
+
}
|
|
3229
|
+
}
|
|
3230
|
+
}
|
|
3231
|
+
async openRead(key) {
|
|
3232
|
+
try {
|
|
3233
|
+
return await promises.open(coldObjectPath(this.root, key), fs.constants.O_RDONLY | O_NOFOLLOW);
|
|
3234
|
+
} catch (err) {
|
|
3235
|
+
if (isCode(err, "ENOENT")) {
|
|
3236
|
+
throw new NotFoundError(`no such generation: ${key.segment}.${key.generation}`);
|
|
3237
|
+
}
|
|
3238
|
+
if (isCode(err, "ELOOP")) {
|
|
3239
|
+
throw new NotFoundError(
|
|
3240
|
+
`generation path is a symlink, refusing: ${key.segment}.${key.generation}`
|
|
3241
|
+
);
|
|
3242
|
+
}
|
|
3243
|
+
throw mapFsError(err);
|
|
3244
|
+
}
|
|
3245
|
+
}
|
|
3246
|
+
};
|
|
3247
|
+
var HEADER_BYTES2 = 5;
|
|
3248
|
+
var COUNTER_MAX = 4294967295;
|
|
3249
|
+
var DEFAULT_MAX_ROW_BYTES = 64 * 1024 * 1024;
|
|
3250
|
+
var LocalFsWarmDriver = class {
|
|
3251
|
+
constructor(root, maxRowBytes = DEFAULT_MAX_ROW_BYTES) {
|
|
3252
|
+
this.root = root;
|
|
3253
|
+
this.maxRowBytes = maxRowBytes;
|
|
3254
|
+
}
|
|
3255
|
+
root;
|
|
3256
|
+
maxRowBytes;
|
|
3257
|
+
/** Per-row promise chain — serializes read-modify-write so an in-process CAS never loses an update. */
|
|
3258
|
+
chain = /* @__PURE__ */ new Map();
|
|
3259
|
+
// Methods are `async` so boundary-validation throws (warmRowPath) surface as rejections, not sync throws.
|
|
3260
|
+
// LocalFs reads are always strongly consistent (single filesystem) — the `WarmReadOptions` hint would be a
|
|
3261
|
+
// no-op, so the structurally-optional param is simply omitted (still satisfies IWarmDriver).
|
|
3262
|
+
async get(ref) {
|
|
3263
|
+
const row = await this.readRow(warmRowPath(this.root, ref));
|
|
3264
|
+
return row && !row.deleted ? { token: String(row.counter), bytes: row.bytes } : null;
|
|
3265
|
+
}
|
|
3266
|
+
async putConditional(ref, bytes, expected) {
|
|
3267
|
+
const path = warmRowPath(this.root, ref);
|
|
3268
|
+
return this.withRowLock(path, async () => {
|
|
3269
|
+
const current = await this.readRow(path);
|
|
3270
|
+
const live = current !== null && !current.deleted;
|
|
3271
|
+
if (expected === NO_ROW) {
|
|
3272
|
+
if (live) throw new WriteConflictError(`row already exists: chunk ${ref.chunkKey}`);
|
|
3273
|
+
const counter = current ? current.counter + 1 : 0;
|
|
3274
|
+
await this.writeRow(path, counter, false, bytes);
|
|
3275
|
+
return { token: String(counter) };
|
|
3276
|
+
}
|
|
3277
|
+
if (!live || String(current.counter) !== expected) {
|
|
3278
|
+
throw new WriteConflictError(`OCC conflict on chunk ${ref.chunkKey}`);
|
|
3279
|
+
}
|
|
3280
|
+
await this.writeRow(path, current.counter + 1, false, bytes);
|
|
3281
|
+
return { token: String(current.counter + 1) };
|
|
3282
|
+
});
|
|
3283
|
+
}
|
|
3284
|
+
async deleteConditional(ref, expected) {
|
|
3285
|
+
const path = warmRowPath(this.root, ref);
|
|
3286
|
+
return this.withRowLock(path, async () => {
|
|
3287
|
+
const current = await this.readRow(path);
|
|
3288
|
+
if (current === null || current.deleted || String(current.counter) !== expected) {
|
|
3289
|
+
throw new WriteConflictError(`OCC conflict on delete of chunk ${ref.chunkKey}`);
|
|
3290
|
+
}
|
|
3291
|
+
await this.writeRow(path, current.counter + 1, true, new Uint8Array());
|
|
3292
|
+
});
|
|
3293
|
+
}
|
|
3294
|
+
async *listChunks(ref) {
|
|
3295
|
+
const dir = warmSegmentDir(this.root, ref);
|
|
3296
|
+
let names;
|
|
3297
|
+
try {
|
|
3298
|
+
names = await promises.readdir(dir);
|
|
3299
|
+
} catch (err) {
|
|
3300
|
+
if (isCode(err, "ENOENT")) return;
|
|
3301
|
+
throw mapFsError(err);
|
|
3302
|
+
}
|
|
3303
|
+
const keys = names.map((n) => parseChunkRow(n)).filter((k) => k !== null).sort((a, b) => a - b);
|
|
3304
|
+
for (const chunkKey of keys) {
|
|
3305
|
+
const row = await this.readRow(warmRowPath(this.root, { ...ref, chunkKey }));
|
|
3306
|
+
if (row && !row.deleted) yield { chunkKey, token: String(row.counter), bytes: row.bytes };
|
|
3307
|
+
}
|
|
3308
|
+
}
|
|
3309
|
+
async readRow(path) {
|
|
3310
|
+
let handle;
|
|
3311
|
+
try {
|
|
3312
|
+
handle = await promises.open(path, fs.constants.O_RDONLY | O_NOFOLLOW);
|
|
3313
|
+
} catch (err) {
|
|
3314
|
+
if (isCode(err, "ENOENT") || isCode(err, "ELOOP")) return null;
|
|
3315
|
+
throw mapFsError(err);
|
|
3316
|
+
}
|
|
3317
|
+
try {
|
|
3318
|
+
const { size } = await handle.stat();
|
|
3319
|
+
if (size > this.maxRowBytes) {
|
|
3320
|
+
throw new IntegrityError(`warm row ${size}B exceeds cap ${this.maxRowBytes}B`);
|
|
3321
|
+
}
|
|
3322
|
+
if (size < HEADER_BYTES2) {
|
|
3323
|
+
throw new IntegrityError(`warm row truncated: ${size}B < ${HEADER_BYTES2}B header`);
|
|
3324
|
+
}
|
|
3325
|
+
const buf = await handle.readFile();
|
|
3326
|
+
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
3327
|
+
return {
|
|
3328
|
+
counter: view.getUint32(0, true),
|
|
3329
|
+
deleted: buf[4] !== 0,
|
|
3330
|
+
bytes: new Uint8Array(buf.subarray(HEADER_BYTES2))
|
|
3331
|
+
};
|
|
3332
|
+
} finally {
|
|
3333
|
+
await handle.close();
|
|
3334
|
+
}
|
|
3335
|
+
}
|
|
3336
|
+
async writeRow(path$1, counter, deleted, bytes) {
|
|
3337
|
+
if (counter > COUNTER_MAX) {
|
|
3338
|
+
throw new IntegrityError(`warm row counter overflow on ${path$1}`);
|
|
3339
|
+
}
|
|
3340
|
+
await promises.mkdir(path.dirname(path$1), { recursive: true });
|
|
3341
|
+
const out = new Uint8Array(HEADER_BYTES2 + bytes.length);
|
|
3342
|
+
new DataView(out.buffer).setUint32(0, counter, true);
|
|
3343
|
+
out[4] = deleted ? 1 : 0;
|
|
3344
|
+
out.set(bytes, HEADER_BYTES2);
|
|
3345
|
+
const tmp = `${path$1}.${crypto.randomUUID()}.tmp`;
|
|
3346
|
+
let handle;
|
|
3347
|
+
try {
|
|
3348
|
+
handle = await promises.open(tmp, fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY | O_NOFOLLOW);
|
|
3349
|
+
} catch (err) {
|
|
3350
|
+
throw mapFsError(err);
|
|
3351
|
+
}
|
|
3352
|
+
try {
|
|
3353
|
+
await handle.write(out);
|
|
3354
|
+
await handle.sync();
|
|
3355
|
+
} finally {
|
|
3356
|
+
await handle.close();
|
|
3357
|
+
}
|
|
3358
|
+
await promises.rename(tmp, path$1).catch(async (err) => {
|
|
3359
|
+
await promises.unlink(tmp).catch(() => {
|
|
3360
|
+
});
|
|
3361
|
+
throw mapFsError(err);
|
|
3362
|
+
});
|
|
3363
|
+
await fsyncDir(path.dirname(path$1));
|
|
3364
|
+
}
|
|
3365
|
+
/**
|
|
3366
|
+
* Serialize callbacks for a given row path so read-modify-write is atomic in-process. `prev.then(fn,
|
|
3367
|
+
* fn)` runs `fn` regardless of whether the predecessor fulfilled or rejected — a conflict on one
|
|
3368
|
+
* writer must not stall the row's chain.
|
|
3369
|
+
*/
|
|
3370
|
+
withRowLock(key, fn) {
|
|
3371
|
+
const prev = this.chain.get(key) ?? Promise.resolve();
|
|
3372
|
+
const result = prev.then(fn, fn);
|
|
3373
|
+
const tail = result.then(
|
|
3374
|
+
() => void 0,
|
|
3375
|
+
() => void 0
|
|
3376
|
+
);
|
|
3377
|
+
this.chain.set(key, tail);
|
|
3378
|
+
void tail.then(() => {
|
|
3379
|
+
if (this.chain.get(key) === tail) this.chain.delete(key);
|
|
3380
|
+
});
|
|
3381
|
+
return result;
|
|
3382
|
+
}
|
|
3383
|
+
};
|
|
3384
|
+
var DEFAULT_MAX_ROW_BYTES2 = 1 * 1024 * 1024;
|
|
3385
|
+
var LocalFsRegistryDriver = class {
|
|
3386
|
+
constructor(root, options = {}) {
|
|
3387
|
+
this.root = root;
|
|
3388
|
+
this.now = options.now ?? (() => Date.now());
|
|
3389
|
+
}
|
|
3390
|
+
root;
|
|
3391
|
+
/** Per-row promise chain — serializes read-modify-write so an in-process CAS never loses an update. */
|
|
3392
|
+
chain = /* @__PURE__ */ new Map();
|
|
3393
|
+
now;
|
|
3394
|
+
capabilities() {
|
|
3395
|
+
return { strongRead: true };
|
|
3396
|
+
}
|
|
3397
|
+
async get(ref) {
|
|
3398
|
+
const env = await this.readRow(registryRowPath(this.root, ref));
|
|
3399
|
+
return env && !env.deleted ? env.record : null;
|
|
3400
|
+
}
|
|
3401
|
+
async create(ref, record) {
|
|
3402
|
+
validateNewRegistryRecord(record);
|
|
3403
|
+
const path = registryRowPath(this.root, ref);
|
|
3404
|
+
return this.withRowLock(path, async () => {
|
|
3405
|
+
const current = await this.readRow(path);
|
|
3406
|
+
if (current !== null && !current.deleted) {
|
|
3407
|
+
throw new WriteConflictError(`registry row already exists for segment ${ref.segment}`);
|
|
3408
|
+
}
|
|
3409
|
+
const counter = current ? registryCounterOf(current.record) + 1 : 0;
|
|
3410
|
+
const token = String(counter);
|
|
3411
|
+
await this.writeRow(path, false, recordFromNew(ref, record, this.now(), token));
|
|
3412
|
+
return { token };
|
|
3413
|
+
});
|
|
3414
|
+
}
|
|
3415
|
+
async compareAndSwap(ref, expected, patch) {
|
|
3416
|
+
validateRegistryPatch(patch);
|
|
3417
|
+
const path = registryRowPath(this.root, ref);
|
|
3418
|
+
return this.withRowLock(path, async () => {
|
|
3419
|
+
const current = await this.readRow(path);
|
|
3420
|
+
if (current === null || current.deleted || current.record.token !== expected) {
|
|
3421
|
+
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
3422
|
+
}
|
|
3423
|
+
const token = String(registryCounterOf(current.record) + 1);
|
|
3424
|
+
await this.writeRow(
|
|
3425
|
+
path,
|
|
3426
|
+
false,
|
|
3427
|
+
applyRegistryPatch(current.record, patch, this.now(), token)
|
|
3428
|
+
);
|
|
3429
|
+
return { token };
|
|
3430
|
+
});
|
|
3431
|
+
}
|
|
3432
|
+
async *list(namespace) {
|
|
3433
|
+
for (const ns of await this.namespaceDirs(namespace)) {
|
|
3434
|
+
const dir = registryDir(this.root, ns);
|
|
3435
|
+
let names;
|
|
3436
|
+
try {
|
|
3437
|
+
names = await promises.readdir(dir);
|
|
3438
|
+
} catch (err) {
|
|
3439
|
+
if (isCode(err, "ENOENT")) continue;
|
|
3440
|
+
throw mapFsError(err);
|
|
3441
|
+
}
|
|
3442
|
+
for (const name of names) {
|
|
3443
|
+
const segment = parseRegistryRow(name);
|
|
3444
|
+
if (segment === null) continue;
|
|
3445
|
+
const env = await this.readRow(registryRowPath(this.root, { namespace: ns, segment }));
|
|
3446
|
+
if (env && !env.deleted) yield env.record;
|
|
3447
|
+
}
|
|
3448
|
+
}
|
|
3449
|
+
}
|
|
3450
|
+
async delete(ref) {
|
|
3451
|
+
const path = registryRowPath(this.root, ref);
|
|
3452
|
+
return this.withRowLock(path, async () => {
|
|
3453
|
+
const current = await this.readRow(path);
|
|
3454
|
+
if (current === null || current.deleted) return;
|
|
3455
|
+
const token = String(registryCounterOf(current.record) + 1);
|
|
3456
|
+
await this.writeRow(path, true, { ...current.record, token, updatedAt: this.now() });
|
|
3457
|
+
});
|
|
3458
|
+
}
|
|
3459
|
+
/** Namespaces to scan: just the one requested, or every namespace dir under the root. */
|
|
3460
|
+
async namespaceDirs(namespace) {
|
|
3461
|
+
if (namespace !== void 0) return [namespace];
|
|
3462
|
+
let entries;
|
|
3463
|
+
try {
|
|
3464
|
+
entries = await promises.readdir(this.root);
|
|
3465
|
+
} catch (err) {
|
|
3466
|
+
if (isCode(err, "ENOENT")) return [];
|
|
3467
|
+
throw mapFsError(err);
|
|
3468
|
+
}
|
|
3469
|
+
return entries.map((e) => e === "_default" ? void 0 : e);
|
|
3470
|
+
}
|
|
3471
|
+
async readRow(path) {
|
|
3472
|
+
let handle;
|
|
3473
|
+
try {
|
|
3474
|
+
handle = await promises.open(path, fs.constants.O_RDONLY | O_NOFOLLOW);
|
|
3475
|
+
} catch (err) {
|
|
3476
|
+
if (isCode(err, "ENOENT") || isCode(err, "ELOOP")) return null;
|
|
3477
|
+
throw mapFsError(err);
|
|
3478
|
+
}
|
|
3479
|
+
try {
|
|
3480
|
+
const { size } = await handle.stat();
|
|
3481
|
+
if (size > DEFAULT_MAX_ROW_BYTES2) {
|
|
3482
|
+
throw new IntegrityError(`registry row ${size}B exceeds cap ${DEFAULT_MAX_ROW_BYTES2}B`);
|
|
3483
|
+
}
|
|
3484
|
+
const text = (await handle.readFile()).toString("utf8");
|
|
3485
|
+
return parseRegistryEnvelope(text, path);
|
|
3486
|
+
} finally {
|
|
3487
|
+
await handle.close();
|
|
3488
|
+
}
|
|
3489
|
+
}
|
|
3490
|
+
async writeRow(path$1, deleted, record) {
|
|
3491
|
+
await promises.mkdir(path.dirname(path$1), { recursive: true });
|
|
3492
|
+
const out = Buffer.from(serializeRegistryEnvelope({ deleted, record }), "utf8");
|
|
3493
|
+
if (out.length > DEFAULT_MAX_ROW_BYTES2) {
|
|
3494
|
+
throw new ValidationError(
|
|
3495
|
+
`registry row ${out.length}B exceeds cap ${DEFAULT_MAX_ROW_BYTES2}B`
|
|
3496
|
+
);
|
|
3497
|
+
}
|
|
3498
|
+
const tmp = `${path$1}.${crypto.randomUUID()}.tmp`;
|
|
3499
|
+
let handle;
|
|
3500
|
+
try {
|
|
3501
|
+
handle = await promises.open(tmp, fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY | O_NOFOLLOW);
|
|
3502
|
+
} catch (err) {
|
|
3503
|
+
throw mapFsError(err);
|
|
3504
|
+
}
|
|
3505
|
+
try {
|
|
3506
|
+
await handle.write(out);
|
|
3507
|
+
await handle.sync();
|
|
3508
|
+
} finally {
|
|
3509
|
+
await handle.close();
|
|
3510
|
+
}
|
|
3511
|
+
await promises.rename(tmp, path$1).catch(async (err) => {
|
|
3512
|
+
await promises.unlink(tmp).catch(() => {
|
|
3513
|
+
});
|
|
3514
|
+
throw mapFsError(err);
|
|
3515
|
+
});
|
|
3516
|
+
await fsyncDir(path.dirname(path$1));
|
|
3517
|
+
}
|
|
3518
|
+
/** Serialize callbacks for a row path so read-modify-write is atomic in-process (mirrors the Warm tier). */
|
|
3519
|
+
withRowLock(key, fn) {
|
|
3520
|
+
const prev = this.chain.get(key) ?? Promise.resolve();
|
|
3521
|
+
const result = prev.then(fn, fn);
|
|
3522
|
+
const tail = result.then(
|
|
3523
|
+
() => void 0,
|
|
3524
|
+
() => void 0
|
|
3525
|
+
);
|
|
3526
|
+
this.chain.set(key, tail);
|
|
3527
|
+
void tail.then(() => {
|
|
3528
|
+
if (this.chain.get(key) === tail) this.chain.delete(key);
|
|
3529
|
+
});
|
|
3530
|
+
return result;
|
|
3531
|
+
}
|
|
3532
|
+
};
|
|
3533
|
+
var KEY_BYTES = 32;
|
|
3534
|
+
var NONCE_BYTES = 12;
|
|
3535
|
+
var TAG_BYTES = 16;
|
|
3536
|
+
var NodeAead = class {
|
|
3537
|
+
key;
|
|
3538
|
+
constructor(key) {
|
|
3539
|
+
if (key.length !== KEY_BYTES)
|
|
3540
|
+
throw new ValidationError(`AEAD key must be ${KEY_BYTES} bytes, got ${key.length}`);
|
|
3541
|
+
this.key = Buffer.from(key);
|
|
3542
|
+
}
|
|
3543
|
+
seal(plaintext, aad) {
|
|
3544
|
+
const nonce = crypto.randomBytes(NONCE_BYTES);
|
|
3545
|
+
const cipher = crypto.createCipheriv("aes-256-gcm", this.key, nonce);
|
|
3546
|
+
cipher.setAAD(aad);
|
|
3547
|
+
const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]);
|
|
3548
|
+
return { nonce, ciphertext, tag: cipher.getAuthTag() };
|
|
3549
|
+
}
|
|
3550
|
+
open(sealed, aad) {
|
|
3551
|
+
if (sealed.nonce.length !== NONCE_BYTES || sealed.tag.length !== TAG_BYTES)
|
|
3552
|
+
throw new IntegrityError("AEAD nonce/tag have the wrong length");
|
|
3553
|
+
const decipher = crypto.createDecipheriv("aes-256-gcm", this.key, sealed.nonce);
|
|
3554
|
+
decipher.setAAD(aad);
|
|
3555
|
+
decipher.setAuthTag(Buffer.from(sealed.tag));
|
|
3556
|
+
try {
|
|
3557
|
+
return Buffer.concat([decipher.update(sealed.ciphertext), decipher.final()]);
|
|
3558
|
+
} catch {
|
|
3559
|
+
throw new IntegrityError(
|
|
3560
|
+
"AEAD authentication failed (wrong key, tampered data, or wrong context)"
|
|
3561
|
+
);
|
|
3562
|
+
}
|
|
3563
|
+
}
|
|
3564
|
+
};
|
|
3565
|
+
function wrapKey(dek, kek, keyId) {
|
|
3566
|
+
const sealed = new NodeAead(kek).seal(dek, new TextEncoder().encode(keyId));
|
|
3567
|
+
return Buffer.concat([sealed.nonce, sealed.ciphertext, sealed.tag]).toString("base64");
|
|
3568
|
+
}
|
|
3569
|
+
function unwrapKey(wrapped, kek, keyId) {
|
|
3570
|
+
const blob = Buffer.from(wrapped, "base64");
|
|
3571
|
+
if (blob.length < NONCE_BYTES + TAG_BYTES)
|
|
3572
|
+
throw new IntegrityError("wrapped DEK is too short to be valid");
|
|
3573
|
+
const nonce = blob.subarray(0, NONCE_BYTES);
|
|
3574
|
+
const tag = blob.subarray(blob.length - TAG_BYTES);
|
|
3575
|
+
const ciphertext = blob.subarray(NONCE_BYTES, blob.length - TAG_BYTES);
|
|
3576
|
+
return new NodeAead(kek).open({ nonce, ciphertext, tag }, new TextEncoder().encode(keyId));
|
|
3577
|
+
}
|
|
3578
|
+
var InProcessKeystore = class {
|
|
3579
|
+
keys;
|
|
3580
|
+
activeKeyId;
|
|
3581
|
+
recoveryKeyId;
|
|
3582
|
+
constructor(options) {
|
|
3583
|
+
const entries = Object.entries(options.keys);
|
|
3584
|
+
if (entries.length === 0) throw new ValidationError("keystore needs at least one KEK");
|
|
3585
|
+
for (const [id, k] of entries) {
|
|
3586
|
+
if (k.length !== KEY_BYTES)
|
|
3587
|
+
throw new ValidationError(`KEK "${id}" must be ${KEY_BYTES} bytes, got ${k.length}`);
|
|
3588
|
+
}
|
|
3589
|
+
if (!(options.activeKeyId in options.keys))
|
|
3590
|
+
throw new ValidationError(`activeKeyId "${options.activeKeyId}" is not in keys`);
|
|
3591
|
+
if (options.recoveryKeyId !== void 0 && !(options.recoveryKeyId in options.keys))
|
|
3592
|
+
throw new ValidationError(`recoveryKeyId "${options.recoveryKeyId}" is not in keys`);
|
|
3593
|
+
this.keys = new Map(entries.map(([id, k]) => [id, Buffer.from(k)]));
|
|
3594
|
+
this.activeKeyId = options.activeKeyId;
|
|
3595
|
+
this.recoveryKeyId = options.recoveryKeyId === options.activeKeyId ? void 0 : options.recoveryKeyId;
|
|
3596
|
+
}
|
|
3597
|
+
// async to satisfy the IKeystore seam (a KMS adapter awaits a network call here); the in-process path is
|
|
3598
|
+
// synchronous, but staying async keeps unwrap failures as rejections, matching a remote keystore.
|
|
3599
|
+
async createDek() {
|
|
3600
|
+
const dek = crypto.randomBytes(KEY_BYTES);
|
|
3601
|
+
const ids = this.recoveryKeyId ? [this.activeKeyId, this.recoveryKeyId] : [this.activeKeyId];
|
|
3602
|
+
const wrapped = ids.map((keyId) => ({
|
|
3603
|
+
keyId,
|
|
3604
|
+
wrapped: wrapKey(dek, this.keys.get(keyId), keyId)
|
|
3605
|
+
}));
|
|
3606
|
+
return { wrapped, aead: new NodeAead(dek) };
|
|
3607
|
+
}
|
|
3608
|
+
// async for the same reason as createDek — a synchronous unwrap failure surfaces as a promise rejection.
|
|
3609
|
+
async openDek(wrapped) {
|
|
3610
|
+
let anyKekHeld = false;
|
|
3611
|
+
let lastUnwrapError;
|
|
3612
|
+
for (const w of wrapped) {
|
|
3613
|
+
const kek = this.keys.get(w.keyId);
|
|
3614
|
+
if (kek === void 0) continue;
|
|
3615
|
+
anyKekHeld = true;
|
|
3616
|
+
try {
|
|
3617
|
+
return new NodeAead(unwrapKey(w.wrapped, kek, w.keyId));
|
|
3618
|
+
} catch (err) {
|
|
3619
|
+
lastUnwrapError = err;
|
|
3620
|
+
}
|
|
3621
|
+
}
|
|
3622
|
+
if (anyKekHeld) throw lastUnwrapError;
|
|
3623
|
+
const have = [...this.keys.keys()].join(", ") || "(none)";
|
|
3624
|
+
const need = wrapped.map((w) => w.keyId).join(", ") || "(none)";
|
|
3625
|
+
throw new KeyUnavailableError(
|
|
3626
|
+
`no held KEK can unwrap this DEK \u2014 have [${have}], wrappings reference [${need}]. Restore the missing KEK (or its recovery KEK), or rebuild the segment from source.`
|
|
3627
|
+
);
|
|
3628
|
+
}
|
|
3629
|
+
};
|
|
3630
|
+
|
|
3631
|
+
// src/core/erasure.ts
|
|
3632
|
+
var MAX_CAS_ATTEMPTS = 8;
|
|
3633
|
+
var MAX_WARM_PASSES = 4;
|
|
3634
|
+
async function destroySegment(ref, deps, options) {
|
|
3635
|
+
if (options.confirmSegment !== ref.segment) {
|
|
3636
|
+
throw new ValidationError(
|
|
3637
|
+
`destroySegment: confirmSegment must equal the segment name "${ref.segment}" (guard against accidental crypto-shred)`
|
|
3638
|
+
);
|
|
3639
|
+
}
|
|
3640
|
+
const result = await shredSegment(ref, deps, options.allowCleartext ?? false);
|
|
3641
|
+
if (result.cryptoShredded) {
|
|
3642
|
+
safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
|
|
3643
|
+
kind: "segment.erase",
|
|
3644
|
+
namespace: ref.namespace,
|
|
3645
|
+
segment: ref.segment
|
|
3646
|
+
});
|
|
3647
|
+
}
|
|
3648
|
+
return result;
|
|
3649
|
+
}
|
|
3650
|
+
async function eraseNamespace(namespace, deps, options) {
|
|
3651
|
+
if (typeof namespace !== "string" || namespace.length === 0) {
|
|
3652
|
+
throw new ValidationError("eraseNamespace: namespace must be a non-empty string");
|
|
3653
|
+
}
|
|
3654
|
+
if (options.confirmNamespace !== namespace) {
|
|
3655
|
+
throw new ValidationError(
|
|
3656
|
+
`eraseNamespace: confirmNamespace must equal the namespace "${namespace}" (guard against accidental erasure)`
|
|
3657
|
+
);
|
|
3658
|
+
}
|
|
3659
|
+
const refs = [];
|
|
3660
|
+
for await (const rec of deps.registry.list(namespace)) {
|
|
3661
|
+
refs.push({ namespace: rec.namespace, segment: rec.segment });
|
|
3662
|
+
}
|
|
3663
|
+
const audit = safeAudit(options.audit ?? NOOP_AUDIT);
|
|
3664
|
+
const destroyed = [];
|
|
3665
|
+
let segmentsShredded = 0;
|
|
3666
|
+
for (const ref of refs) {
|
|
3667
|
+
const result = await shredSegment(ref, deps, options.allowCleartext ?? false);
|
|
3668
|
+
destroyed.push(result);
|
|
3669
|
+
if (result.cryptoShredded) {
|
|
3670
|
+
segmentsShredded += 1;
|
|
3671
|
+
audit.onEvent({ kind: "segment.erase", namespace: ref.namespace, segment: ref.segment });
|
|
3672
|
+
}
|
|
3673
|
+
}
|
|
3674
|
+
audit.onEvent({ kind: "namespace.erase", namespace, segmentsShredded });
|
|
3675
|
+
return { destroyed };
|
|
3676
|
+
}
|
|
3677
|
+
async function shredSegment(ref, deps, allowCleartext) {
|
|
3678
|
+
const base = { segment: ref.segment, namespace: ref.namespace };
|
|
3679
|
+
for (let attempt = 0; attempt < MAX_CAS_ATTEMPTS; attempt++) {
|
|
3680
|
+
const record = await deps.registry.get(ref);
|
|
3681
|
+
if (record === null) {
|
|
3682
|
+
const warmRowsDeleted2 = await eraseWarm(deps.warm, ref);
|
|
3683
|
+
return {
|
|
3684
|
+
...base,
|
|
3685
|
+
destroyed: false,
|
|
3686
|
+
cryptoShredded: false,
|
|
3687
|
+
reason: "absent",
|
|
3688
|
+
warmRowsDeleted: warmRowsDeleted2
|
|
3689
|
+
};
|
|
3690
|
+
}
|
|
3691
|
+
if (record.status === "destroyed") {
|
|
3692
|
+
const warmRowsDeleted2 = await eraseWarm(deps.warm, ref);
|
|
3693
|
+
return {
|
|
3694
|
+
...base,
|
|
3695
|
+
destroyed: true,
|
|
3696
|
+
cryptoShredded: false,
|
|
3697
|
+
reason: "already",
|
|
3698
|
+
warmRowsDeleted: warmRowsDeleted2
|
|
3699
|
+
};
|
|
3700
|
+
}
|
|
3701
|
+
const encrypted = record.wrappedDeks !== void 0 && record.wrappedDeks.length > 0;
|
|
3702
|
+
if (!encrypted && !allowCleartext) {
|
|
3703
|
+
return {
|
|
3704
|
+
...base,
|
|
3705
|
+
destroyed: false,
|
|
3706
|
+
cryptoShredded: false,
|
|
3707
|
+
reason: "cleartext",
|
|
3708
|
+
warmRowsDeleted: 0
|
|
3709
|
+
};
|
|
3710
|
+
}
|
|
3711
|
+
const warmRowsDeleted = await eraseWarm(deps.warm, ref);
|
|
3712
|
+
try {
|
|
3713
|
+
await deps.registry.compareAndSwap(ref, record.token, {
|
|
3714
|
+
status: "destroyed",
|
|
3715
|
+
wrappedDeks: void 0,
|
|
3716
|
+
// ← the crypto-shred: the only copy of the DEK wrappings is gone
|
|
3717
|
+
keyId: void 0,
|
|
3718
|
+
leaseOwner: void 0,
|
|
3719
|
+
leaseExpiresAt: void 0
|
|
3720
|
+
});
|
|
3721
|
+
return { ...base, destroyed: true, cryptoShredded: encrypted, warmRowsDeleted };
|
|
3722
|
+
} catch (err) {
|
|
3723
|
+
if (!isWriteConflictError(err)) throw err;
|
|
3724
|
+
}
|
|
3725
|
+
}
|
|
3726
|
+
throw new WriteConflictError(`destroySegment: contention shredding "${ref.segment}" \u2014 retry`);
|
|
3727
|
+
}
|
|
3728
|
+
async function eraseWarm(warm, ref) {
|
|
3729
|
+
let deleted = 0;
|
|
3730
|
+
for (let pass = 0; pass < MAX_WARM_PASSES; pass++) {
|
|
3731
|
+
const rows = [];
|
|
3732
|
+
for await (const row of warm.listChunks(ref))
|
|
3733
|
+
rows.push({ chunkKey: row.chunkKey, token: row.token });
|
|
3734
|
+
if (rows.length === 0) break;
|
|
3735
|
+
let conflicts = 0;
|
|
3736
|
+
for (const { chunkKey, token } of rows) {
|
|
3737
|
+
const chunkRef = { namespace: ref.namespace, segment: ref.segment, chunkKey };
|
|
3738
|
+
try {
|
|
3739
|
+
await warm.deleteConditional(chunkRef, token);
|
|
3740
|
+
deleted += 1;
|
|
3741
|
+
} catch (err) {
|
|
3742
|
+
if (!isWriteConflictError(err)) throw err;
|
|
3743
|
+
conflicts += 1;
|
|
3744
|
+
}
|
|
3745
|
+
}
|
|
3746
|
+
if (conflicts === 0) break;
|
|
3747
|
+
}
|
|
3748
|
+
return deleted;
|
|
3749
|
+
}
|
|
3750
|
+
|
|
3751
|
+
// src/core/consistency.ts
|
|
3752
|
+
var DEFAULT_CHECK_CONCURRENCY = 8;
|
|
3753
|
+
async function listGenerations(cold, ref) {
|
|
3754
|
+
const present = /* @__PURE__ */ new Set();
|
|
3755
|
+
for await (const key of cold.list(ref)) present.add(key.generation);
|
|
3756
|
+
return present;
|
|
3757
|
+
}
|
|
3758
|
+
async function runConsistencyCheck(deps, options = {}) {
|
|
3759
|
+
const concurrency = options.concurrency ?? DEFAULT_CHECK_CONCURRENCY;
|
|
3760
|
+
if (!Number.isInteger(concurrency) || concurrency < 1) {
|
|
3761
|
+
throw new ValidationError(`concurrency must be a positive integer; got ${concurrency}`);
|
|
3762
|
+
}
|
|
3763
|
+
const recs = [];
|
|
3764
|
+
for await (const rec of deps.registry.list(options.namespace)) recs.push(rec);
|
|
3765
|
+
const results = await mapWithConcurrency(recs, concurrency, async (rec) => {
|
|
3766
|
+
if (rec.status === "destroyed") return { kind: "ok" };
|
|
3767
|
+
const ref = { segment: rec.segment, namespace: rec.namespace };
|
|
3768
|
+
try {
|
|
3769
|
+
const live = await deps.registry.get(ref);
|
|
3770
|
+
if (!live || live.status === "destroyed") return { kind: "ok" };
|
|
3771
|
+
const present = await listGenerations(deps.cold, ref);
|
|
3772
|
+
if (present.has(live.currentGen)) return { kind: "ok" };
|
|
3773
|
+
return {
|
|
3774
|
+
kind: "issue",
|
|
3775
|
+
issue: {
|
|
3776
|
+
segment: rec.segment,
|
|
3777
|
+
namespace: rec.namespace,
|
|
3778
|
+
currentGen: live.currentGen,
|
|
3779
|
+
issue: "missing-cold-generation"
|
|
3780
|
+
}
|
|
3781
|
+
};
|
|
3782
|
+
} catch (error) {
|
|
3783
|
+
return {
|
|
3784
|
+
kind: "error",
|
|
3785
|
+
error: {
|
|
3786
|
+
segment: rec.segment,
|
|
3787
|
+
namespace: rec.namespace,
|
|
3788
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3789
|
+
}
|
|
3790
|
+
};
|
|
3791
|
+
}
|
|
3792
|
+
});
|
|
3793
|
+
const inconsistent = [];
|
|
3794
|
+
const errored = [];
|
|
3795
|
+
for (const r of results) {
|
|
3796
|
+
if (r.kind === "issue") inconsistent.push(r.issue);
|
|
3797
|
+
else if (r.kind === "error") errored.push(r.error);
|
|
3798
|
+
}
|
|
3799
|
+
return { checked: recs.length, inconsistent, errored };
|
|
3800
|
+
}
|
|
3801
|
+
|
|
3802
|
+
// src/drivers/retry/retrying-drivers.ts
|
|
3803
|
+
function toRetry(opts) {
|
|
3804
|
+
return {
|
|
3805
|
+
policy: opts.policy ?? DEFAULT_RETRY_POLICY,
|
|
3806
|
+
deps: {
|
|
3807
|
+
clock: opts.clock,
|
|
3808
|
+
rng: opts.rng,
|
|
3809
|
+
isRetryable: opts.isRetryable ?? isTransient,
|
|
3810
|
+
onRetry: opts.onRetry
|
|
3811
|
+
}
|
|
3812
|
+
};
|
|
3813
|
+
}
|
|
3814
|
+
var RetryingWarmDriver = class {
|
|
3815
|
+
inner;
|
|
3816
|
+
policy;
|
|
3817
|
+
deps;
|
|
3818
|
+
constructor(inner, opts) {
|
|
3819
|
+
this.inner = inner;
|
|
3820
|
+
const r = toRetry(opts);
|
|
3821
|
+
this.policy = r.policy;
|
|
3822
|
+
this.deps = r.deps;
|
|
3823
|
+
}
|
|
3824
|
+
get(ref, opts) {
|
|
3825
|
+
return withRetry(() => this.inner.get(ref, opts), this.policy, this.deps);
|
|
3826
|
+
}
|
|
3827
|
+
putConditional(ref, bytes, expected) {
|
|
3828
|
+
return withRetry(() => this.inner.putConditional(ref, bytes, expected), this.policy, this.deps);
|
|
3829
|
+
}
|
|
3830
|
+
deleteConditional(ref, expected) {
|
|
3831
|
+
return withRetry(() => this.inner.deleteConditional(ref, expected), this.policy, this.deps);
|
|
3832
|
+
}
|
|
3833
|
+
async *listChunks(ref, opts) {
|
|
3834
|
+
const rows = await withRetry(
|
|
3835
|
+
async () => {
|
|
3836
|
+
const out = [];
|
|
3837
|
+
for await (const row of this.inner.listChunks(ref, opts)) out.push(row);
|
|
3838
|
+
return out;
|
|
3839
|
+
},
|
|
3840
|
+
this.policy,
|
|
3841
|
+
this.deps
|
|
3842
|
+
);
|
|
3843
|
+
yield* rows;
|
|
3844
|
+
}
|
|
3845
|
+
};
|
|
3846
|
+
var RetryingColdChunkSource = class {
|
|
3847
|
+
inner;
|
|
3848
|
+
policy;
|
|
3849
|
+
deps;
|
|
3850
|
+
/** Present only when the inner source supports it — so capability detection stays honest. */
|
|
3851
|
+
sizeOf;
|
|
3852
|
+
cardinalities;
|
|
3853
|
+
currentGeneration;
|
|
3854
|
+
constructor(inner, opts) {
|
|
3855
|
+
this.inner = inner;
|
|
3856
|
+
const r = toRetry(opts);
|
|
3857
|
+
this.policy = r.policy;
|
|
3858
|
+
this.deps = r.deps;
|
|
3859
|
+
const innerSizeOf = inner.sizeOf;
|
|
3860
|
+
if (innerSizeOf) {
|
|
3861
|
+
this.sizeOf = (ref) => withRetry(() => innerSizeOf.call(inner, ref), this.policy, this.deps);
|
|
3862
|
+
}
|
|
3863
|
+
const innerCardinalities = inner.cardinalities;
|
|
3864
|
+
if (innerCardinalities) {
|
|
3865
|
+
this.cardinalities = (ref) => withRetry(() => innerCardinalities.call(inner, ref), this.policy, this.deps);
|
|
3866
|
+
}
|
|
3867
|
+
const innerCurrentGeneration = inner.currentGeneration;
|
|
3868
|
+
if (innerCurrentGeneration) {
|
|
3869
|
+
this.currentGeneration = (ref) => withRetry(() => innerCurrentGeneration.call(inner, ref), this.policy, this.deps);
|
|
3870
|
+
}
|
|
3871
|
+
}
|
|
3872
|
+
getChunk(ref) {
|
|
3873
|
+
return withRetry(() => this.inner.getChunk(ref), this.policy, this.deps);
|
|
3874
|
+
}
|
|
3875
|
+
listChunkKeys(ref) {
|
|
3876
|
+
return withRetry(() => this.inner.listChunkKeys(ref), this.policy, this.deps);
|
|
3877
|
+
}
|
|
3878
|
+
};
|
|
3879
|
+
var RetryingColdDriver = class {
|
|
3880
|
+
inner;
|
|
3881
|
+
policy;
|
|
3882
|
+
deps;
|
|
3883
|
+
constructor(inner, opts) {
|
|
3884
|
+
this.inner = inner;
|
|
3885
|
+
const r = toRetry(opts);
|
|
3886
|
+
this.policy = r.policy;
|
|
3887
|
+
this.deps = r.deps;
|
|
3888
|
+
}
|
|
3889
|
+
capabilities() {
|
|
3890
|
+
return this.inner.capabilities();
|
|
3891
|
+
}
|
|
3892
|
+
putImmutable(key, write) {
|
|
3893
|
+
return withRetry(() => this.inner.putImmutable(key, write), this.policy, this.deps);
|
|
3894
|
+
}
|
|
3895
|
+
getRange(key, offset, length) {
|
|
3896
|
+
return withRetry(() => this.inner.getRange(key, offset, length), this.policy, this.deps);
|
|
3897
|
+
}
|
|
3898
|
+
getTail(key, maxBytes) {
|
|
3899
|
+
return withRetry(() => this.inner.getTail(key, maxBytes), this.policy, this.deps);
|
|
3900
|
+
}
|
|
3901
|
+
delete(key) {
|
|
3902
|
+
return withRetry(() => this.inner.delete(key), this.policy, this.deps);
|
|
3903
|
+
}
|
|
3904
|
+
async *list(ref) {
|
|
3905
|
+
const keys = await withRetry(
|
|
3906
|
+
async () => {
|
|
3907
|
+
const out = [];
|
|
3908
|
+
for await (const k of this.inner.list(ref)) out.push(k);
|
|
3909
|
+
return out;
|
|
3910
|
+
},
|
|
3911
|
+
this.policy,
|
|
3912
|
+
this.deps
|
|
3913
|
+
);
|
|
3914
|
+
yield* keys;
|
|
3915
|
+
}
|
|
3916
|
+
};
|
|
3917
|
+
var RetryingRegistryDriver = class {
|
|
3918
|
+
inner;
|
|
3919
|
+
policy;
|
|
3920
|
+
deps;
|
|
3921
|
+
constructor(inner, opts) {
|
|
3922
|
+
this.inner = inner;
|
|
3923
|
+
const r = toRetry(opts);
|
|
3924
|
+
this.policy = r.policy;
|
|
3925
|
+
this.deps = r.deps;
|
|
3926
|
+
}
|
|
3927
|
+
capabilities() {
|
|
3928
|
+
return this.inner.capabilities();
|
|
3929
|
+
}
|
|
3930
|
+
get(ref) {
|
|
3931
|
+
return withRetry(() => this.inner.get(ref), this.policy, this.deps);
|
|
3932
|
+
}
|
|
3933
|
+
create(ref, record) {
|
|
3934
|
+
return withRetry(() => this.inner.create(ref, record), this.policy, this.deps);
|
|
3935
|
+
}
|
|
3936
|
+
compareAndSwap(ref, expected, patch) {
|
|
3937
|
+
return withRetry(() => this.inner.compareAndSwap(ref, expected, patch), this.policy, this.deps);
|
|
3938
|
+
}
|
|
3939
|
+
async *list(namespace) {
|
|
3940
|
+
const records = await withRetry(
|
|
3941
|
+
async () => {
|
|
3942
|
+
const out = [];
|
|
3943
|
+
for await (const r of this.inner.list(namespace)) out.push(r);
|
|
3944
|
+
return out;
|
|
3945
|
+
},
|
|
3946
|
+
this.policy,
|
|
3947
|
+
this.deps
|
|
3948
|
+
);
|
|
3949
|
+
yield* records;
|
|
3950
|
+
}
|
|
3951
|
+
delete(ref) {
|
|
3952
|
+
return withRetry(() => this.inner.delete(ref), this.policy, this.deps);
|
|
3953
|
+
}
|
|
3954
|
+
};
|
|
3955
|
+
|
|
3956
|
+
exports.AWS_US_EAST_1_ONDEMAND = AWS_US_EAST_1_ONDEMAND;
|
|
3957
|
+
exports.BoundedLru = BoundedLru;
|
|
3958
|
+
exports.BudgetExceededError = BudgetExceededError;
|
|
3959
|
+
exports.BufferReader = BufferReader;
|
|
3960
|
+
exports.BufferSink = BufferSink;
|
|
3961
|
+
exports.CapabilityError = CapabilityError;
|
|
3962
|
+
exports.CloudRoaringError = CloudRoaringError;
|
|
3963
|
+
exports.CountingMetricsSink = CountingMetricsSink;
|
|
3964
|
+
exports.CrbmColdChunkSource = CrbmColdChunkSource;
|
|
3965
|
+
exports.CrbmReader = CrbmReader;
|
|
3966
|
+
exports.CrbmWriter = CrbmWriter;
|
|
3967
|
+
exports.DEFAULT_BUDGET = DEFAULT_BUDGET;
|
|
3968
|
+
exports.DEFAULT_OCC_BACKOFF = DEFAULT_OCC_BACKOFF;
|
|
3969
|
+
exports.DEFAULT_PRICING = DEFAULT_PRICING;
|
|
3970
|
+
exports.DEFAULT_RETRY_POLICY = DEFAULT_RETRY_POLICY;
|
|
3971
|
+
exports.InProcessKeystore = InProcessKeystore;
|
|
3972
|
+
exports.IntegrityError = IntegrityError;
|
|
3973
|
+
exports.KeyUnavailableError = KeyUnavailableError;
|
|
3974
|
+
exports.LocalFsColdDriver = LocalFsColdDriver;
|
|
3975
|
+
exports.LocalFsRegistryDriver = LocalFsRegistryDriver;
|
|
3976
|
+
exports.LocalFsWarmDriver = LocalFsWarmDriver;
|
|
3977
|
+
exports.MemoryColdChunkSource = MemoryColdChunkSource;
|
|
3978
|
+
exports.MemoryColdDriver = MemoryColdDriver;
|
|
3979
|
+
exports.MemoryRegistryDriver = MemoryRegistryDriver;
|
|
3980
|
+
exports.MemoryWarmDriver = MemoryWarmDriver;
|
|
3981
|
+
exports.NOOP_AUDIT = NOOP_AUDIT;
|
|
3982
|
+
exports.NOOP_METRICS = NOOP_METRICS;
|
|
3983
|
+
exports.NO_ROW = NO_ROW;
|
|
3984
|
+
exports.NodeAead = NodeAead;
|
|
3985
|
+
exports.NotFoundError = NotFoundError;
|
|
3986
|
+
exports.RecordingAuditSink = RecordingAuditSink;
|
|
3987
|
+
exports.RetryingColdChunkSource = RetryingColdChunkSource;
|
|
3988
|
+
exports.RetryingColdDriver = RetryingColdDriver;
|
|
3989
|
+
exports.RetryingRegistryDriver = RetryingRegistryDriver;
|
|
3990
|
+
exports.RetryingWarmDriver = RetryingWarmDriver;
|
|
3991
|
+
exports.SegmentEngine = SegmentEngine;
|
|
3992
|
+
exports.TimeoutError = TimeoutError;
|
|
3993
|
+
exports.TransientError = TransientError;
|
|
3994
|
+
exports.UnsupportedError = UnsupportedError;
|
|
3995
|
+
exports.ValidationError = ValidationError;
|
|
3996
|
+
exports.WriteConflictError = WriteConflictError;
|
|
3997
|
+
exports.aadFor = aadFor;
|
|
3998
|
+
exports.bulkLoadCrbmGeneration = bulkLoadCrbmGeneration;
|
|
3999
|
+
exports.checkBudget = checkBudget;
|
|
4000
|
+
exports.chunkRefKey = chunkRefKey;
|
|
4001
|
+
exports.compactSegment = compactSegment;
|
|
4002
|
+
exports.destroySegment = destroySegment;
|
|
4003
|
+
exports.eraseNamespace = eraseNamespace;
|
|
4004
|
+
exports.estimateCost = estimateCost;
|
|
4005
|
+
exports.findCompactable = findCompactable;
|
|
4006
|
+
exports.gcOrphanGenerations = gcOrphanGenerations;
|
|
4007
|
+
exports.groundedReport = groundedReport;
|
|
4008
|
+
exports.isCloudRoaringError = isCloudRoaringError;
|
|
4009
|
+
exports.isIntegrityError = isIntegrityError;
|
|
4010
|
+
exports.isNotFoundError = isNotFoundError;
|
|
4011
|
+
exports.isTransient = isTransient;
|
|
4012
|
+
exports.isTransientError = isTransientError;
|
|
4013
|
+
exports.isValidationError = isValidationError;
|
|
4014
|
+
exports.isWriteConflictError = isWriteConflictError;
|
|
4015
|
+
exports.joinId = joinId;
|
|
4016
|
+
exports.mapWithConcurrency = mapWithConcurrency;
|
|
4017
|
+
exports.publishGeneration = publishGeneration;
|
|
4018
|
+
exports.resolveBudget = resolveBudget;
|
|
4019
|
+
exports.resolvePerOpBudget = resolvePerOpBudget;
|
|
4020
|
+
exports.runCompactionCycle = runCompactionCycle;
|
|
4021
|
+
exports.runConsistencyCheck = runConsistencyCheck;
|
|
4022
|
+
exports.runExport = runExport;
|
|
4023
|
+
exports.safeMetrics = safeMetrics;
|
|
4024
|
+
exports.segmentKey = segmentKey;
|
|
4025
|
+
exports.splitId = splitId;
|
|
4026
|
+
exports.validateCompactionOptions = validateCompactionOptions;
|
|
4027
|
+
exports.validateSegmentRef = validateSegmentRef;
|
|
4028
|
+
exports.withRetry = withRetry;
|
|
4029
|
+
exports.writeCrbmGeneration = writeCrbmGeneration;
|
|
4030
|
+
//# sourceMappingURL=index.cjs.map
|
|
4031
|
+
//# sourceMappingURL=index.cjs.map
|