@cloudbitmaps/core 0.23.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -9
- package/dist/{chunk-MVUD4R72.js → chunk-2B2QF4FU.js} +6 -1
- package/dist/chunk-2B2QF4FU.js.map +7 -0
- package/dist/{chunk-7GT632UF.js → chunk-45ML6YU4.js} +2 -2
- package/dist/{chunk-CGILDXGP.js → chunk-BUDAVGBI.js} +2 -2
- package/dist/{chunk-GKHERVVQ.js → chunk-WPER7R2R.js} +2 -2
- package/dist/core/check.d.ts +2 -1
- package/dist/core/codec.d.ts +16 -6
- package/dist/core/cooperative.d.ts +9 -6
- package/dist/core/crbm/reader.d.ts +1 -1
- package/dist/core/engine.d.ts +1 -1
- package/dist/core/erase-id.d.ts +1 -1
- package/dist/core/load-input.d.ts +10 -4
- package/dist/core/load.d.ts +2 -2
- package/dist/driver-kit.js +3 -3
- package/dist/index.d.ts +2 -2
- package/dist/index.js +135 -82
- package/dist/index.js.map +2 -2
- package/dist/node.js +3 -3
- package/dist/node.js.map +1 -1
- package/package.json +1 -2
- package/dist/chunk-MVUD4R72.js.map +0 -7
- /package/dist/{chunk-7GT632UF.js.map → chunk-45ML6YU4.js.map} +0 -0
- /package/dist/{chunk-CGILDXGP.js.map → chunk-BUDAVGBI.js.map} +0 -0
- /package/dist/{chunk-GKHERVVQ.js.map → chunk-WPER7R2R.js.map} +0 -0
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
boundedAll,
|
|
3
3
|
mapWithConcurrency
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-BUDAVGBI.js";
|
|
5
5
|
import {
|
|
6
6
|
BufferReader,
|
|
7
7
|
KeptSegmentKeys,
|
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
segmentKey,
|
|
10
10
|
segmentPrefix,
|
|
11
11
|
shardOf
|
|
12
|
-
} from "./chunk-
|
|
12
|
+
} from "./chunk-WPER7R2R.js";
|
|
13
13
|
import {
|
|
14
14
|
BudgetExceededError,
|
|
15
15
|
CHUNK_COUNT,
|
|
@@ -99,6 +99,7 @@ import {
|
|
|
99
99
|
rollbackKnown,
|
|
100
100
|
sameIncarnation,
|
|
101
101
|
sameKeys,
|
|
102
|
+
sameKnownGens,
|
|
102
103
|
sizeOfFingerprint,
|
|
103
104
|
sizePart,
|
|
104
105
|
splitId,
|
|
@@ -115,7 +116,7 @@ import {
|
|
|
115
116
|
validateUserRef,
|
|
116
117
|
windowOf,
|
|
117
118
|
withRetry
|
|
118
|
-
} from "./chunk-
|
|
119
|
+
} from "./chunk-2B2QF4FU.js";
|
|
119
120
|
|
|
120
121
|
// src/core/budget.ts
|
|
121
122
|
var DEFAULT_BUDGET = { maxRequests: 1e6 };
|
|
@@ -182,7 +183,7 @@ async function refuseAbsent(op, candidates, limit, exists, nameOf) {
|
|
|
182
183
|
// src/core/chunk-checks.ts
|
|
183
184
|
function assertChunkKeyInRange(k) {
|
|
184
185
|
if (!Number.isInteger(k) || k < 0 || k >= CHUNK_COUNT) {
|
|
185
|
-
throw new IntegrityError(
|
|
186
|
+
throw new IntegrityError("a chunk key from a tier is out of range");
|
|
186
187
|
}
|
|
187
188
|
}
|
|
188
189
|
function assertChunkCardinalityInRange(c) {
|
|
@@ -205,22 +206,22 @@ function checkedChunkKeys(listed2) {
|
|
|
205
206
|
keys.sort((a, b) => a - b);
|
|
206
207
|
for (let i = 1; i < keys.length; i++) {
|
|
207
208
|
if (keys[i] === keys[i - 1]) {
|
|
208
|
-
throw new IntegrityError(
|
|
209
|
+
throw new IntegrityError("a chunk key from a tier is listed twice");
|
|
209
210
|
}
|
|
210
211
|
}
|
|
211
212
|
return keys;
|
|
212
213
|
}
|
|
213
|
-
function assertChunkPayloadInRange(bitmap
|
|
214
|
+
function assertChunkPayloadInRange(bitmap) {
|
|
214
215
|
const max = bitmap.maximum?.();
|
|
215
216
|
if (max !== void 0 && max > MAX_REMAINDER) {
|
|
216
217
|
throw new IntegrityError(
|
|
217
|
-
`chunk
|
|
218
|
+
`a chunk payload holds a value outside the 16-bit remainder range [0, ${MAX_REMAINDER}] (a container key other than 0) \u2014 the stored object is corrupt or was not written by this codec`
|
|
218
219
|
);
|
|
219
220
|
}
|
|
220
221
|
}
|
|
221
|
-
function decodeChunkBytes(codec, bytes,
|
|
222
|
+
function decodeChunkBytes(codec, bytes, maxBitmapBytes) {
|
|
222
223
|
const bitmap = codec.safeDeserialize(bytes, maxBitmapBytes);
|
|
223
|
-
assertChunkPayloadInRange(bitmap
|
|
224
|
+
assertChunkPayloadInRange(bitmap);
|
|
224
225
|
return bitmap;
|
|
225
226
|
}
|
|
226
227
|
|
|
@@ -237,12 +238,10 @@ var ChunkStream = class {
|
|
|
237
238
|
return this.iterator.next().then(
|
|
238
239
|
(step) => {
|
|
239
240
|
if (step.done === true) {
|
|
240
|
-
throw this.failure?.error ?? new IntegrityError(
|
|
241
|
+
throw this.failure?.error ?? new IntegrityError("chunk stream ended before a chunk it was opened over");
|
|
241
242
|
}
|
|
242
243
|
if (step.value.key !== key) {
|
|
243
|
-
throw new IntegrityError(
|
|
244
|
-
`chunk stream answered chunk ${step.value.key} where ${key} was due`
|
|
245
|
-
);
|
|
244
|
+
throw new IntegrityError("chunk stream answered a chunk other than the one due");
|
|
246
245
|
}
|
|
247
246
|
return step.value;
|
|
248
247
|
},
|
|
@@ -758,6 +757,23 @@ function chunkIds(chunk, chunkKey, w) {
|
|
|
758
757
|
for (let i = 0; i < rem.length; i++) rem[i] = base + (rem[i] & MAX_REMAINDER);
|
|
759
758
|
return rem;
|
|
760
759
|
}
|
|
760
|
+
function smallestFirst(chunks) {
|
|
761
|
+
const out = chunks.slice();
|
|
762
|
+
const sizes = out.map((c) => c.size);
|
|
763
|
+
for (let i = 1; i < out.length; i++) {
|
|
764
|
+
const c = out[i];
|
|
765
|
+
const size = sizes[i];
|
|
766
|
+
let j = i - 1;
|
|
767
|
+
while (j >= 0 && sizes[j] > size) {
|
|
768
|
+
out[j + 1] = out[j];
|
|
769
|
+
sizes[j + 1] = sizes[j];
|
|
770
|
+
j--;
|
|
771
|
+
}
|
|
772
|
+
out[j + 1] = c;
|
|
773
|
+
sizes[j + 1] = size;
|
|
774
|
+
}
|
|
775
|
+
return out;
|
|
776
|
+
}
|
|
761
777
|
function alignKeys(operands, mode) {
|
|
762
778
|
const candidates = [];
|
|
763
779
|
if (mode === "all") {
|
|
@@ -1183,9 +1199,7 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1183
1199
|
const expected = counts.get(chunkKey);
|
|
1184
1200
|
if (chunk === null || chunk.size !== expected) {
|
|
1185
1201
|
const got = chunk === null ? "no bytes" : `${chunk.size} ids`;
|
|
1186
|
-
throw new IntegrityError(
|
|
1187
|
-
`chunk ${chunkKey} holds ${got}, but the object's index says ${expected}`
|
|
1188
|
-
);
|
|
1202
|
+
throw new IntegrityError(`a chunk holds ${got}, but the object's index says ${expected}`);
|
|
1189
1203
|
}
|
|
1190
1204
|
return chunk.toUint32Array ? chunk.toUint32Array() : Uint32Array.from(chunk);
|
|
1191
1205
|
}
|
|
@@ -1504,7 +1518,7 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1504
1518
|
* The AND (or OR) of one chunk key across all operands, minus the excludes that hold it, or `null` if empty.
|
|
1505
1519
|
* Operand chunks are fetched **in parallel** (the spec's parallel byte-range reads).
|
|
1506
1520
|
*
|
|
1507
|
-
* The accumulator is a **clone** of
|
|
1521
|
+
* The accumulator is a **clone** of one operand's chunk (the first, or for a codec that asks the one with the fewest values), and that clone is load-bearing: every other
|
|
1508
1522
|
* bitmap here is the cached, shared Storage instance and must never be mutated, or the cache is poisoned for
|
|
1509
1523
|
* every later reader. The remaining operands and the excludes are only *read* by the in-place ops, so they are
|
|
1510
1524
|
* used as-is — one clone per key, not one per operand.
|
|
@@ -1518,10 +1532,11 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1518
1532
|
const fetched = await Promise.all(reads.map((op) => this.operandChunk(op, chunkKey, moves)));
|
|
1519
1533
|
const chunks = together ? fetched.slice(0, present.length) : fetched;
|
|
1520
1534
|
if (mode === "all" && chunks.some((c) => c === null)) return null;
|
|
1521
|
-
const
|
|
1535
|
+
const ordered = mode === "all" && chunks.length > 2 && this.codec.intersectSmallestFirst === true ? smallestFirst(chunks) : chunks;
|
|
1536
|
+
const first = ordered[0];
|
|
1522
1537
|
const acc = first ? first.clone() : this.codec.empty();
|
|
1523
|
-
for (let i = 1; i <
|
|
1524
|
-
const other =
|
|
1538
|
+
for (let i = 1; i < ordered.length; i++) {
|
|
1539
|
+
const other = ordered[i];
|
|
1525
1540
|
if (other === null || other === void 0) continue;
|
|
1526
1541
|
if (mode === "all") {
|
|
1527
1542
|
acc.andInPlace(other);
|
|
@@ -1852,7 +1867,7 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1852
1867
|
streamed.moves.moved = true;
|
|
1853
1868
|
}
|
|
1854
1869
|
if (read.bytes === null) return null;
|
|
1855
|
-
const bitmap = decodeChunkBytes(this.codec, read.bytes,
|
|
1870
|
+
const bitmap = decodeChunkBytes(this.codec, read.bytes, this.maxBitmapBytes);
|
|
1856
1871
|
if (this.cache && !streamed.invalidated && !this.scope.invalidatedSince(streamed.prefix, streamed.openedAt)) {
|
|
1857
1872
|
const version = this.streamedVersion(streamed.gen, read.version);
|
|
1858
1873
|
if (version !== null) {
|
|
@@ -2065,7 +2080,7 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
2065
2080
|
}
|
|
2066
2081
|
}
|
|
2067
2082
|
if (!bytes) return null;
|
|
2068
|
-
const bitmap = decodeChunkBytes(this.codec, bytes,
|
|
2083
|
+
const bitmap = decodeChunkBytes(this.codec, bytes, this.maxBitmapBytes);
|
|
2069
2084
|
if (key !== "" && this.scope.isOpen(cacheKey, token)) this.cache?.set(key, bitmap);
|
|
2070
2085
|
return bitmap;
|
|
2071
2086
|
}
|
|
@@ -2301,6 +2316,22 @@ function requireCodec(codec, api) {
|
|
|
2301
2316
|
// src/export/index.ts
|
|
2302
2317
|
var UTF8 = new TextEncoder();
|
|
2303
2318
|
var DEFAULT_NDJSON_BATCH_BYTES = 64 * 1024;
|
|
2319
|
+
var EXPORT_BATCH = 1 << 16;
|
|
2320
|
+
async function addAll(bitmap, ids) {
|
|
2321
|
+
const batch = new Uint32Array(EXPORT_BATCH);
|
|
2322
|
+
let held = 0;
|
|
2323
|
+
let count = 0;
|
|
2324
|
+
for await (const id of ids) {
|
|
2325
|
+
batch[held++] = id;
|
|
2326
|
+
count += 1;
|
|
2327
|
+
if (held === EXPORT_BATCH) {
|
|
2328
|
+
bitmap.addMany(batch);
|
|
2329
|
+
held = 0;
|
|
2330
|
+
}
|
|
2331
|
+
}
|
|
2332
|
+
if (held > 0) bitmap.addMany(batch.subarray(0, held));
|
|
2333
|
+
return count;
|
|
2334
|
+
}
|
|
2304
2335
|
var errMessage = (err) => err instanceof Error ? err.message : String(err);
|
|
2305
2336
|
async function runExport(reader, registry, sink, options = {}) {
|
|
2306
2337
|
if (sink === null || typeof sink !== "object" || typeof sink.open !== "function") {
|
|
@@ -2360,10 +2391,7 @@ async function runExport(reader, registry, sink, options = {}) {
|
|
|
2360
2391
|
writer = await sink.open(ref, ext);
|
|
2361
2392
|
if (format === "roaring") {
|
|
2362
2393
|
const bm = requireCodec(options.codec, "runExport({ format: 'roaring' })").empty();
|
|
2363
|
-
|
|
2364
|
-
bm.add(id);
|
|
2365
|
-
count += 1;
|
|
2366
|
-
}
|
|
2394
|
+
count = await addAll(bm, ids);
|
|
2367
2395
|
if (count === 0 && await rowIsGone(ref)) return await skipGone(ref, writer);
|
|
2368
2396
|
const out = bm.serialize();
|
|
2369
2397
|
await writer.write(out);
|
|
@@ -2513,7 +2541,7 @@ function* planChunkReads(extents, region) {
|
|
|
2513
2541
|
for (const extent of extents) {
|
|
2514
2542
|
const extentEnd = extent.offset + extent.length;
|
|
2515
2543
|
if (!(Number.isSafeInteger(extent.offset) && Number.isSafeInteger(extent.length) && extent.length >= 1 && extent.offset >= previousEnd && extentEnd <= region.end)) {
|
|
2516
|
-
throw crbmIntegrity(
|
|
2544
|
+
throw crbmIntegrity("a chunk is not inside the chunk region, in order");
|
|
2517
2545
|
}
|
|
2518
2546
|
previousEnd = extentEnd;
|
|
2519
2547
|
if (chunks.length > 0 && extent.offset - end <= MAX_COALESCE_GAP_BYTES && extentEnd - start <= MAX_COALESCED_READ_BYTES) {
|
|
@@ -2537,7 +2565,7 @@ function* planChunkReads(extents, region) {
|
|
|
2537
2565
|
var U32_CEIL = U32_MAX + 1;
|
|
2538
2566
|
function varintLength(value) {
|
|
2539
2567
|
if (!Number.isInteger(value) || value < 0 || value >= U32_CEIL) {
|
|
2540
|
-
throw new RangeError(
|
|
2568
|
+
throw new RangeError("varint value must be a u32");
|
|
2541
2569
|
}
|
|
2542
2570
|
if (value < 128) return 1;
|
|
2543
2571
|
if (value < 16384) return 2;
|
|
@@ -2974,7 +3002,7 @@ var CrbmReader = class _CrbmReader {
|
|
|
2974
3002
|
}
|
|
2975
3003
|
/**
|
|
2976
3004
|
* Range-GET one chunk's payload, verifying its CRC32C before returning. Returns `null` if the chunk is
|
|
2977
|
-
* absent. Throws `IntegrityError` on a CRC mismatch (the bytes must never reach the
|
|
3005
|
+
* absent. Throws `IntegrityError` on a CRC mismatch (the bytes must never reach the codec's
|
|
2978
3006
|
* deserializer). The returned bytes may be shared (a small generation's chunks are kept by the reader and handed out
|
|
2979
3007
|
* again), so callers must not change them.
|
|
2980
3008
|
*/
|
|
@@ -2984,7 +3012,7 @@ var CrbmReader = class _CrbmReader {
|
|
|
2984
3012
|
const offset = this.index.offsets[slot];
|
|
2985
3013
|
const length = this.index.lengths[slot];
|
|
2986
3014
|
if (offset < PAYLOAD_START || offset + length > this.objectSize - FOOTER_BYTES) {
|
|
2987
|
-
throw crbmIntegrity(`chunk ${
|
|
3015
|
+
throw crbmIntegrity(`chunk at position ${slot} payload out of bounds`);
|
|
2988
3016
|
}
|
|
2989
3017
|
const held = this.keptRange(offset, length);
|
|
2990
3018
|
return this.openChunk(slot, chunkKey, held ?? await this.blob.getRange(offset, length));
|
|
@@ -3011,11 +3039,11 @@ var CrbmReader = class _CrbmReader {
|
|
|
3011
3039
|
*/
|
|
3012
3040
|
openChunk(slot, chunkKey, bytes) {
|
|
3013
3041
|
if (crc32c(bytes) !== this.index.crcs[slot]) {
|
|
3014
|
-
throw crbmIntegrity(`chunk ${
|
|
3042
|
+
throw crbmIntegrity(`chunk at position ${slot} payload CRC mismatch`);
|
|
3015
3043
|
}
|
|
3016
3044
|
if (this.crypto === void 0) return bytes;
|
|
3017
3045
|
if (bytes.length < AEAD_NONCE_BYTES + AEAD_TAG_BYTES) {
|
|
3018
|
-
throw crbmIntegrity(`chunk ${
|
|
3046
|
+
throw crbmIntegrity(`chunk at position ${slot} encrypted payload too short`);
|
|
3019
3047
|
}
|
|
3020
3048
|
return this.crypto.aead.open(
|
|
3021
3049
|
{
|
|
@@ -3261,19 +3289,20 @@ function parseIndex(indexBytes, payloadEnd, maxPayloadBytes, minPayloadBytes = 1
|
|
|
3261
3289
|
const chunkKey = prevKey + keyDelta;
|
|
3262
3290
|
const offset = prevEnd + offDelta;
|
|
3263
3291
|
if (i > 0 && keyDelta === 0) {
|
|
3264
|
-
throw crbmIntegrity(`.crbm index
|
|
3292
|
+
throw crbmIntegrity(`.crbm index entry ${i} repeats a chunk key`);
|
|
3265
3293
|
}
|
|
3266
|
-
if (chunkKey > 65535)
|
|
3294
|
+
if (chunkKey > 65535)
|
|
3295
|
+
throw crbmIntegrity(`.crbm index entry ${i} has a chunk key out of range`);
|
|
3267
3296
|
if (len < minPayloadBytes || len > maxPayloadBytes) {
|
|
3268
3297
|
throw crbmIntegrity(
|
|
3269
|
-
`.crbm
|
|
3298
|
+
`.crbm index entry ${i} has a payload length ${len} invalid (most is ${maxPayloadBytes})`
|
|
3270
3299
|
);
|
|
3271
3300
|
}
|
|
3272
3301
|
if (card < 1 || card > MAX_CHUNK_CARDINALITY) {
|
|
3273
|
-
throw crbmIntegrity(`.crbm
|
|
3302
|
+
throw crbmIntegrity(`.crbm index entry ${i} has a cardinality ${card} invalid`);
|
|
3274
3303
|
}
|
|
3275
3304
|
if (offset + len > payloadEnd) {
|
|
3276
|
-
throw crbmIntegrity(`.crbm
|
|
3305
|
+
throw crbmIntegrity(`.crbm index entry ${i} has a payload out of bounds`);
|
|
3277
3306
|
}
|
|
3278
3307
|
keys[count] = chunkKey;
|
|
3279
3308
|
cardinalityMinusOne[count] = card - 1;
|
|
@@ -3357,8 +3386,9 @@ async function provenObject(storage, key, fingerprint) {
|
|
|
3357
3386
|
version ??= tail.version;
|
|
3358
3387
|
});
|
|
3359
3388
|
return await CrbmReader.sameObject(blob, fingerprint) ? { version } : null;
|
|
3360
|
-
} catch {
|
|
3361
|
-
return null;
|
|
3389
|
+
} catch (err) {
|
|
3390
|
+
if (isNotFoundError(err)) return null;
|
|
3391
|
+
throw err;
|
|
3362
3392
|
}
|
|
3363
3393
|
}
|
|
3364
3394
|
function sameObjectUnder(storage, key, fingerprint) {
|
|
@@ -3409,12 +3439,7 @@ async function footerCrcUnder(storage, key) {
|
|
|
3409
3439
|
return (await readFooter(storageBlobReader(storage, key))).footerCrc;
|
|
3410
3440
|
}
|
|
3411
3441
|
async function provesOwnObject(storage, key, fingerprint) {
|
|
3412
|
-
|
|
3413
|
-
return await CrbmReader.sameObject(storageBlobReader(storage, key), fingerprint);
|
|
3414
|
-
} catch (err) {
|
|
3415
|
-
if (isNotFoundError(err)) return false;
|
|
3416
|
-
throw err;
|
|
3417
|
-
}
|
|
3442
|
+
return await provenObject(storage, key, fingerprint) !== null;
|
|
3418
3443
|
}
|
|
3419
3444
|
var openedVersions = /* @__PURE__ */ new WeakMap();
|
|
3420
3445
|
function openGenerationReader(storage, key, crypto, options = {}, named) {
|
|
@@ -5042,11 +5067,11 @@ var PinnedStorageChunkSource = class {
|
|
|
5042
5067
|
constructor(inner, pins) {
|
|
5043
5068
|
this.inner = inner;
|
|
5044
5069
|
this.pins = pins;
|
|
5045
|
-
for (const [
|
|
5070
|
+
for (const [pinKey, pin] of pins) {
|
|
5046
5071
|
const held = pin;
|
|
5047
5072
|
if (held.generation !== null && (typeof held.version !== "string" || typeof held.fingerprint !== "string"))
|
|
5048
5073
|
throw new ValidationError(
|
|
5049
|
-
`pin for "${
|
|
5074
|
+
`pin for "${pinKey}" names generation ${held.generation} without its version and object fingerprint`
|
|
5050
5075
|
);
|
|
5051
5076
|
}
|
|
5052
5077
|
}
|
|
@@ -5247,14 +5272,15 @@ function plainView(bytes, what) {
|
|
|
5247
5272
|
}
|
|
5248
5273
|
return length === 0 ? new Uint8Array(0) : new Uint8Array(buffer, offset, length);
|
|
5249
5274
|
}
|
|
5250
|
-
var
|
|
5275
|
+
var plainByteLength = Object.getOwnPropertyDescriptor(ArrayBuffer.prototype, "byteLength")?.get;
|
|
5276
|
+
var hasSharedMemory = typeof SharedArrayBuffer === "function";
|
|
5251
5277
|
function isShared(buffer) {
|
|
5252
|
-
if (
|
|
5278
|
+
if (!hasSharedMemory) return false;
|
|
5253
5279
|
try {
|
|
5254
|
-
|
|
5255
|
-
return true;
|
|
5256
|
-
} catch {
|
|
5280
|
+
plainByteLength.call(buffer);
|
|
5257
5281
|
return false;
|
|
5282
|
+
} catch {
|
|
5283
|
+
return true;
|
|
5258
5284
|
}
|
|
5259
5285
|
}
|
|
5260
5286
|
function overCap(what, length) {
|
|
@@ -5295,11 +5321,22 @@ function prepareLoadInput(input, codec) {
|
|
|
5295
5321
|
`a load takes ids (an iterable of integers), { serialized } or { bitmap }, as the only key; got an object with keys [${keys.join(", ")}]. Load options go in the next argument.`
|
|
5296
5322
|
);
|
|
5297
5323
|
}
|
|
5298
|
-
function decodeSerialized(bytes, codec, what = "the buffer") {
|
|
5324
|
+
function decodeSerialized(bytes, codec, what = "the buffer", options = {}) {
|
|
5299
5325
|
if (typeof bytes !== "object" || bytes === null || tagOf(bytes) !== "Uint8Array") {
|
|
5300
5326
|
throw new ValidationError(`${what} must be a Uint8Array of portable Roaring bytes`);
|
|
5301
5327
|
}
|
|
5302
5328
|
const view = plainView(bytes, what);
|
|
5329
|
+
if (view.byteLength === 0 && options.refuseEmpty === true) {
|
|
5330
|
+
throw new ValidationError(
|
|
5331
|
+
`${what} is empty or detached: an empty buffer is not a bitmap, and an empty bitmap is built without bytes`
|
|
5332
|
+
);
|
|
5333
|
+
}
|
|
5334
|
+
const { maxBytes } = options;
|
|
5335
|
+
if (maxBytes !== void 0 && maxBytes < MAX_SERIALIZED_LOAD_BYTES && view.byteLength > maxBytes) {
|
|
5336
|
+
throw new ValidationError(
|
|
5337
|
+
`${what} is ${view.byteLength} bytes, more than the ${maxBytes} this call allows (maxBytes)`
|
|
5338
|
+
);
|
|
5339
|
+
}
|
|
5303
5340
|
if (view.byteLength > MAX_SERIALIZED_LOAD_BYTES) throw overCap(what, view.byteLength);
|
|
5304
5341
|
const own = isShared(view.buffer) ? new Uint8Array(view) : view;
|
|
5305
5342
|
try {
|
|
@@ -5437,12 +5474,10 @@ var CrbmWriter = class {
|
|
|
5437
5474
|
async addChunk(chunkKey, payload, cardinality) {
|
|
5438
5475
|
if (this.finished) throw new ValidationError("CrbmWriter already finished");
|
|
5439
5476
|
if (!Number.isInteger(chunkKey) || chunkKey < 0 || chunkKey > 65535) {
|
|
5440
|
-
throw new ValidationError(
|
|
5477
|
+
throw new ValidationError("chunkKey must be a u16");
|
|
5441
5478
|
}
|
|
5442
5479
|
if (chunkKey <= this.lastKey) {
|
|
5443
|
-
throw new ValidationError(
|
|
5444
|
-
`chunkKey ${chunkKey} not strictly ascending (last ${this.lastKey})`
|
|
5445
|
-
);
|
|
5480
|
+
throw new ValidationError("chunkKey is not strictly ascending");
|
|
5446
5481
|
}
|
|
5447
5482
|
if (!Number.isInteger(cardinality) || cardinality < 1 || cardinality > MAX_CHUNK_CARDINALITY) {
|
|
5448
5483
|
throw new ValidationError(
|
|
@@ -6252,7 +6287,7 @@ async function collectChunks(input, codec, clock) {
|
|
|
6252
6287
|
throw new ValidationError("a chunk's key must be an integer in [0, 65535]");
|
|
6253
6288
|
}
|
|
6254
6289
|
if (chunkKey <= last) {
|
|
6255
|
-
throw new ValidationError(
|
|
6290
|
+
throw new ValidationError("a chunk's key does not ascend past the previous chunk's");
|
|
6256
6291
|
}
|
|
6257
6292
|
last = chunkKey;
|
|
6258
6293
|
const max = bitmap.maximum?.();
|
|
@@ -7271,7 +7306,7 @@ var KeyBits = class {
|
|
|
7271
7306
|
};
|
|
7272
7307
|
|
|
7273
7308
|
// src/core/combine-ledger.ts
|
|
7274
|
-
var RESIDENT_BITMAP_OVERHEAD =
|
|
7309
|
+
var RESIDENT_BITMAP_OVERHEAD = 832;
|
|
7275
7310
|
var MAX_CHUNK_SERIALIZED = 8208;
|
|
7276
7311
|
function serializedBound(cardinality) {
|
|
7277
7312
|
return Math.min(2 * cardinality + 16, MAX_CHUNK_SERIALIZED);
|
|
@@ -7279,7 +7314,7 @@ function serializedBound(cardinality) {
|
|
|
7279
7314
|
var MAX_RESIDENT_DATA = 8800;
|
|
7280
7315
|
function residentBytes(serialized) {
|
|
7281
7316
|
return Math.ceil(
|
|
7282
|
-
RESIDENT_BITMAP_OVERHEAD + Math.min(1.6 * serialized, serialized +
|
|
7317
|
+
RESIDENT_BITMAP_OVERHEAD + Math.min(1.6 * serialized, serialized + 2200, MAX_RESIDENT_DATA)
|
|
7283
7318
|
);
|
|
7284
7319
|
}
|
|
7285
7320
|
function residentBound(cardinality) {
|
|
@@ -8643,7 +8678,7 @@ var Run = class {
|
|
|
8643
8678
|
failOperand(
|
|
8644
8679
|
c,
|
|
8645
8680
|
new IntegrityError(
|
|
8646
|
-
`operand "${c.op.spec.name}" lists chunk
|
|
8681
|
+
`operand "${c.op.spec.name}" lists a chunk its generation holds no bytes for`
|
|
8647
8682
|
),
|
|
8648
8683
|
victims ?? wanting(c)
|
|
8649
8684
|
);
|
|
@@ -8652,13 +8687,10 @@ var Run = class {
|
|
|
8652
8687
|
try {
|
|
8653
8688
|
const resident = residentBytes(bytes.length);
|
|
8654
8689
|
if (!this.ledger.reserve(resident, makeRoom)) {
|
|
8655
|
-
throw this.refuse(resident, `operand "${c.op.spec.name}" at
|
|
8690
|
+
throw this.refuse(resident, `operand "${c.op.spec.name}" at one of its chunks`);
|
|
8656
8691
|
}
|
|
8657
8692
|
held += resident;
|
|
8658
|
-
decoded.set(
|
|
8659
|
-
c.op.index,
|
|
8660
|
-
decodeChunkBytes(this.codec, bytes, key, this.maxBitmapBytes)
|
|
8661
|
-
);
|
|
8693
|
+
decoded.set(c.op.index, decodeChunkBytes(this.codec, bytes, this.maxBitmapBytes));
|
|
8662
8694
|
} catch (error) {
|
|
8663
8695
|
failOperand(c, error, victims ?? wanting(c));
|
|
8664
8696
|
}
|
|
@@ -9565,7 +9597,7 @@ async function holds(c, generation) {
|
|
|
9565
9597
|
}
|
|
9566
9598
|
throw err;
|
|
9567
9599
|
}
|
|
9568
|
-
return bytes !== null && codec
|
|
9600
|
+
return bytes !== null && decodeChunkBytes(codec, bytes, maxBytes).has(remainder);
|
|
9569
9601
|
} catch (err) {
|
|
9570
9602
|
if (isNotFoundError(err)) return null;
|
|
9571
9603
|
throw err;
|
|
@@ -9681,7 +9713,8 @@ async function fence(c, plan) {
|
|
|
9681
9713
|
const ledger = usableKnownGens(row);
|
|
9682
9714
|
const doomed = doomedGenerations(c, plan, byName, ledger, pointer);
|
|
9683
9715
|
const named = ledger !== void 0 && ledger.some((e) => doomed.has(e.generation));
|
|
9684
|
-
const
|
|
9716
|
+
const sweepOwes = plan.sweeps && !named && row.pointerId === record.pointerId && sameKnownGens(row.knownGens, record.knownGens);
|
|
9717
|
+
const renew = byName.length > 0 || sweepOwes;
|
|
9685
9718
|
const renewedAfter = held && row.pointerId !== first.pointerId;
|
|
9686
9719
|
if (!named && (!renew || renewedAfter)) {
|
|
9687
9720
|
c.premise = row;
|
|
@@ -9975,8 +10008,7 @@ async function stage(c) {
|
|
|
9975
10008
|
c.fromFingerprint = reader.fingerprint;
|
|
9976
10009
|
const bytes = await read(() => reader.getChunk(chunkKey));
|
|
9977
10010
|
if (bytes === null) return await notInCurrent(c);
|
|
9978
|
-
const target = codec
|
|
9979
|
-
assertRemaindersInRange(target, chunkKey);
|
|
10011
|
+
const target = decodeChunkBytes(codec, bytes, maxBytes);
|
|
9980
10012
|
if (!target.has(remainder)) return await notInCurrent(c);
|
|
9981
10013
|
target.remove(remainder);
|
|
9982
10014
|
assertRegistryCanWrite(deps.registry, "eraseIdFromSegment");
|
|
@@ -10039,6 +10071,34 @@ async function stage(c) {
|
|
|
10039
10071
|
throw err;
|
|
10040
10072
|
}
|
|
10041
10073
|
}
|
|
10074
|
+
async function provenOwn(c, key, fingerprint) {
|
|
10075
|
+
const { deps } = c;
|
|
10076
|
+
const retry = deps.readRetry;
|
|
10077
|
+
const base = retry?.isRetryable ?? isTransientError;
|
|
10078
|
+
let fault;
|
|
10079
|
+
try {
|
|
10080
|
+
return await retryRead(
|
|
10081
|
+
async () => {
|
|
10082
|
+
try {
|
|
10083
|
+
return await provenObject(deps.storage, key, fingerprint);
|
|
10084
|
+
} catch (err) {
|
|
10085
|
+
if (!isIntegrityError(err)) fault = err;
|
|
10086
|
+
throw err;
|
|
10087
|
+
}
|
|
10088
|
+
},
|
|
10089
|
+
retry === void 0 ? void 0 : {
|
|
10090
|
+
...retry,
|
|
10091
|
+
isRetryable: (err) => isIntegrityError(err) || base(err),
|
|
10092
|
+
onRetry: (info) => {
|
|
10093
|
+
if (!isIntegrityError(info.err)) retry.onRetry?.(info);
|
|
10094
|
+
}
|
|
10095
|
+
}
|
|
10096
|
+
);
|
|
10097
|
+
} catch (err) {
|
|
10098
|
+
if (isIntegrityError(err) && fault === void 0) return null;
|
|
10099
|
+
throw fault ?? err;
|
|
10100
|
+
}
|
|
10101
|
+
}
|
|
10042
10102
|
async function discardRefused(c, own) {
|
|
10043
10103
|
const { ref, deps, base, from } = c;
|
|
10044
10104
|
const key = { ...base, generation: own.generation };
|
|
@@ -10055,7 +10115,7 @@ async function discardRefused(c, own) {
|
|
|
10055
10115
|
}
|
|
10056
10116
|
if (row === null || row.status !== "active" || row.currentGen === null) return row;
|
|
10057
10117
|
if (row.currentGen <= from || row.currentGen >= own.generation) return row;
|
|
10058
|
-
const proven = await
|
|
10118
|
+
const proven = await provenOwn(c, key, own.fingerprint);
|
|
10059
10119
|
if (proven === null) return row;
|
|
10060
10120
|
const renewed = await renewal.renew(row);
|
|
10061
10121
|
if ("row" in renewed) {
|
|
@@ -10234,26 +10294,17 @@ async function* rewrite(reader, chunkKey, replacement, codec, maxBytes, read) {
|
|
|
10234
10294
|
}
|
|
10235
10295
|
const item = await stream.next();
|
|
10236
10296
|
if (item.done === true || item.value.key !== k) {
|
|
10237
|
-
throw new IntegrityError(
|
|
10297
|
+
throw new IntegrityError("the chunk stream ended before a chunk the rewrite needs");
|
|
10238
10298
|
}
|
|
10239
10299
|
const bytes = item.value.bytes;
|
|
10240
10300
|
if (bytes === null) continue;
|
|
10241
|
-
const bitmap = codec
|
|
10242
|
-
assertRemaindersInRange(bitmap, k);
|
|
10301
|
+
const bitmap = decodeChunkBytes(codec, bytes, maxBytes);
|
|
10243
10302
|
yield { chunkKey: k, bitmap };
|
|
10244
10303
|
}
|
|
10245
10304
|
} finally {
|
|
10246
10305
|
await stream.return(void 0);
|
|
10247
10306
|
}
|
|
10248
10307
|
}
|
|
10249
|
-
function assertRemaindersInRange(bitmap, chunkKey) {
|
|
10250
|
-
const max = bitmap.maximum?.();
|
|
10251
|
-
if (max !== void 0 && max > MAX_REMAINDER) {
|
|
10252
|
-
throw new IntegrityError(
|
|
10253
|
-
`chunk ${chunkKey} payload holds value ${max}, outside the 16-bit remainder range [0, ${MAX_REMAINDER}] \u2014 the stored object is corrupt or was not written by this codec; refusing to carry it into a new generation`
|
|
10254
|
-
);
|
|
10255
|
-
}
|
|
10256
|
-
}
|
|
10257
10308
|
|
|
10258
10309
|
// src/core/cas-retry.ts
|
|
10259
10310
|
async function casRetry(deps, ref, plan, step) {
|
|
@@ -12047,6 +12098,7 @@ export {
|
|
|
12047
12098
|
LeaseLimitError,
|
|
12048
12099
|
MAX_LEASES_PER_SEGMENT,
|
|
12049
12100
|
MAX_LEASE_MS,
|
|
12101
|
+
MAX_SERIALIZED_LOAD_BYTES,
|
|
12050
12102
|
MIN_EXPIRES_AT_MS,
|
|
12051
12103
|
NOOP_METRICS,
|
|
12052
12104
|
NotFoundError,
|
|
@@ -12082,6 +12134,7 @@ export {
|
|
|
12082
12134
|
mapWithConcurrency,
|
|
12083
12135
|
readRetentionPolicy,
|
|
12084
12136
|
rebindCombineMany,
|
|
12137
|
+
refusalOfId,
|
|
12085
12138
|
refuseUnknownRefKeys,
|
|
12086
12139
|
releaseLease,
|
|
12087
12140
|
resolveBudget,
|