@cloudbitmaps/core 0.26.0 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-LJLZRVYN.js → chunk-A2UDUKNL.js} +2 -2
- package/dist/{chunk-MNXTZJZ6.js → chunk-KO3Z2RBA.js} +2 -2
- package/dist/{chunk-UFOMYONZ.js → chunk-UUNBQVKR.js} +203 -203
- package/dist/{chunk-UFOMYONZ.js.map → chunk-UUNBQVKR.js.map} +3 -3
- package/dist/{chunk-7VNEBKC4.js → chunk-YNNGMHX2.js} +2 -2
- package/dist/core/codec.d.ts +2 -2
- package/dist/core/crbm-storage-source.d.ts +42 -2
- package/dist/core/erase-id.d.ts +2 -1
- package/dist/core/erasure.d.ts +2 -1
- package/dist/core/load.d.ts +11 -0
- package/dist/core/pinned-storage-source.d.ts +9 -1
- package/dist/core/rollback.d.ts +2 -1
- package/dist/driver-kit.js +16 -4
- package/dist/driver-kit.js.map +2 -2
- package/dist/index.js +272 -44
- package/dist/index.js.map +4 -4
- package/dist/node.js +9 -6
- package/dist/node.js.map +2 -2
- package/package.json +1 -1
- /package/dist/{chunk-LJLZRVYN.js.map → chunk-A2UDUKNL.js.map} +0 -0
- /package/dist/{chunk-MNXTZJZ6.js.map → chunk-KO3Z2RBA.js.map} +0 -0
- /package/dist/{chunk-7VNEBKC4.js.map → chunk-YNNGMHX2.js.map} +0 -0
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
boundedAll,
|
|
3
3
|
mapWithConcurrency
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-YNNGMHX2.js";
|
|
5
5
|
import {
|
|
6
6
|
BufferReader,
|
|
7
7
|
KeptSegmentKeys,
|
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
segmentKey,
|
|
10
10
|
segmentPrefix,
|
|
11
11
|
shardOf
|
|
12
|
-
} from "./chunk-
|
|
12
|
+
} from "./chunk-KO3Z2RBA.js";
|
|
13
13
|
import {
|
|
14
14
|
BudgetExceededError,
|
|
15
15
|
CHUNK_COUNT,
|
|
@@ -116,7 +116,7 @@ import {
|
|
|
116
116
|
validateUserRef,
|
|
117
117
|
windowOf,
|
|
118
118
|
withRetry
|
|
119
|
-
} from "./chunk-
|
|
119
|
+
} from "./chunk-UUNBQVKR.js";
|
|
120
120
|
|
|
121
121
|
// src/core/budget.ts
|
|
122
122
|
var DEFAULT_BUDGET = { maxRequests: 1e6 };
|
|
@@ -220,7 +220,7 @@ function assertChunkPayloadInRange(bitmap) {
|
|
|
220
220
|
}
|
|
221
221
|
}
|
|
222
222
|
function decodeChunkBytes(codec, bytes, maxBitmapBytes) {
|
|
223
|
-
const bitmap = codec.safeDeserialize(bytes, maxBitmapBytes);
|
|
223
|
+
const bitmap = codec.safeDeserialize(bytes, maxBitmapBytes, { whole: true });
|
|
224
224
|
assertChunkPayloadInRange(bitmap);
|
|
225
225
|
return bitmap;
|
|
226
226
|
}
|
|
@@ -2852,8 +2852,39 @@ function checkedFooter(tail, size) {
|
|
|
2852
2852
|
if (crc32c(footer.subarray(0, FOOTER_CRC_COVERAGE)) !== storedFooterCrc) {
|
|
2853
2853
|
throw crbmIntegrity(".crbm footer CRC mismatch");
|
|
2854
2854
|
}
|
|
2855
|
+
if (footer[FOOTER.versionMajor] === VERSION_MAJOR) checkFooterFields(footer, fview, size);
|
|
2855
2856
|
return { footer, fview, storedFooterCrc };
|
|
2856
2857
|
}
|
|
2858
|
+
function checkPreamble(tail, versionMajor, versionMinor) {
|
|
2859
|
+
if (!magicMatches(tail, 0) || tail[4] !== versionMajor || tail[5] !== versionMinor) {
|
|
2860
|
+
throw crbmIntegrity(".crbm preamble magic/version mismatch with footer");
|
|
2861
|
+
}
|
|
2862
|
+
if (tail[6] !== 0 || tail[7] !== 0) {
|
|
2863
|
+
throw crbmIntegrity(".crbm preamble reserved bytes are not zero");
|
|
2864
|
+
}
|
|
2865
|
+
}
|
|
2866
|
+
function allZero(bytes, from, to) {
|
|
2867
|
+
for (let i = from; i < to; i++) if (bytes[i] !== 0) return false;
|
|
2868
|
+
return true;
|
|
2869
|
+
}
|
|
2870
|
+
function checkFooterFields(footer, fview, size) {
|
|
2871
|
+
if (fview.getUint16(FOOTER.reserved2, true) !== 0) {
|
|
2872
|
+
throw crbmIntegrity(".crbm footer reserved2 is not zero");
|
|
2873
|
+
}
|
|
2874
|
+
if (!allZero(footer, FOOTER.keyId, FOOTER.chunkCount)) {
|
|
2875
|
+
throw crbmIntegrity(".crbm footer key_id is not zero");
|
|
2876
|
+
}
|
|
2877
|
+
if ((fview.getUint32(FOOTER.flags, true) & FLAG_ENCRYPTED) === 0 && !allZero(footer, FOOTER.indexTag, FOOTER.indexTag + AEAD_TAG_BYTES)) {
|
|
2878
|
+
throw crbmIntegrity(".crbm cleartext footer index_tag is not zero");
|
|
2879
|
+
}
|
|
2880
|
+
const indexOffset = readU64(fview, FOOTER.indexOffset, "index_offset");
|
|
2881
|
+
const indexLength = readU64(fview, FOOTER.indexLength, "index_length");
|
|
2882
|
+
if (indexOffset + indexLength !== size - FOOTER_BYTES) {
|
|
2883
|
+
throw crbmIntegrity(
|
|
2884
|
+
`.crbm index region [${indexOffset}, +${indexLength}) does not end where the footer begins (${size - FOOTER_BYTES})`
|
|
2885
|
+
);
|
|
2886
|
+
}
|
|
2887
|
+
}
|
|
2857
2888
|
async function readFooter(blob) {
|
|
2858
2889
|
const { bytes: tail, size } = await blob.getTail(FOOTER_BYTES);
|
|
2859
2890
|
const { fview, storedFooterCrc } = checkedFooter(tail, size);
|
|
@@ -3023,7 +3054,7 @@ var CrbmReader = class _CrbmReader {
|
|
|
3023
3054
|
}
|
|
3024
3055
|
const versionMinor = footer[FOOTER.versionMinor];
|
|
3025
3056
|
const hasExtension = (flags & FLAG_EXTENSION) !== 0;
|
|
3026
|
-
if (indexOffset < PAYLOAD_START
|
|
3057
|
+
if (indexOffset < PAYLOAD_START) {
|
|
3027
3058
|
throw crbmIntegrity(`.crbm index region [${indexOffset}, +${indexLength}) out of bounds`);
|
|
3028
3059
|
}
|
|
3029
3060
|
if (indexLength > maxIndexBytes) {
|
|
@@ -3031,9 +3062,7 @@ var CrbmReader = class _CrbmReader {
|
|
|
3031
3062
|
}
|
|
3032
3063
|
const tailCoversFrom = size - tail.length;
|
|
3033
3064
|
if (tailCoversFrom === 0) {
|
|
3034
|
-
|
|
3035
|
-
throw crbmIntegrity(".crbm preamble magic/version mismatch with footer");
|
|
3036
|
-
}
|
|
3065
|
+
checkPreamble(tail, versionMajor, versionMinor);
|
|
3037
3066
|
}
|
|
3038
3067
|
const lowest = hasExtension ? Math.max(PAYLOAD_START, indexOffset - EXT_TRAILER_BYTES - MAX_EXT_BYTES) : indexOffset;
|
|
3039
3068
|
let region;
|
|
@@ -3518,6 +3547,8 @@ function parseIndex(indexBytes, payloadEnd, maxPayloadBytes, minPayloadBytes = 1
|
|
|
3518
3547
|
if (card < 1 || card > MAX_CHUNK_CARDINALITY) {
|
|
3519
3548
|
throw crbmIntegrity(`.crbm index entry ${i} has a cardinality ${card} invalid`);
|
|
3520
3549
|
}
|
|
3550
|
+
if (offDelta !== 0)
|
|
3551
|
+
throw crbmIntegrity(`.crbm index entry ${i} leaves a gap before its payload`);
|
|
3521
3552
|
if (offset + len > payloadEnd) {
|
|
3522
3553
|
throw crbmIntegrity(`.crbm index entry ${i} has a payload out of bounds`);
|
|
3523
3554
|
}
|
|
@@ -3531,6 +3562,11 @@ function parseIndex(indexBytes, payloadEnd, maxPayloadBytes, minPayloadBytes = 1
|
|
|
3531
3562
|
prevKey = chunkKey;
|
|
3532
3563
|
prevEnd = offset + len;
|
|
3533
3564
|
}
|
|
3565
|
+
if (prevEnd !== payloadEnd) {
|
|
3566
|
+
throw crbmIntegrity(
|
|
3567
|
+
`.crbm payloads end at ${prevEnd}, not at the end of the payload region ${payloadEnd}`
|
|
3568
|
+
);
|
|
3569
|
+
}
|
|
3534
3570
|
if (count === capacity) {
|
|
3535
3571
|
return { keys, cardinalityMinusOne, lengths, crcs, offsets, cardinalitySum };
|
|
3536
3572
|
}
|
|
@@ -3818,6 +3854,9 @@ var notThePinned = (ref, generation) => new NotFoundError(
|
|
|
3818
3854
|
var goneUnder = (ref, generation) => new NotFoundError(
|
|
3819
3855
|
`segment "${ref.segment}" generation ${generation} can no longer be read: its registry row is gone or destroyed`
|
|
3820
3856
|
);
|
|
3857
|
+
var pinGone = (ref, generation) => new NotFoundError(
|
|
3858
|
+
`segment "${ref.segment}" generation ${generation}, which this handle pinned, can no longer be read: its registry row is gone or destroyed`
|
|
3859
|
+
);
|
|
3821
3860
|
function genKeyOf(ref, target) {
|
|
3822
3861
|
return { namespace: ref.namespace, segment: ref.segment, generation: target.generation };
|
|
3823
3862
|
}
|
|
@@ -4033,6 +4072,10 @@ var CrbmStorageChunkSource = class {
|
|
|
4033
4072
|
maxBytes: (maxOpenIndexBytes ?? DEFAULT_MAX_OPEN_INDEX_BYTES) / RESOLUTION_BYTES_DIVISOR,
|
|
4034
4073
|
clock: { now: () => 0 }
|
|
4035
4074
|
}) : void 0;
|
|
4075
|
+
this.pinRows = this.timedRefresh ? new BoundedLru({
|
|
4076
|
+
maxEntries: maxOpenSegments ?? DEFAULT_MAX_OPEN_SEGMENTS,
|
|
4077
|
+
clock: { now: () => 0 }
|
|
4078
|
+
}) : void 0;
|
|
4036
4079
|
this.distrusted = new BoundedLru({
|
|
4037
4080
|
maxEntries: maxOpenSegments ?? DEFAULT_MAX_OPEN_SEGMENTS,
|
|
4038
4081
|
clock: { now: () => 0 }
|
|
@@ -4087,6 +4130,8 @@ var CrbmStorageChunkSource = class {
|
|
|
4087
4130
|
* read. One forgotten before it ends, by {@link invalidate} or a pin, keeps nothing it finds.
|
|
4088
4131
|
*/
|
|
4089
4132
|
checking = /* @__PURE__ */ new Map();
|
|
4133
|
+
/** When each footer read in {@link checking} was sent. */
|
|
4134
|
+
footerSentAt = /* @__PURE__ */ new WeakMap();
|
|
4090
4135
|
/**
|
|
4091
4136
|
* How many times {@link invalidate} has run, so a read that waited can tell whether the store was told something
|
|
4092
4137
|
* meanwhile. An eviction from the reader cache is not an invalidation, and what a read found stands through one. It
|
|
@@ -4129,6 +4174,13 @@ var CrbmStorageChunkSource = class {
|
|
|
4129
4174
|
* lapse, so it is decided once.
|
|
4130
4175
|
*/
|
|
4131
4176
|
timedRefresh;
|
|
4177
|
+
/**
|
|
4178
|
+
* The row reads a pinned segment's rechecks share, by segment: one `registry.get` per `cache.genTtlMs` serves every pin,
|
|
4179
|
+
* handle and combine of the segment in this store. Only a source with a timed refresh keeps any.
|
|
4180
|
+
*/
|
|
4181
|
+
pinRows;
|
|
4182
|
+
/** Each pin's last check, by the record the handle holds, so a handle that is let go takes its record with it. */
|
|
4183
|
+
pinChecks = /* @__PURE__ */ new WeakMap();
|
|
4132
4184
|
/**
|
|
4133
4185
|
* The keys of the segments this source reads, kept rather than encoded again: a read asks for its segment's snapshot
|
|
4134
4186
|
* once per operand, and a stream before every cached chunk it serves, and encoding the names cost about a
|
|
@@ -4342,6 +4394,7 @@ var CrbmStorageChunkSource = class {
|
|
|
4342
4394
|
this.invalidated.invalidate(key);
|
|
4343
4395
|
this.snapshots.delete(key);
|
|
4344
4396
|
this.resolutions?.delete(key);
|
|
4397
|
+
this.pinRows?.delete(key);
|
|
4345
4398
|
const pinnedPrefix = `${key}@`;
|
|
4346
4399
|
this.distrusted.deleteWhere((k) => k.startsWith(pinnedPrefix));
|
|
4347
4400
|
this.snapshots.deleteWhere((k) => k.startsWith(pinnedPrefix));
|
|
@@ -4452,6 +4505,7 @@ var CrbmStorageChunkSource = class {
|
|
|
4452
4505
|
let installed;
|
|
4453
4506
|
let pinKey;
|
|
4454
4507
|
let memoised = false;
|
|
4508
|
+
const checkedAtMs = this.now();
|
|
4455
4509
|
if (this.registry !== void 0) {
|
|
4456
4510
|
const record = await this.registry.get(ref);
|
|
4457
4511
|
if (record === null || record.status === "destroyed")
|
|
@@ -4497,9 +4551,10 @@ var CrbmStorageChunkSource = class {
|
|
|
4497
4551
|
this.replacedPins.delete(held);
|
|
4498
4552
|
this.checking.delete(held);
|
|
4499
4553
|
}
|
|
4500
|
-
return { generation, version, fingerprint };
|
|
4554
|
+
return { generation, version, fingerprint, checkedAtMs };
|
|
4501
4555
|
}
|
|
4502
4556
|
async pinOnce(ref, lease) {
|
|
4557
|
+
const checkedAtMs = this.now();
|
|
4503
4558
|
const live = await this.resolveLive(ref);
|
|
4504
4559
|
if (live === null) return null;
|
|
4505
4560
|
if (lease !== void 0 && live.row !== void 0) {
|
|
@@ -4556,7 +4611,7 @@ var CrbmStorageChunkSource = class {
|
|
|
4556
4611
|
this.replacedPins.delete(held);
|
|
4557
4612
|
this.checking.delete(held);
|
|
4558
4613
|
}
|
|
4559
|
-
return { generation: target.generation, version, fingerprint: reader.fingerprint };
|
|
4614
|
+
return { generation: target.generation, version, fingerprint: reader.fingerprint, checkedAtMs };
|
|
4560
4615
|
}
|
|
4561
4616
|
/**
|
|
4562
4617
|
* Where a pinned reader of one object is memoised. With a registry that is its version, which names the object:
|
|
@@ -4592,7 +4647,7 @@ var CrbmStorageChunkSource = class {
|
|
|
4592
4647
|
* crypto-shred must not keep unwrapping a DEK whose wrapping the shred removed; a destroyed segment resolves no
|
|
4593
4648
|
* generation
|
|
4594
4649
|
* for anyone, and a pin's read of one fails. That holds from the pin's next open: a reader already open keeps
|
|
4595
|
-
* the key it unwrapped until it is evicted or
|
|
4650
|
+
* the key it unwrapped until it is evicted, invalidated, or its next check ({@link recheckPin}) finds the pin over.
|
|
4596
4651
|
*
|
|
4597
4652
|
* **And the object is checked**, when the caller says which one it pinned. A generation number is not an
|
|
4598
4653
|
* identity (invariant 1): once a name is purged and loaded again, its new segment starts again at generation 0,
|
|
@@ -4630,9 +4685,7 @@ var CrbmStorageChunkSource = class {
|
|
|
4630
4685
|
}
|
|
4631
4686
|
if (reader === null) {
|
|
4632
4687
|
if (version === void 0) return null;
|
|
4633
|
-
throw
|
|
4634
|
-
`segment "${ref.segment}" generation ${generation}, which this handle pinned, can no longer be read: its registry row is gone or destroyed`
|
|
4635
|
-
);
|
|
4688
|
+
throw pinGone(ref, generation);
|
|
4636
4689
|
}
|
|
4637
4690
|
if (fingerprint !== void 0 && reader.fingerprint !== fingerprint) {
|
|
4638
4691
|
if (this.invalidations === epoch)
|
|
@@ -4671,6 +4724,113 @@ var CrbmStorageChunkSource = class {
|
|
|
4671
4724
|
throw err;
|
|
4672
4725
|
}
|
|
4673
4726
|
}
|
|
4727
|
+
/**
|
|
4728
|
+
* Whether a pin's segment is due a look at its row: the pin was not checked within `cache.genTtlMs` (counted by
|
|
4729
|
+
* `Clock.now()` on access, no timer), the segment has been invalidated since, a check is under way, or the pin has
|
|
4730
|
+
* ended. A pin counts as checked from the instant its own row read was sent ({@link startPinChecks}), so one left
|
|
4731
|
+
* idle past `cache.genTtlMs` is checked at its first read; one with no record is due. Always `false` without a timed
|
|
4732
|
+
* refresh (no registry, no clock, or `cache.genTtlMs: 0`), where nothing bounds a pin. A compare and no await
|
|
4733
|
+
* otherwise, which is all a pinned read pays between checks.
|
|
4734
|
+
*/
|
|
4735
|
+
pinDue(ref, pin) {
|
|
4736
|
+
if (!this.timedRefresh) return false;
|
|
4737
|
+
const check = this.pinChecks.get(pin);
|
|
4738
|
+
return check === void 0 || check.check !== void 0 || check.ended !== void 0 || this.now() - check.at >= this.currentGenTtlMs || this.invalidated.invalidatedSince(this.keyOf(ref), check.since);
|
|
4739
|
+
}
|
|
4740
|
+
/** Start a new pin's checks from `checkedAtMs`, when the row read that verified it was sent. */
|
|
4741
|
+
startPinChecks(pin, checkedAtMs) {
|
|
4742
|
+
if (this.timedRefresh)
|
|
4743
|
+
this.pinChecks.set(pin, { at: checkedAtMs, since: this.invalidated.epoch });
|
|
4744
|
+
}
|
|
4745
|
+
/**
|
|
4746
|
+
* Settle a pin whose check is due: throw its `NotFoundError` when its segment has been destroyed or is gone, is
|
|
4747
|
+
* another incarnation, or holds another object under the pin's generation; return when it is still the pin's. One row
|
|
4748
|
+
* read per interval serves every pin of the segment ({@link pinRows}), and it decides alone for a row that is gone or
|
|
4749
|
+
* destroyed, a new incarnation, and a ledger that names the generation (its checksum is the pin's or not). A
|
|
4750
|
+
* generation the ledger does not name, or a row with no usable ledger, is not a verdict: the object's footer is read
|
|
4751
|
+
* once ({@link throwIfReplaced}, strict: a footer it cannot read fails the read). The pin counts as checked from
|
|
4752
|
+
* when the row read its verdict came from was sent, never from when the check began, so a row read shared from
|
|
4753
|
+
* earlier does not make a pin look fresher than it is. Every invalidation of the segment in this store makes the
|
|
4754
|
+
* next check read the row afresh, beside the one per interval. A transient fault keeps serving and asks again after {@link REFRESH_RETRY_MS}; any
|
|
4755
|
+
* other fault fails this read and is not remembered. A pin that has ended drops its reader, and so the key it
|
|
4756
|
+
* unwrapped, and fails every later read at once until the segment is invalidated.
|
|
4757
|
+
*/
|
|
4758
|
+
recheckPin(ref, generation, pin) {
|
|
4759
|
+
const prior = this.pinChecks.get(pin);
|
|
4760
|
+
if (prior?.check !== void 0) return prior.check;
|
|
4761
|
+
const segment = this.keyOf(ref);
|
|
4762
|
+
if (prior?.ended !== void 0 && !this.invalidated.invalidatedSince(segment, prior.since))
|
|
4763
|
+
return Promise.reject(
|
|
4764
|
+
prior.ended === "gone" ? pinGone(ref, generation) : notThePinned(ref, generation)
|
|
4765
|
+
);
|
|
4766
|
+
const next = { at: this.now(), since: this.invalidated.epoch };
|
|
4767
|
+
this.pinChecks.set(pin, next);
|
|
4768
|
+
next.check = this.settlePin(ref, generation, pin, next, segment).finally(() => {
|
|
4769
|
+
next.check = void 0;
|
|
4770
|
+
});
|
|
4771
|
+
return next.check;
|
|
4772
|
+
}
|
|
4773
|
+
async settlePin(ref, generation, pin, next, segment) {
|
|
4774
|
+
let ended;
|
|
4775
|
+
try {
|
|
4776
|
+
const read = this.pinRow(ref, segment);
|
|
4777
|
+
const facts = await read.facts;
|
|
4778
|
+
next.at = read.sentAtMs;
|
|
4779
|
+
const incarnation = incarnationOf(pin.version.slice(pin.version.indexOf(":") + 1));
|
|
4780
|
+
const named = facts.ledger?.find((e) => e.generation === generation);
|
|
4781
|
+
if (facts.ended || pin.version.includes(":") && incarnation !== void 0 && facts.incarnation !== void 0 && incarnation !== facts.incarnation) {
|
|
4782
|
+
ended = "gone";
|
|
4783
|
+
} else if (named === void 0) {
|
|
4784
|
+
try {
|
|
4785
|
+
const footerAt = await this.throwIfReplaced(
|
|
4786
|
+
ref,
|
|
4787
|
+
generation,
|
|
4788
|
+
pin.version,
|
|
4789
|
+
pin.fingerprint,
|
|
4790
|
+
true
|
|
4791
|
+
);
|
|
4792
|
+
next.at = Math.min(next.at, footerAt);
|
|
4793
|
+
} catch (err) {
|
|
4794
|
+
if (!isNotFoundError(err)) throw err;
|
|
4795
|
+
ended = "replaced";
|
|
4796
|
+
}
|
|
4797
|
+
} else if (named.footerCrc !== fingerprintParts(pin.fingerprint)?.footerCrc) {
|
|
4798
|
+
ended = "replaced";
|
|
4799
|
+
}
|
|
4800
|
+
} catch (err) {
|
|
4801
|
+
if (isTransientError(err)) {
|
|
4802
|
+
next.at = this.now() - (this.currentGenTtlMs - Math.min(this.currentGenTtlMs, REFRESH_RETRY_MS));
|
|
4803
|
+
return;
|
|
4804
|
+
}
|
|
4805
|
+
next.at = Number.NEGATIVE_INFINITY;
|
|
4806
|
+
throw err;
|
|
4807
|
+
}
|
|
4808
|
+
if (ended === void 0) return;
|
|
4809
|
+
if (!this.invalidated.invalidatedSince(segment, next.since)) next.ended = ended;
|
|
4810
|
+
if (ended === "replaced")
|
|
4811
|
+
this.replacedPins.set(this.heldKey(ref, pin.version, pin.fingerprint), true);
|
|
4812
|
+
this.snapshots.delete(this.pinnedKey(ref, pin.version, pin.fingerprint));
|
|
4813
|
+
throw ended === "gone" ? pinGone(ref, generation) : notThePinned(ref, generation);
|
|
4814
|
+
}
|
|
4815
|
+
/** The segment's row read of the last `cache.genTtlMs`, shared; sent now when there is none. */
|
|
4816
|
+
pinRow(ref, segment) {
|
|
4817
|
+
const now = this.now();
|
|
4818
|
+
const held = this.pinRows.peek(segment);
|
|
4819
|
+
if (held !== void 0 && now - held.sentAtMs < this.currentGenTtlMs) return held;
|
|
4820
|
+
const sent = {
|
|
4821
|
+
sentAtMs: now,
|
|
4822
|
+
facts: this.registry.get(ref).then((record) => ({
|
|
4823
|
+
ended: record === null || record.status === "destroyed",
|
|
4824
|
+
incarnation: record === null ? void 0 : incarnationOf(record.token),
|
|
4825
|
+
ledger: usableKnownGens(record)
|
|
4826
|
+
}))
|
|
4827
|
+
};
|
|
4828
|
+
this.pinRows.set(segment, sent);
|
|
4829
|
+
sent.facts.catch(() => {
|
|
4830
|
+
if (this.pinRows.peek(segment) === sent) this.pinRows.delete(segment);
|
|
4831
|
+
});
|
|
4832
|
+
return sent;
|
|
4833
|
+
}
|
|
4674
4834
|
/**
|
|
4675
4835
|
* Throws the pin's NotFoundError if the object under `generation`'s key is no longer the one the pin holds. Its
|
|
4676
4836
|
* footer says, read with no key and no row ({@link CrbmReader.sameObject}), so an object written under a key this
|
|
@@ -4678,7 +4838,8 @@ var CrbmStorageChunkSource = class {
|
|
|
4678
4838
|
* ({@link replacedPins}), and a check under way is shared by the reads that ask at once; neither "gone" nor "the
|
|
4679
4839
|
* same" is kept, since either can change. A transient fault is rethrown, for the store's retries to see for what it
|
|
4680
4840
|
* is. A footer that cannot be read otherwise says nothing about which object is there, so this returns, and the
|
|
4681
|
-
* caller's own error stands; a caller with no error of its own passes `strict`, and that fault is raised.
|
|
4841
|
+
* caller's own error stands; a caller with no error of its own passes `strict`, and that fault is raised. Returns
|
|
4842
|
+
* when the footer read that answered was sent (`Infinity` when none did).
|
|
4682
4843
|
*/
|
|
4683
4844
|
async throwIfReplaced(ref, generation, version, fingerprint, strict = false) {
|
|
4684
4845
|
const held = this.heldKey(ref, version, fingerprint);
|
|
@@ -4702,6 +4863,7 @@ var CrbmStorageChunkSource = class {
|
|
|
4702
4863
|
}
|
|
4703
4864
|
);
|
|
4704
4865
|
this.checking.set(held, started);
|
|
4866
|
+
this.footerSentAt.set(started, this.now());
|
|
4705
4867
|
check = started;
|
|
4706
4868
|
}
|
|
4707
4869
|
let replaced;
|
|
@@ -4709,9 +4871,10 @@ var CrbmStorageChunkSource = class {
|
|
|
4709
4871
|
replaced = await check;
|
|
4710
4872
|
} catch (err) {
|
|
4711
4873
|
if (strict || isTransientError(err)) throw err;
|
|
4712
|
-
return;
|
|
4874
|
+
return Number.POSITIVE_INFINITY;
|
|
4713
4875
|
}
|
|
4714
4876
|
if (replaced) throw notThePinned(ref, generation);
|
|
4877
|
+
return this.footerSentAt.get(check) ?? Number.NEGATIVE_INFINITY;
|
|
4715
4878
|
}
|
|
4716
4879
|
/**
|
|
4717
4880
|
* Throws the pin's `NotFoundError` when the object under `generation`'s key is no longer the one `held` names: one
|
|
@@ -5074,7 +5237,9 @@ var CrbmStorageChunkSource = class {
|
|
|
5074
5237
|
} catch (err) {
|
|
5075
5238
|
if (held === void 0) throw err;
|
|
5076
5239
|
if (!(isIntegrityError(err) || isValidationError(err))) throw err;
|
|
5077
|
-
const check = () =>
|
|
5240
|
+
const check = async () => {
|
|
5241
|
+
await this.throwIfReplaced(ref, generation, held.version, held.fingerprint);
|
|
5242
|
+
};
|
|
5078
5243
|
await viaRetry(retry, check);
|
|
5079
5244
|
throw err;
|
|
5080
5245
|
}
|
|
@@ -5313,10 +5478,21 @@ var PinnedStorageChunkSource = class {
|
|
|
5313
5478
|
pinFor(ref) {
|
|
5314
5479
|
return this.pins.get(segmentKey(ref));
|
|
5315
5480
|
}
|
|
5481
|
+
/**
|
|
5482
|
+
* The pin's recheck when it is due, else `undefined`: a compare and no await between checks. A pinned read waits for
|
|
5483
|
+
* it before it touches the reader or the chunk cache, and fails with its `NotFoundError` once the segment is
|
|
5484
|
+
* destroyed or the pin's generation erased ({@link CrbmStorageChunkSource.recheckPin}).
|
|
5485
|
+
*/
|
|
5486
|
+
due(ref, generation, pin) {
|
|
5487
|
+
return this.inner.pinDue(ref, pin) ? this.inner.recheckPin(ref, generation, pin) : void 0;
|
|
5488
|
+
}
|
|
5316
5489
|
getChunk(ref) {
|
|
5317
5490
|
const pin = this.pinFor(ref);
|
|
5318
5491
|
if (pin === void 0) return this.inner.getChunk(ref);
|
|
5319
|
-
|
|
5492
|
+
if (pin.generation === null) return Promise.resolve(null);
|
|
5493
|
+
const generation = pin.generation;
|
|
5494
|
+
const due = this.due(ref, generation, pin);
|
|
5495
|
+
return due === void 0 ? this.inner.getChunkAt(ref, generation, heldBy(pin)) : due.then(() => this.inner.getChunkAt(ref, generation, heldBy(pin)));
|
|
5320
5496
|
}
|
|
5321
5497
|
/**
|
|
5322
5498
|
* A pinned segment's chunks come from the pinned generation alone, under the version a pinned read keys by
|
|
@@ -5337,7 +5513,10 @@ var PinnedStorageChunkSource = class {
|
|
|
5337
5513
|
this.inner.getChunksAt(ref, pin.generation, keys, heldBy(pin), options)
|
|
5338
5514
|
);
|
|
5339
5515
|
try {
|
|
5340
|
-
while (await pull.advance())
|
|
5516
|
+
while (await pull.advance()) {
|
|
5517
|
+
if (this.inner.pinDue(ref, pin)) await this.inner.recheckPin(ref, pin.generation, pin);
|
|
5518
|
+
yield pull.take((chunk) => ({ ...chunk, version }));
|
|
5519
|
+
}
|
|
5341
5520
|
} finally {
|
|
5342
5521
|
await pull.close();
|
|
5343
5522
|
}
|
|
@@ -5345,27 +5524,42 @@ var PinnedStorageChunkSource = class {
|
|
|
5345
5524
|
listChunkKeys(ref) {
|
|
5346
5525
|
const pin = this.pinFor(ref);
|
|
5347
5526
|
if (pin === void 0) return this.inner.listChunkKeys(ref);
|
|
5348
|
-
|
|
5527
|
+
if (pin.generation === null) return Promise.resolve([]);
|
|
5528
|
+
const generation = pin.generation;
|
|
5529
|
+
const due = this.due(ref, generation, pin);
|
|
5530
|
+
return due === void 0 ? this.inner.listChunkKeysAt(ref, generation, heldBy(pin)) : due.then(() => this.inner.listChunkKeysAt(ref, generation, heldBy(pin)));
|
|
5349
5531
|
}
|
|
5350
5532
|
sizeOf(ref) {
|
|
5351
5533
|
const pin = this.pinFor(ref);
|
|
5352
5534
|
if (pin === void 0) return this.inner.sizeOf(ref);
|
|
5353
|
-
|
|
5535
|
+
if (pin.generation === null) return Promise.resolve(null);
|
|
5536
|
+
const generation = pin.generation;
|
|
5537
|
+
const due = this.due(ref, generation, pin);
|
|
5538
|
+
return due === void 0 ? this.inner.sizeOfAt(ref, generation, heldBy(pin)) : due.then(() => this.inner.sizeOfAt(ref, generation, heldBy(pin)));
|
|
5354
5539
|
}
|
|
5355
5540
|
cardinalities(ref) {
|
|
5356
5541
|
const pin = this.pinFor(ref);
|
|
5357
5542
|
if (pin === void 0) return this.inner.cardinalities(ref);
|
|
5358
|
-
|
|
5543
|
+
if (pin.generation === null) return Promise.resolve(null);
|
|
5544
|
+
const generation = pin.generation;
|
|
5545
|
+
const due = this.due(ref, generation, pin);
|
|
5546
|
+
return due === void 0 ? this.inner.cardinalitiesAt(ref, generation, heldBy(pin)) : due.then(() => this.inner.cardinalitiesAt(ref, generation, heldBy(pin)));
|
|
5359
5547
|
}
|
|
5360
5548
|
summary(ref) {
|
|
5361
5549
|
const pin = this.pinFor(ref);
|
|
5362
5550
|
if (pin === void 0) return this.inner.summary(ref);
|
|
5363
|
-
|
|
5551
|
+
if (pin.generation === null) return Promise.resolve(null);
|
|
5552
|
+
const generation = pin.generation;
|
|
5553
|
+
const due = this.due(ref, generation, pin);
|
|
5554
|
+
return due === void 0 ? this.inner.summaryAt(ref, generation, heldBy(pin)) : due.then(() => this.inner.summaryAt(ref, generation, heldBy(pin)));
|
|
5364
5555
|
}
|
|
5365
5556
|
stat(ref) {
|
|
5366
5557
|
const pin = this.pinFor(ref);
|
|
5367
5558
|
if (pin === void 0) return this.inner.stat(ref);
|
|
5368
|
-
|
|
5559
|
+
if (pin.generation === null) return Promise.resolve(null);
|
|
5560
|
+
const generation = pin.generation;
|
|
5561
|
+
const due = this.due(ref, generation, pin);
|
|
5562
|
+
return due === void 0 ? this.inner.statAt(ref, generation, heldBy(pin)) : due.then(() => this.inner.statAt(ref, generation, heldBy(pin)));
|
|
5369
5563
|
}
|
|
5370
5564
|
currentGeneration(ref) {
|
|
5371
5565
|
const pin = this.pinFor(ref);
|
|
@@ -5392,7 +5586,9 @@ var PinnedStorageChunkSource = class {
|
|
|
5392
5586
|
const pin = this.pinFor(ref);
|
|
5393
5587
|
if (pin === void 0) return this.inner.currentVersion(ref);
|
|
5394
5588
|
if (pin.version === null) return Promise.resolve(null);
|
|
5395
|
-
|
|
5589
|
+
const version = `pin ${pin.version}#${pin.fingerprint}`;
|
|
5590
|
+
const due = this.due(ref, pin.generation, pin);
|
|
5591
|
+
return due === void 0 ? Promise.resolve(version) : due.then(() => version);
|
|
5396
5592
|
}
|
|
5397
5593
|
exists(ref) {
|
|
5398
5594
|
return this.inner.exists(ref);
|
|
@@ -6657,6 +6853,25 @@ async function rowDeleteSettled(registry, ref, expected, waits = {}) {
|
|
|
6657
6853
|
}
|
|
6658
6854
|
}
|
|
6659
6855
|
|
|
6856
|
+
// src/core/generation-floor.ts
|
|
6857
|
+
var STRAY_MARGIN = 1024;
|
|
6858
|
+
var MAX_REPORTED_STRAYS = 64;
|
|
6859
|
+
var NO_STRAYS = Object.freeze({
|
|
6860
|
+
count: 0,
|
|
6861
|
+
generations: Object.freeze([])
|
|
6862
|
+
});
|
|
6863
|
+
function floorOf(row) {
|
|
6864
|
+
let floor = row?.currentGen ?? -1;
|
|
6865
|
+
for (const e of row?.knownGens ?? []) if (e.generation > floor) floor = e.generation;
|
|
6866
|
+
for (const e of row?.leases ?? []) if (e.generation > floor) floor = e.generation;
|
|
6867
|
+
return floor;
|
|
6868
|
+
}
|
|
6869
|
+
function reportStrays(strays) {
|
|
6870
|
+
if (strays.length === 0) return NO_STRAYS;
|
|
6871
|
+
const generations = [...strays].sort((a, b) => a - b).slice(0, MAX_REPORTED_STRAYS);
|
|
6872
|
+
return { count: strays.length, generations };
|
|
6873
|
+
}
|
|
6874
|
+
|
|
6660
6875
|
// src/core/generation-gc.ts
|
|
6661
6876
|
function deleteGeneration(deps, ref, generation) {
|
|
6662
6877
|
const key = { namespace: ref.namespace, segment: ref.segment, generation };
|
|
@@ -6715,15 +6930,23 @@ function tookEffectOverMissing(ref, published, missing) {
|
|
|
6715
6930
|
return pointerOnDeletedError(ref, { kind: "load", published }, missing);
|
|
6716
6931
|
}
|
|
6717
6932
|
async function nextGeneration(ref, deps) {
|
|
6718
|
-
return aboveEverything(ref, deps, await deps.registry.get(ref));
|
|
6933
|
+
return (await aboveEverything(ref, deps, await deps.registry.get(ref))).generation;
|
|
6719
6934
|
}
|
|
6720
|
-
async function aboveEverything(ref, deps, record
|
|
6935
|
+
async function aboveEverything(ref, deps, record) {
|
|
6936
|
+
const all = [];
|
|
6937
|
+
for await (const key of deps.storage.list(ref)) all.push(key.generation);
|
|
6938
|
+
const limit = floorOf(record) + STRAY_MARGIN;
|
|
6939
|
+
const listed2 = all.filter((g) => g <= limit);
|
|
6721
6940
|
let highest = record?.currentGen ?? -1;
|
|
6722
|
-
for
|
|
6723
|
-
|
|
6724
|
-
|
|
6725
|
-
|
|
6726
|
-
return
|
|
6941
|
+
for (const g of listed2) if (g > highest) highest = g;
|
|
6942
|
+
const held = new Set(all);
|
|
6943
|
+
let generation = numberAbove(highest);
|
|
6944
|
+
while (held.has(generation)) generation = numberAbove(generation);
|
|
6945
|
+
return {
|
|
6946
|
+
generation,
|
|
6947
|
+
listed: listed2,
|
|
6948
|
+
strays: reportStrays(all.filter((g) => g > limit && g > generation))
|
|
6949
|
+
};
|
|
6727
6950
|
}
|
|
6728
6951
|
function numberAbove(highest) {
|
|
6729
6952
|
if (highest >= Number.MAX_SAFE_INTEGER) {
|
|
@@ -6736,10 +6959,9 @@ async function nextLoadGeneration(ref, deps, row) {
|
|
|
6736
6959
|
try {
|
|
6737
6960
|
await deps.storage.getTail({ namespace: ref.namespace, segment: ref.segment, generation }, 0);
|
|
6738
6961
|
} catch (err) {
|
|
6739
|
-
if (isNotFoundError(err)) return { generation, checked: true };
|
|
6962
|
+
if (isNotFoundError(err)) return { generation, checked: true, strays: NO_STRAYS };
|
|
6740
6963
|
}
|
|
6741
|
-
|
|
6742
|
-
return { generation: await aboveEverything(ref, deps, row, listed2), checked: false, listed: listed2 };
|
|
6964
|
+
return { ...await aboveEverything(ref, deps, row), checked: false };
|
|
6743
6965
|
}
|
|
6744
6966
|
var LIST_COLLECTION_CADENCE = 16;
|
|
6745
6967
|
async function gcOrphanGenerations(ref, deps, options = {}) {
|
|
@@ -7046,6 +7268,13 @@ async function judgeLoad(ref, cardinality, deps, options = {}) {
|
|
|
7046
7268
|
function sizeNeeded(guard, allowEmpty) {
|
|
7047
7269
|
return guard?.minRetained !== void 0 || guard?.maxGrowth !== void 0 || allowEmpty !== true;
|
|
7048
7270
|
}
|
|
7271
|
+
var WROTE_NOTHING = Object.freeze({
|
|
7272
|
+
size: 0,
|
|
7273
|
+
sha256: "",
|
|
7274
|
+
chunkCount: 0,
|
|
7275
|
+
cardinality: 0,
|
|
7276
|
+
collected: Object.freeze([])
|
|
7277
|
+
});
|
|
7049
7278
|
function lostNumberRace(err) {
|
|
7050
7279
|
return isWriteConflictError(err) && !isSourceError(err);
|
|
7051
7280
|
}
|
|
@@ -7160,7 +7389,7 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
7160
7389
|
);
|
|
7161
7390
|
} catch (err) {
|
|
7162
7391
|
if (!lostNumberRace(err)) throw err;
|
|
7163
|
-
const { generation: generation2 } = await numbering;
|
|
7392
|
+
const { generation: generation2, strays: strays2 } = await numbering;
|
|
7164
7393
|
audit.onEvent({
|
|
7165
7394
|
kind: "segment.load-refused",
|
|
7166
7395
|
segment: ref.segment,
|
|
@@ -7174,15 +7403,12 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
7174
7403
|
generation: generation2,
|
|
7175
7404
|
published: false,
|
|
7176
7405
|
reason: "superseded",
|
|
7177
|
-
|
|
7178
|
-
sha256: "",
|
|
7179
|
-
chunkCount: 0,
|
|
7180
|
-
cardinality: 0,
|
|
7406
|
+
...WROTE_NOTHING,
|
|
7181
7407
|
cardinalityBefore: before,
|
|
7182
|
-
|
|
7408
|
+
strays: strays2
|
|
7183
7409
|
};
|
|
7184
7410
|
}
|
|
7185
|
-
const { generation, checked, listed: listed2 } = await numbering;
|
|
7411
|
+
const { generation, checked, listed: listed2, strays } = await numbering;
|
|
7186
7412
|
const key = { namespace: ref.namespace, segment: ref.segment, generation };
|
|
7187
7413
|
const reclaim = async () => {
|
|
7188
7414
|
const watch = watchDeletes(deps, ref);
|
|
@@ -7225,7 +7451,8 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
7225
7451
|
chunkCount: written.chunkCount,
|
|
7226
7452
|
cardinality: written.cardinality,
|
|
7227
7453
|
cardinalityBefore: before,
|
|
7228
|
-
collected: []
|
|
7454
|
+
collected: [],
|
|
7455
|
+
strays
|
|
7229
7456
|
};
|
|
7230
7457
|
};
|
|
7231
7458
|
const refusal = guardRefusal(written.cardinality, before, guard, options.allowEmpty);
|
|
@@ -7321,7 +7548,8 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
7321
7548
|
chunkCount: written.chunkCount,
|
|
7322
7549
|
cardinality: written.cardinality,
|
|
7323
7550
|
cardinalityBefore: before,
|
|
7324
|
-
collected
|
|
7551
|
+
collected,
|
|
7552
|
+
strays
|
|
7325
7553
|
};
|
|
7326
7554
|
}
|
|
7327
7555
|
|