@cloudbitmaps/core 0.19.0 → 0.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-TNWTIAON.js → chunk-RSLVTRJY.js} +1 -1
- package/dist/{chunk-TNWTIAON.js.map → chunk-RSLVTRJY.js.map} +2 -2
- package/dist/core/combine-many.d.ts +6 -0
- package/dist/core/crbm/reader.d.ts +5 -0
- package/dist/core/crbm-storage-source.d.ts +35 -22
- package/dist/core/engine.d.ts +5 -0
- package/dist/core/erase-id.d.ts +3 -2
- package/dist/core/generation-gc.d.ts +2 -2
- package/dist/core/keys.d.ts +28 -0
- package/dist/core/load.d.ts +6 -4
- package/dist/core/ports.d.ts +4 -0
- package/dist/driver-kit.js +1 -1
- package/dist/index.js +138 -58
- package/dist/index.js.map +3 -3
- package/package.json +1 -1
|
@@ -310,3 +310,9 @@ export declare class GroupCost {
|
|
|
310
310
|
* budget, both before any chunk is read.
|
|
311
311
|
*/
|
|
312
312
|
export declare function runCombineMany<R>(compiled: CompiledCombineMany<R>, deps: CombineManyDeps): Promise<CombineManyRun<R>>;
|
|
313
|
+
/**
|
|
314
|
+
* The token of a row's version (`<generation>:<token>`, as a pin holds it), or `undefined` where the version names no
|
|
315
|
+
* row. A live version from the `.crbm` source ends with the opened object's fingerprint: read it through
|
|
316
|
+
* `rowVersionOf` first.
|
|
317
|
+
*/
|
|
318
|
+
export declare function tokenOfVersion(version: string | null | undefined): string | undefined;
|
|
@@ -51,6 +51,11 @@ export interface CrbmReaderOptions {
|
|
|
51
51
|
}
|
|
52
52
|
/** An object's fingerprint, from its size and its footer's CRC (see {@link CrbmReader.fingerprint}). */
|
|
53
53
|
export declare const fingerprintFor: (size: number, footerCrc: number) => string;
|
|
54
|
+
/**
|
|
55
|
+
* A fingerprint spelled short, for a key a read builds on every chunk: its size and its footer's CRC in base 36,
|
|
56
|
+
* `<size>.<crc>`. Two compact spellings are equal exactly when the fingerprints are.
|
|
57
|
+
*/
|
|
58
|
+
export declare function compactFingerprint(fingerprint: string): string;
|
|
54
59
|
/**
|
|
55
60
|
* Whether the object behind `blob` says it is encrypted, from one tail read of a footer's worth, with no key and no
|
|
56
61
|
* index; a footer that fails its own checks throws, as an open would. For a caller that must not open an object with
|
|
@@ -80,6 +80,15 @@ export interface PinnedObject {
|
|
|
80
80
|
readonly version: string;
|
|
81
81
|
readonly fingerprint?: string;
|
|
82
82
|
}
|
|
83
|
+
/**
|
|
84
|
+
* What a live version's resolution named, without the object's fingerprint: `<generation>:<row token>`, or
|
|
85
|
+
* `<generation>` with no registry. For a caller that compares a live version, from
|
|
86
|
+
* {@link CrbmStorageChunkSource.currentVersion} or a chunk of its `getChunks`, with a row it read, which names no
|
|
87
|
+
* object. A live version always ends with the suffix, which holds no `#`, so the match starts at its last `#` and a row
|
|
88
|
+
* token holding `#` of its own is kept whole. Known limit: a string with no suffix, such as a pin's version or one
|
|
89
|
+
* from another source, loses its end when its row token itself ends in `#<base 36>.<base 36>`; no caller passes one.
|
|
90
|
+
*/
|
|
91
|
+
export declare function rowVersionOf(version: string): string;
|
|
83
92
|
/** What a pin that leases its generation does between reading the row and opening the object. */
|
|
84
93
|
export interface PinLease {
|
|
85
94
|
/** Write the lease against the row the pin read: its token, or `'moved'` when the pointer is no longer at the generation. */
|
|
@@ -94,7 +103,8 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
94
103
|
* needs a clock). Within the TTL a segment's Storage bytes are treated as an immutable snapshot; when the TTL
|
|
95
104
|
* elapses the next read cheaply re-resolves `currentGen` and, only if it advanced (a load published),
|
|
96
105
|
* opens the new generation — so a long-lived source observes new generations within the TTL. The engine pairs
|
|
97
|
-
* this with a
|
|
106
|
+
* this with a cache keyed by {@link currentVersion}, which names the object a reader opened, so a bump never serves
|
|
107
|
+
* a stale decoded chunk, and neither does a number taken again by another object. Without a clock or a
|
|
98
108
|
* registry there is no timed refresh, and a segment's reader is replaced only when it is evicted, when a read
|
|
99
109
|
* has to fetch from a generation a sweep deleted, or when {@link invalidate} drops it. A segment with no generation yet is not memoized, so
|
|
100
110
|
* it's re-checked until one exists. **Bounded** by a {@link BoundedLru} ({@link CrbmStorageChunkSourceOptions.maxOpenSegments}, default
|
|
@@ -147,11 +157,11 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
147
157
|
private readonly timedRefresh;
|
|
148
158
|
constructor(driver: IStorageDriver, options?: CrbmStorageChunkSourceOptions);
|
|
149
159
|
/**
|
|
150
|
-
* The
|
|
151
|
-
*
|
|
152
|
-
*
|
|
160
|
+
* The keys of the segments this source reads, kept rather than encoded again: a read asks for its segment's snapshot
|
|
161
|
+
* once per operand, and a stream before every cached chunk it serves, and encoding the names cost about a
|
|
162
|
+
* microsecond each time.
|
|
153
163
|
*/
|
|
154
|
-
private readonly
|
|
164
|
+
private readonly keys;
|
|
155
165
|
private keyOf;
|
|
156
166
|
/**
|
|
157
167
|
* The segment's live snapshot, refreshing on the TTL. Cheap within the TTL window. The snapshot holds the resolved
|
|
@@ -221,13 +231,16 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
221
231
|
*/
|
|
222
232
|
invalidate(ref: SegmentRef): void;
|
|
223
233
|
/**
|
|
224
|
-
* `<generation>` for a registry-less source, or `<generation>:<row token>` with one
|
|
225
|
-
*
|
|
226
|
-
*
|
|
227
|
-
*
|
|
228
|
-
*
|
|
229
|
-
*
|
|
230
|
-
*
|
|
234
|
+
* `<generation>#<object>` for a registry-less source, or `<generation>:<row token>#<object>` with one: the generation
|
|
235
|
+
* the segment's snapshot resolved, and the object its reader opened, by that object's size and footer checksum in
|
|
236
|
+
* base 36, which the open read anyway. The token moves on every row write, so an unrelated write (a `setRetention`)
|
|
237
|
+
* costs the segment's decoded chunks once — bounded, and on an admin path. Two objects do not share a version
|
|
238
|
+
* string, as far as their sizes and footer checksums tell them apart, whichever way a number came to name a second
|
|
239
|
+
* one: a name purged and loaded again (a new row, or with no registry the same bare number), a number taken again
|
|
240
|
+
* within one row once its object was deleted, or an object put back under its key from outside the library. That
|
|
241
|
+
* holds however long after its row read a reader opens: a snapshot that a `count()` resolved from the row opens its
|
|
242
|
+
* reader only when a later read needs it, and the object under the key may have changed in between. The version is
|
|
243
|
+
* opaque: compare two for equality.
|
|
231
244
|
*
|
|
232
245
|
* This is the call the engine makes **once per operand of every read**, before any chunk fetch, to key its
|
|
233
246
|
* chunk cache, so it heals a swept generation exactly as {@link currentGeneration} does: unhealed, a cold read
|
|
@@ -583,17 +596,17 @@ export declare function publishGenerationKept(registry: IRegistryDriver, key: Ge
|
|
|
583
596
|
/**
|
|
584
597
|
* Publish only while the segment still has **no registry row**.
|
|
585
598
|
*
|
|
586
|
-
* The fence for a caller whose decision rests on the row being ABSENT
|
|
587
|
-
* cannot express it: both compare against a value read from a
|
|
588
|
-
* nothing to compare and both are simply omitted — leaving the publish a
|
|
589
|
-
* happily lands over whatever appeared in the meantime.
|
|
599
|
+
* The fence for a caller whose decision rests on the row being ABSENT: every load that found no row sets it,
|
|
600
|
+
* guarded or not. `expectFrom` and `expectToken` cannot express it: both compare against a value read from a
|
|
601
|
+
* row, so when there was no row there is nothing to compare and both are simply omitted — leaving the publish a
|
|
602
|
+
* bare forward-only advance that happily lands over whatever appeared in the meantime.
|
|
590
603
|
*
|
|
591
|
-
*
|
|
592
|
-
*
|
|
593
|
-
*
|
|
594
|
-
*
|
|
595
|
-
*
|
|
596
|
-
*
|
|
604
|
+
* Without it, a guarded load into a segment that did not exist yet would judge "no row", pass its empty and
|
|
605
|
+
* `minRetained` bounds vacuously, and land an EMPTY generation over the ids a concurrent writer had just loaded,
|
|
606
|
+
* reporting `published: true`. An unguarded load would move the pointer over another first load's generation or
|
|
607
|
+
* onto a row `setRetention` made, and, where an erasure had deleted its object above that row's pointer, onto an
|
|
608
|
+
* object that is gone. Materialising into a destination that does not exist yet is the ordinary first run of a
|
|
609
|
+
* pipeline, so the `*Into` verbs reach this as often as `loadSegment` does.
|
|
597
610
|
*/
|
|
598
611
|
expectAbsent?: boolean;
|
|
599
612
|
/**
|
package/dist/core/engine.d.ts
CHANGED
|
@@ -106,6 +106,11 @@ export declare class SegmentEngine {
|
|
|
106
106
|
private readonly openStreams;
|
|
107
107
|
/** How many times {@link invalidate} has been called: a read compares it with the count it began under. */
|
|
108
108
|
private invalidations;
|
|
109
|
+
/**
|
|
110
|
+
* The cache-key prefixes ({@link segmentPrefix}) of the segments this engine reads, kept rather than encoded for each
|
|
111
|
+
* chunk. A kept prefix is the one {@link invalidate} finds the segment's chunks by.
|
|
112
|
+
*/
|
|
113
|
+
private readonly prefixes;
|
|
109
114
|
constructor(deps: EngineDeps);
|
|
110
115
|
/** Membership: one chunk lookup — the cache, else one Storage fetch of that chunk. */
|
|
111
116
|
has(seg: SegmentRef, id: number): Promise<boolean>;
|
package/dist/core/erase-id.d.ts
CHANGED
|
@@ -70,8 +70,9 @@
|
|
|
70
70
|
* carries whatever its source held, and the library cannot know that source was meant to exclude the id.
|
|
71
71
|
* A load already in flight writes its object above the pointer before it publishes. An erasure that finds the id in that
|
|
72
72
|
* object writes the row before deleting it, so the load's publish, fenced on the row it read, is refused
|
|
73
|
-
* (`published: false`) and the pointer never names a missing object (see `fenceInFlight`).
|
|
74
|
-
*
|
|
73
|
+
* (`published: false`) and the pointer never names a missing object (see `fenceInFlight`). On a row with no pointer
|
|
74
|
+
* the erasure writes nothing to the row, so a first load's object there may still be published; an erasure that finds
|
|
75
|
+
* the id in it refuses instead of deleting it.
|
|
75
76
|
*/
|
|
76
77
|
import { type IAuditSink } from './audit.js';
|
|
77
78
|
import type { CodecInterface } from './codec.js';
|
|
@@ -45,8 +45,8 @@ export interface LoadNumber {
|
|
|
45
45
|
* `superseded`. The orphan above stays unpublished; the load whose check meets it numbers past it by the listing,
|
|
46
46
|
* and collection takes it once a generation above it is current. Like a listing's, the number can be one whose
|
|
47
47
|
* object was deleted (an erasure removes the generations above a rolled-back pointer that held the id), which is
|
|
48
|
-
* why nothing identifies a generation by its number alone: the reader cache
|
|
49
|
-
* the
|
|
48
|
+
* why nothing identifies a generation by its number alone: the reader cache and the chunk cache key on the number and
|
|
49
|
+
* the row's token, checked against the object a reader opens, and a pin on its object's fingerprint.
|
|
50
50
|
*/
|
|
51
51
|
export declare function nextLoadGeneration(ref: SegmentRef, deps: GenerationDeps, row: RegistryRecord | null): Promise<LoadNumber>;
|
|
52
52
|
/**
|
package/dist/core/keys.d.ts
CHANGED
|
@@ -1,7 +1,35 @@
|
|
|
1
1
|
import type { ChunkRef, SegmentRef } from './ports.js';
|
|
2
2
|
export declare function segmentKey(ref: SegmentRef): string;
|
|
3
3
|
export declare function segmentPrefix(ref: SegmentRef): string;
|
|
4
|
+
/**
|
|
5
|
+
* How many segments' encoded keys a {@link KeptSegmentKeys} holds before it empties: the reader cache's default
|
|
6
|
+
* segment count, a working set a long-lived reader keeps open. With 30-character names that is about 150 KB.
|
|
7
|
+
*/
|
|
8
|
+
export declare const KEPT_SEGMENT_KEYS = 1024;
|
|
9
|
+
/**
|
|
10
|
+
* Segments' encoded keys ({@link segmentKey}, or {@link segmentPrefix}: whichever `encode` gives), kept so a read that
|
|
11
|
+
* looks up chunk after chunk compares names rather than encoding them again, which costs about a third of a warm
|
|
12
|
+
* `has()`. Held by namespace and then by segment, so segments of one name in several namespaces sit side by side, and
|
|
13
|
+
* no number of segments read in turn makes every lookup encode again. Bounded: at `max` entries it empties and fills
|
|
14
|
+
* again. An encoding is the names' and nothing else, so a kept one is the one an encoding would give now.
|
|
15
|
+
*/
|
|
16
|
+
export declare class KeptSegmentKeys {
|
|
17
|
+
private readonly encode;
|
|
18
|
+
private readonly max;
|
|
19
|
+
private readonly byNamespace;
|
|
20
|
+
private kept;
|
|
21
|
+
constructor(encode: (ref: SegmentRef) => string, max?: number);
|
|
22
|
+
/** The segment's encoded key. */
|
|
23
|
+
of(ref: SegmentRef): string;
|
|
24
|
+
/** How many encoded keys are kept. */
|
|
25
|
+
get size(): number;
|
|
26
|
+
}
|
|
4
27
|
export declare function chunkRefKey(ref: ChunkRef): string;
|
|
28
|
+
/**
|
|
29
|
+
* {@link chunkRefKey}, or {@link chunkGenKey} with a `version`, from the segment's {@link segmentPrefix} already
|
|
30
|
+
* encoded: for a caller that looks up many chunks of a few segments and keeps their prefixes.
|
|
31
|
+
*/
|
|
32
|
+
export declare function chunkKeyUnder(prefix: string, chunkKey: number, version?: string | number): string;
|
|
5
33
|
/**
|
|
6
34
|
* Cache key for a chunk scoped to a specific **version** of the segment — {@link chunkRefKey} plus that
|
|
7
35
|
* version (space-delimited, injection-proof exactly as above — the name parts are already encoded). A publish bumps the version, so it naturally
|
package/dist/core/load.d.ts
CHANGED
|
@@ -169,10 +169,12 @@ export type LoadRefusal = 'empty' | 'min-cardinality' | 'min-retained' | 'max-gr
|
|
|
169
169
|
/**
|
|
170
170
|
* Another writer got there first. Either another load wrote the same generation number first, so the write-once
|
|
171
171
|
* put refused this one and it wrote nothing (`size: 0`); or the segment's registry row changed while the load was
|
|
172
|
-
* writing: another load published, a retention change, a rollback or an erasure wrote the row,
|
|
173
|
-
* deleted. Once another write has changed the row, the object stays in
|
|
174
|
-
*
|
|
175
|
-
*
|
|
172
|
+
* writing: another load published, a retention change, a rollback or an erasure wrote the row, the row was
|
|
173
|
+
* deleted, or a row appeared where the load found none. Once another write has changed the row, the object stays in
|
|
174
|
+
* the bucket, still billed, until a later load's collection deletes it (at the latest the listing a load runs
|
|
175
|
+
* every sixteenth generation) or the segment is dropped, because its generation number may by then name another
|
|
176
|
+
* incarnation's live object; once the row is `destroyed` (dropped or crypto-shredded), the object is deleted, and
|
|
177
|
+
* once the row is gone, so is an object its footer proves this load's own.
|
|
176
178
|
*/
|
|
177
179
|
| 'superseded';
|
|
178
180
|
export interface LoadResult {
|
package/dist/core/ports.d.ts
CHANGED
|
@@ -194,6 +194,10 @@ export interface StorageChunkSource {
|
|
|
194
194
|
* Note that putting the incarnation in the *object key* would not fix this: the new incarnation still starts
|
|
195
195
|
* at generation 0, so the cache key collides either way. The identity has to reach the cache.
|
|
196
196
|
*
|
|
197
|
+
* Within one incarnation a number can be taken again once its object is deleted, so two objects must never share
|
|
198
|
+
* a version either, however long after its pointer read a source opens the object: the `.crbm` source's version
|
|
199
|
+
* ends with the fingerprint of the object it opened.
|
|
200
|
+
*
|
|
197
201
|
* Returns `null` when the segment resolves to no generation. A source that only ever serves one immutable
|
|
198
202
|
* generation may omit this, and the engine falls back to the generation alone.
|
|
199
203
|
*/
|
package/dist/driver-kit.js
CHANGED
package/dist/index.js
CHANGED
|
@@ -74,7 +74,7 @@ import {
|
|
|
74
74
|
validateUserNamespace,
|
|
75
75
|
validateUserRef,
|
|
76
76
|
webCryptoEntropy
|
|
77
|
-
} from "./chunk-
|
|
77
|
+
} from "./chunk-RSLVTRJY.js";
|
|
78
78
|
|
|
79
79
|
// src/core/bit-route.ts
|
|
80
80
|
var U32_MAX = 4294967295;
|
|
@@ -329,11 +329,42 @@ function segmentKey(ref) {
|
|
|
329
329
|
function segmentPrefix(ref) {
|
|
330
330
|
return `${segmentKey(ref)}${FIELD}`;
|
|
331
331
|
}
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
332
|
+
var KEPT_SEGMENT_KEYS = 1024;
|
|
333
|
+
var KeptSegmentKeys = class {
|
|
334
|
+
constructor(encode, max = KEPT_SEGMENT_KEYS) {
|
|
335
|
+
this.encode = encode;
|
|
336
|
+
this.max = max;
|
|
337
|
+
}
|
|
338
|
+
encode;
|
|
339
|
+
max;
|
|
340
|
+
byNamespace = /* @__PURE__ */ new Map();
|
|
341
|
+
kept = 0;
|
|
342
|
+
/** The segment's encoded key. */
|
|
343
|
+
of(ref) {
|
|
344
|
+
let names = this.byNamespace.get(ref.namespace);
|
|
345
|
+
const found = names?.get(ref.segment);
|
|
346
|
+
if (found !== void 0) return found;
|
|
347
|
+
const encoded = this.encode(ref);
|
|
348
|
+
if (this.kept >= this.max) {
|
|
349
|
+
this.byNamespace.clear();
|
|
350
|
+
this.kept = 0;
|
|
351
|
+
names = void 0;
|
|
352
|
+
}
|
|
353
|
+
if (names === void 0) {
|
|
354
|
+
names = /* @__PURE__ */ new Map();
|
|
355
|
+
this.byNamespace.set(ref.namespace, names);
|
|
356
|
+
}
|
|
357
|
+
names.set(ref.segment, encoded);
|
|
358
|
+
this.kept += 1;
|
|
359
|
+
return encoded;
|
|
360
|
+
}
|
|
361
|
+
/** How many encoded keys are kept. */
|
|
362
|
+
get size() {
|
|
363
|
+
return this.kept;
|
|
364
|
+
}
|
|
365
|
+
};
|
|
366
|
+
function chunkKeyUnder(prefix, chunkKey, version) {
|
|
367
|
+
return version === void 0 ? `${prefix}${chunkKey}` : `${prefix}${chunkKey}${FIELD}${version}`;
|
|
337
368
|
}
|
|
338
369
|
function shardOf(key, totalShards) {
|
|
339
370
|
let h = 2166136261;
|
|
@@ -619,6 +650,11 @@ var SegmentEngine = class {
|
|
|
619
650
|
openStreams = /* @__PURE__ */ new Set();
|
|
620
651
|
/** How many times {@link invalidate} has been called: a read compares it with the count it began under. */
|
|
621
652
|
invalidations = 0;
|
|
653
|
+
/**
|
|
654
|
+
* The cache-key prefixes ({@link segmentPrefix}) of the segments this engine reads, kept rather than encoded for each
|
|
655
|
+
* chunk. A kept prefix is the one {@link invalidate} finds the segment's chunks by.
|
|
656
|
+
*/
|
|
657
|
+
prefixes = new KeptSegmentKeys(segmentPrefix);
|
|
622
658
|
constructor(deps) {
|
|
623
659
|
this.storage = deps.storage;
|
|
624
660
|
this.cache = deps.cache;
|
|
@@ -1306,7 +1342,7 @@ var SegmentEngine = class {
|
|
|
1306
1342
|
}
|
|
1307
1343
|
/** The cache key of a chunk: by the version it was read under, or by segment and key alone for a source with none. */
|
|
1308
1344
|
chunkCacheKey(ref, version) {
|
|
1309
|
-
return
|
|
1345
|
+
return chunkKeyUnder(this.prefixes.of(ref), ref.chunkKey, version);
|
|
1310
1346
|
}
|
|
1311
1347
|
/** The segment's chunk keys, ascending, checked as untrusted tier data. */
|
|
1312
1348
|
async chunkKeys(seg) {
|
|
@@ -3397,6 +3433,12 @@ function magicMatches(bytes, offset) {
|
|
|
3397
3433
|
}
|
|
3398
3434
|
var sizePart = (size) => `${size}:`;
|
|
3399
3435
|
var fingerprintFor = (size, footerCrc) => `${sizePart(size)}${footerCrc}`;
|
|
3436
|
+
function compactFingerprint(fingerprint) {
|
|
3437
|
+
const colon = fingerprint.indexOf(":");
|
|
3438
|
+
const size = Number(fingerprint.slice(0, colon));
|
|
3439
|
+
const crc = Number(fingerprint.slice(colon + 1));
|
|
3440
|
+
return `${size.toString(36)}.${crc.toString(36)}`;
|
|
3441
|
+
}
|
|
3400
3442
|
function checkSize(tail, size) {
|
|
3401
3443
|
if (!Number.isSafeInteger(size) || size < tail.length) {
|
|
3402
3444
|
throw new IntegrityError(`.crbm size is not a byte count its tail fits in: ${size}`);
|
|
@@ -4514,6 +4556,19 @@ var notThePinned = (ref, generation) => new NotFoundError(
|
|
|
4514
4556
|
function versionOf(generation, lineage) {
|
|
4515
4557
|
return lineage === void 0 ? String(generation) : `${generation}:${String(lineage)}`;
|
|
4516
4558
|
}
|
|
4559
|
+
var readerVersions = /* @__PURE__ */ new WeakMap();
|
|
4560
|
+
function versionOfReader(reader) {
|
|
4561
|
+
let version = readerVersions.get(reader);
|
|
4562
|
+
if (version === void 0) {
|
|
4563
|
+
version = `${versionOf(reader.generation, reader.lineage)}#${compactFingerprint(reader.fingerprint)}`;
|
|
4564
|
+
readerVersions.set(reader, version);
|
|
4565
|
+
}
|
|
4566
|
+
return version;
|
|
4567
|
+
}
|
|
4568
|
+
var OBJECT_SUFFIX = /#[0-9a-z]+\.[0-9a-z]+$/;
|
|
4569
|
+
function rowVersionOf(version) {
|
|
4570
|
+
return version.replace(OBJECT_SUFFIX, "");
|
|
4571
|
+
}
|
|
4517
4572
|
var PinMoved = class extends Error {
|
|
4518
4573
|
};
|
|
4519
4574
|
function summaryOfGeneration(generation, description) {
|
|
@@ -4535,13 +4590,14 @@ function once(fn) {
|
|
|
4535
4590
|
};
|
|
4536
4591
|
}
|
|
4537
4592
|
var readerIsOf = (reader, live) => reader.generation === live.target.generation && reader.lineage === live.target.lineage;
|
|
4538
|
-
async function
|
|
4539
|
-
|
|
4540
|
-
|
|
4541
|
-
|
|
4593
|
+
async function readsObjectOf(snap, reader) {
|
|
4594
|
+
const settled2 = snap.settled;
|
|
4595
|
+
if (settled2 !== void 0) {
|
|
4596
|
+
const other = settled2.reader;
|
|
4597
|
+
return other !== null && (other === reader || versionOfReader(other) === versionOfReader(reader));
|
|
4542
4598
|
}
|
|
4543
4599
|
const live = await snap.target;
|
|
4544
|
-
return live
|
|
4600
|
+
return live !== null && versionOf(live.target.generation, live.target.lineage) === versionOf(reader.generation, reader.lineage);
|
|
4545
4601
|
}
|
|
4546
4602
|
function describe(reader) {
|
|
4547
4603
|
return { cardinality: reader.count(), metadata: reader.metadata };
|
|
@@ -4560,32 +4616,57 @@ var Snapshot = class _Snapshot {
|
|
|
4560
4616
|
/** Set by the cache, to hear of a reader as it is opened. */
|
|
4561
4617
|
onReader;
|
|
4562
4618
|
opened;
|
|
4619
|
+
outcome;
|
|
4563
4620
|
static live(target, open4, prior) {
|
|
4564
4621
|
return new _Snapshot(target, open4, prior);
|
|
4565
4622
|
}
|
|
4566
4623
|
static eager(reader) {
|
|
4567
4624
|
const snap = new _Snapshot(void 0, void 0, void 0);
|
|
4568
|
-
snap.
|
|
4625
|
+
snap.hold(reader);
|
|
4569
4626
|
return snap;
|
|
4570
4627
|
}
|
|
4571
4628
|
/** The reader, opened by the first call. */
|
|
4572
4629
|
get reader() {
|
|
4573
4630
|
if (this.opened === void 0) {
|
|
4574
4631
|
const target = this.target;
|
|
4575
|
-
|
|
4576
|
-
|
|
4577
|
-
|
|
4578
|
-
|
|
4579
|
-
|
|
4580
|
-
|
|
4581
|
-
|
|
4632
|
+
const opened = this.hold(
|
|
4633
|
+
target.then(async (live) => {
|
|
4634
|
+
if (live === null) return null;
|
|
4635
|
+
const kept = await this.reuse(live);
|
|
4636
|
+
this.prior = void 0;
|
|
4637
|
+
return kept ?? this.open(live);
|
|
4638
|
+
})
|
|
4639
|
+
);
|
|
4640
|
+
this.onReader?.(opened);
|
|
4641
|
+
return opened;
|
|
4582
4642
|
}
|
|
4583
4643
|
return this.opened;
|
|
4584
4644
|
}
|
|
4645
|
+
/**
|
|
4646
|
+
* Take `reader` as this snapshot's, and note the reader once it has opened. A failed open is not noted: the reader
|
|
4647
|
+
* cache forgets the snapshot as the failure lands, and whoever awaits the reader meets the error.
|
|
4648
|
+
*/
|
|
4649
|
+
hold(reader) {
|
|
4650
|
+
this.opened = reader;
|
|
4651
|
+
reader.then(
|
|
4652
|
+
(r) => {
|
|
4653
|
+
this.outcome = { reader: r };
|
|
4654
|
+
},
|
|
4655
|
+
() => void 0
|
|
4656
|
+
);
|
|
4657
|
+
return reader;
|
|
4658
|
+
}
|
|
4585
4659
|
/** The reader if one has been asked for, without asking for one. */
|
|
4586
4660
|
get openedReader() {
|
|
4587
4661
|
return this.opened;
|
|
4588
4662
|
}
|
|
4663
|
+
/**
|
|
4664
|
+
* The reader once it has opened (`null` for no generation). Undefined while no reader has been asked for, while the
|
|
4665
|
+
* open is under way, and after an open that failed.
|
|
4666
|
+
*/
|
|
4667
|
+
get settled() {
|
|
4668
|
+
return this.outcome;
|
|
4669
|
+
}
|
|
4589
4670
|
/** The prior snapshot's reader, if it is of what this snapshot resolved. */
|
|
4590
4671
|
async reuse(live) {
|
|
4591
4672
|
const prior = this.prior;
|
|
@@ -4598,15 +4679,13 @@ var Snapshot = class _Snapshot {
|
|
|
4598
4679
|
const reader = await this.reuse(live);
|
|
4599
4680
|
this.prior = void 0;
|
|
4600
4681
|
if (reader !== null && this.opened === void 0) {
|
|
4601
|
-
this.
|
|
4602
|
-
this.onReader?.(this.opened);
|
|
4682
|
+
this.onReader?.(this.hold(Promise.resolve(reader)));
|
|
4603
4683
|
}
|
|
4604
4684
|
}
|
|
4605
4685
|
};
|
|
4606
4686
|
var SNAPSHOT_BASE_BYTES = 256;
|
|
4607
4687
|
var BULK_FLUSH_IDS = 1 << 20;
|
|
4608
4688
|
var YIELD_EVERY_IDS = 1 << 14;
|
|
4609
|
-
var RECENT_KEYS = 8;
|
|
4610
4689
|
var CrbmStorageChunkSource = class {
|
|
4611
4690
|
constructor(driver, options = {}) {
|
|
4612
4691
|
this.driver = driver;
|
|
@@ -4666,7 +4745,8 @@ var CrbmStorageChunkSource = class {
|
|
|
4666
4745
|
* needs a clock). Within the TTL a segment's Storage bytes are treated as an immutable snapshot; when the TTL
|
|
4667
4746
|
* elapses the next read cheaply re-resolves `currentGen` and, only if it advanced (a load published),
|
|
4668
4747
|
* opens the new generation — so a long-lived source observes new generations within the TTL. The engine pairs
|
|
4669
|
-
* this with a
|
|
4748
|
+
* this with a cache keyed by {@link currentVersion}, which names the object a reader opened, so a bump never serves
|
|
4749
|
+
* a stale decoded chunk, and neither does a number taken again by another object. Without a clock or a
|
|
4670
4750
|
* registry there is no timed refresh, and a segment's reader is replaced only when it is evicted, when a read
|
|
4671
4751
|
* has to fetch from a generation a sweep deleted, or when {@link invalidate} drops it. A segment with no generation yet is not memoized, so
|
|
4672
4752
|
* it's re-checked until one exists. **Bounded** by a {@link BoundedLru} ({@link CrbmStorageChunkSourceOptions.maxOpenSegments}, default
|
|
@@ -4718,19 +4798,13 @@ var CrbmStorageChunkSource = class {
|
|
|
4718
4798
|
*/
|
|
4719
4799
|
timedRefresh;
|
|
4720
4800
|
/**
|
|
4721
|
-
* The
|
|
4722
|
-
*
|
|
4723
|
-
*
|
|
4801
|
+
* The keys of the segments this source reads, kept rather than encoded again: a read asks for its segment's snapshot
|
|
4802
|
+
* once per operand, and a stream before every cached chunk it serves, and encoding the names cost about a
|
|
4803
|
+
* microsecond each time.
|
|
4724
4804
|
*/
|
|
4725
|
-
|
|
4805
|
+
keys = new KeptSegmentKeys(segmentKey);
|
|
4726
4806
|
keyOf(ref) {
|
|
4727
|
-
|
|
4728
|
-
if (k.segment === ref.segment && k.namespace === ref.namespace) return k.key;
|
|
4729
|
-
}
|
|
4730
|
-
const key = segmentKey(ref);
|
|
4731
|
-
this.recentKeys.unshift({ namespace: ref.namespace, segment: ref.segment, key });
|
|
4732
|
-
if (this.recentKeys.length > RECENT_KEYS) this.recentKeys.pop();
|
|
4733
|
-
return key;
|
|
4807
|
+
return this.keys.of(ref);
|
|
4734
4808
|
}
|
|
4735
4809
|
/**
|
|
4736
4810
|
* The segment's live snapshot, refreshing on the TTL. Cheap within the TTL window. The snapshot holds the resolved
|
|
@@ -4889,13 +4963,16 @@ var CrbmStorageChunkSource = class {
|
|
|
4889
4963
|
for (const k of this.checking.keys()) if (k.startsWith(pinnedPrefix)) this.checking.delete(k);
|
|
4890
4964
|
}
|
|
4891
4965
|
/**
|
|
4892
|
-
* `<generation>` for a registry-less source, or `<generation>:<row token>` with one
|
|
4893
|
-
*
|
|
4894
|
-
*
|
|
4895
|
-
*
|
|
4896
|
-
*
|
|
4897
|
-
*
|
|
4898
|
-
*
|
|
4966
|
+
* `<generation>#<object>` for a registry-less source, or `<generation>:<row token>#<object>` with one: the generation
|
|
4967
|
+
* the segment's snapshot resolved, and the object its reader opened, by that object's size and footer checksum in
|
|
4968
|
+
* base 36, which the open read anyway. The token moves on every row write, so an unrelated write (a `setRetention`)
|
|
4969
|
+
* costs the segment's decoded chunks once — bounded, and on an admin path. Two objects do not share a version
|
|
4970
|
+
* string, as far as their sizes and footer checksums tell them apart, whichever way a number came to name a second
|
|
4971
|
+
* one: a name purged and loaded again (a new row, or with no registry the same bare number), a number taken again
|
|
4972
|
+
* within one row once its object was deleted, or an object put back under its key from outside the library. That
|
|
4973
|
+
* holds however long after its row read a reader opens: a snapshot that a `count()` resolved from the row opens its
|
|
4974
|
+
* reader only when a later read needs it, and the object under the key may have changed in between. The version is
|
|
4975
|
+
* opaque: compare two for equality.
|
|
4899
4976
|
*
|
|
4900
4977
|
* This is the call the engine makes **once per operand of every read**, before any chunk fetch, to key its
|
|
4901
4978
|
* chunk cache, so it heals a swept generation exactly as {@link currentGeneration} does: unhealed, a cold read
|
|
@@ -4913,7 +4990,7 @@ var CrbmStorageChunkSource = class {
|
|
|
4913
4990
|
this.dropStale(segmentKey(ref), snap);
|
|
4914
4991
|
reader = await this.resolvedReader(ref);
|
|
4915
4992
|
}
|
|
4916
|
-
return reader === null ? null :
|
|
4993
|
+
return reader === null ? null : versionOfReader(reader);
|
|
4917
4994
|
}
|
|
4918
4995
|
/**
|
|
4919
4996
|
* Resolve the segment **once** and report what a pin should hold: the generation, and the version that
|
|
@@ -5405,7 +5482,8 @@ var CrbmStorageChunkSource = class {
|
|
|
5405
5482
|
}
|
|
5406
5483
|
return;
|
|
5407
5484
|
}
|
|
5408
|
-
const version =
|
|
5485
|
+
const version = versionOfReader(reader);
|
|
5486
|
+
let seen = snap.settled;
|
|
5409
5487
|
chunks = reader.readChunks(keys.slice(yielded), {
|
|
5410
5488
|
...options?.concurrency === void 0 ? {} : { concurrency: options.concurrency },
|
|
5411
5489
|
...options?.ramp === void 0 ? {} : { ramp: options.ramp },
|
|
@@ -5417,22 +5495,24 @@ var CrbmStorageChunkSource = class {
|
|
|
5417
5495
|
let movedOn = false;
|
|
5418
5496
|
try {
|
|
5419
5497
|
while (await pull.advance()) {
|
|
5420
|
-
if (this.mayHaveMoved(key, snap, epoch)) {
|
|
5498
|
+
if (this.mayHaveMoved(key, snap, epoch) || snap.settled !== seen) {
|
|
5421
5499
|
const live = this.liveSnapshot(ref);
|
|
5422
|
-
if (live !== snap) {
|
|
5423
|
-
|
|
5500
|
+
if (live !== snap || live.settled !== seen) {
|
|
5501
|
+
const looked = live.settled;
|
|
5502
|
+
let same;
|
|
5424
5503
|
try {
|
|
5425
|
-
|
|
5504
|
+
same = await readsObjectOf(live, reader);
|
|
5426
5505
|
} catch (err) {
|
|
5427
5506
|
if (!isTransientError(err)) throw err;
|
|
5428
5507
|
movedOn = true;
|
|
5429
5508
|
break;
|
|
5430
5509
|
}
|
|
5431
|
-
if (
|
|
5510
|
+
if (!same) {
|
|
5432
5511
|
movedOn = true;
|
|
5433
5512
|
break;
|
|
5434
5513
|
}
|
|
5435
5514
|
snap = live;
|
|
5515
|
+
seen = looked;
|
|
5436
5516
|
}
|
|
5437
5517
|
epoch = this.invalidations;
|
|
5438
5518
|
}
|
|
@@ -6674,14 +6754,14 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
6674
6754
|
cleartext: !written.encrypted,
|
|
6675
6755
|
...fromToken === void 0 ? {} : { expectToken: fromToken, expectRow: row },
|
|
6676
6756
|
...needsBefore && fromGeneration !== void 0 ? { expectFrom: fromGeneration } : {},
|
|
6677
|
-
//
|
|
6678
|
-
//
|
|
6679
|
-
//
|
|
6680
|
-
//
|
|
6681
|
-
//
|
|
6682
|
-
//
|
|
6683
|
-
//
|
|
6684
|
-
...
|
|
6757
|
+
// A load that found NO ROW fences on that absence, guarded or not, so a row that appeared since refuses it.
|
|
6758
|
+
// `expectFrom` and `expectToken` both compare against a value read from a row, so with no row both are omitted,
|
|
6759
|
+
// and the publish would otherwise be a bare forward-only advance over whatever appeared in between. For a guarded
|
|
6760
|
+
// load, `before` was `null` and its bounds passed vacuously, so an empty generation would land over the ids
|
|
6761
|
+
// another writer had just loaded. For an unguarded one, the pointer would move over another first load's
|
|
6762
|
+
// generation or onto a row `setRetention` made, and, where an erasure had already deleted this load's object
|
|
6763
|
+
// above that row's pointer, onto an object that is gone, so that every read of the segment would fail.
|
|
6764
|
+
...row === null ? { expectAbsent: true } : {},
|
|
6685
6765
|
// A registry write that fails without an answer is reconciled by reading the row: the pointer at this number
|
|
6686
6766
|
// is this load's publish only over the object this load wrote, which one footer read proves.
|
|
6687
6767
|
holdsOwnObject: () => provesOwnObject(deps.storage, key, written.fingerprint),
|
|
@@ -8669,7 +8749,7 @@ function pinnedStillCurrent(spec, now) {
|
|
|
8669
8749
|
}
|
|
8670
8750
|
function liveMoved(op) {
|
|
8671
8751
|
if (op.endRow === void 0 || !op.read) return void 0;
|
|
8672
|
-
const was = tokenOfVersion(op.startVersion);
|
|
8752
|
+
const was = op.startVersion == null ? void 0 : tokenOfVersion(rowVersionOf(op.startVersion));
|
|
8673
8753
|
if (was !== void 0 && op.endRow !== null && !sameRow2(was, op.endRow.token)) return true;
|
|
8674
8754
|
return op.startGeneration === void 0 ? void 0 : op.endGeneration !== op.startGeneration;
|
|
8675
8755
|
}
|