@cloudbitmaps/core 0.19.1 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +85 -0
- package/dist/{chunk-RSLVTRJY.js → chunk-IGLIZBR7.js} +982 -281
- package/dist/chunk-IGLIZBR7.js.map +7 -0
- package/dist/core/absent-operands.d.ts +8 -0
- package/dist/core/audit.d.ts +67 -4
- package/dist/core/bit-route.d.ts +2 -1
- package/dist/core/blob.d.ts +12 -1
- package/dist/core/bulk-load.d.ts +148 -0
- package/dist/core/cas-retry.d.ts +27 -0
- package/dist/core/check.d.ts +22 -0
- package/dist/core/chunk-checks.d.ts +6 -1
- package/dist/core/chunk-window.d.ts +11 -4
- package/dist/core/combine-expr.d.ts +1 -3
- package/dist/core/combine-feed.d.ts +2 -16
- package/dist/core/combine-many.d.ts +21 -15
- package/dist/core/concurrency.d.ts +8 -0
- package/dist/core/consistency.d.ts +76 -18
- package/dist/core/crbm/fingerprint.d.ts +23 -0
- package/dist/core/crbm/format.d.ts +2 -5
- package/dist/core/crbm/frame.d.ts +10 -0
- package/dist/core/crbm/plan-reads.d.ts +9 -3
- package/dist/core/crbm/reader.d.ts +51 -12
- package/dist/core/crbm/varint.d.ts +2 -1
- package/dist/core/crbm/writer.d.ts +5 -1
- package/dist/core/crbm-open.d.ts +154 -0
- package/dist/core/crbm-storage-source.d.ts +243 -490
- package/dist/core/crbm-write.d.ts +75 -0
- package/dist/core/crypto.d.ts +6 -1
- package/dist/core/engine.d.ts +239 -47
- package/dist/core/erase-id.d.ts +97 -27
- package/dist/core/erasure.d.ts +4 -3
- package/dist/core/errors.d.ts +8 -2
- package/dist/core/generation-gc.d.ts +131 -16
- package/dist/core/generation-list.d.ts +11 -0
- package/dist/core/id-window.d.ts +46 -0
- package/dist/core/invalidation-scope.d.ts +77 -0
- package/dist/core/known-generations-shape.d.ts +13 -0
- package/dist/core/known-generations.d.ts +147 -0
- package/dist/core/leases.d.ts +9 -1
- package/dist/core/ledger-seed.d.ts +17 -0
- package/dist/core/load.d.ts +17 -36
- package/dist/core/lru.d.ts +2 -15
- package/dist/core/metrics.d.ts +9 -7
- package/dist/core/order.d.ts +2 -0
- package/dist/core/pinned-storage-source.d.ts +7 -7
- package/dist/core/pointer-id.d.ts +19 -0
- package/dist/core/ports.d.ts +210 -45
- package/dist/core/publish.d.ts +207 -0
- package/dist/core/reader-defaults.d.ts +14 -3
- package/dist/core/registry-scan.d.ts +0 -2
- package/dist/core/retention-sweep.d.ts +69 -65
- package/dist/core/retention.d.ts +1 -1
- package/dist/core/retry.d.ts +12 -19
- package/dist/core/rollback.d.ts +62 -32
- package/dist/core/summary.d.ts +74 -25
- package/dist/core/token.d.ts +13 -8
- package/dist/core/validate.d.ts +2 -0
- package/dist/driver-kit.d.ts +6 -1
- package/dist/driver-kit.js +90 -10
- package/dist/driver-kit.js.map +3 -3
- package/dist/drivers/_shared/bytes.d.ts +6 -0
- package/dist/drivers/_shared/faults.d.ts +15 -0
- package/dist/drivers/_shared/object-keys.d.ts +18 -0
- package/dist/drivers/_shared/object-registry-keys.d.ts +0 -2
- package/dist/drivers/_shared/object-registry.d.ts +3 -1
- package/dist/drivers/_shared/options.d.ts +19 -0
- package/dist/drivers/_shared/registry.d.ts +31 -19
- package/dist/drivers/localfs/exact-case.d.ts +0 -3
- package/dist/drivers/localfs/fs-util.d.ts +22 -0
- package/dist/drivers/localfs/registry.d.ts +10 -4
- package/dist/drivers/localfs/storage.d.ts +16 -8
- package/dist/drivers/memory.d.ts +14 -9
- package/dist/drivers/retry/retrying-chunk-source.d.ts +1 -5
- package/dist/export/index.d.ts +16 -0
- package/dist/index.d.ts +2 -5
- package/dist/index.js +4303 -2938
- package/dist/index.js.map +4 -4
- package/package.json +1 -1
- package/dist/chunk-RSLVTRJY.js.map +0 -7
- package/dist/core/cost.d.ts +0 -351
- package/dist/core/kept-generations.d.ts +0 -40
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What the engines reading through one chunk cache share about invalidations: how many there have been, the segment
|
|
3
|
+
* each of the latest was of, and the chunk reads open right now. A store's own engine, each pin's and each combine's
|
|
4
|
+
* with a pinned operand read through one cache, and an invalidation through any of them must reach a read running on
|
|
5
|
+
* another: a read that had its bytes before the invalidation would otherwise write them back into the cache it
|
|
6
|
+
* shares, and later reads would be answered from them, an erased id included, until the cache evicted them.
|
|
7
|
+
*
|
|
8
|
+
* ```
|
|
9
|
+
* engine (store) ──┐
|
|
10
|
+
* engine (pin) ──┼── one InvalidationScope ── epoch · latest invalidation of each segment · open reads
|
|
11
|
+
* engine (combine)─┘ └── one chunk cache
|
|
12
|
+
* ```
|
|
13
|
+
*
|
|
14
|
+
* A stream records the epoch it opened at and asks {@link InvalidationRecord.invalidatedSince} before it caches a
|
|
15
|
+
* chunk, so the scope holds no reference to a stream: a read its caller drops without closing holds nothing here. The
|
|
16
|
+
* `.crbm` source keeps an {@link InvalidationRecord} of its own, by segment, which a pinned stream asks before it hands
|
|
17
|
+
* out a chunk, so only its own segment's invalidation makes it open its object again.
|
|
18
|
+
*
|
|
19
|
+
* Internal to core: not exported from any entry point.
|
|
20
|
+
*/
|
|
21
|
+
import type { CodecBitmap } from './codec.js';
|
|
22
|
+
/**
|
|
23
|
+
* How many segments' latest invalidation a record remembers. Past it the oldest is forgotten, and a segment the record
|
|
24
|
+
* does not remember, never-invalidated ones included, counts as invalidated at the newest invalidation it has
|
|
25
|
+
* forgotten, which is always on the safe side. In the engines' scope it means a stream that opened before then caches
|
|
26
|
+
* no more of that segment's chunks, which costs it cache entries, never correctness: so a sweep that invalidates more
|
|
27
|
+
* than this many segments at once (a `retireExpired` or an `eraseSubject` over a large fleet) stops every stream open
|
|
28
|
+
* at the time caching, whatever segment it reads. In the `.crbm` source it means a pinned stream looks its pin up
|
|
29
|
+
* again, which costs a request only where the reader cache has let the pin's reader go. With names at their
|
|
30
|
+
* 256-character limit a record is about 0.6 MB, and about 0.1 MB with 30-character names; a store keeps two.
|
|
31
|
+
*/
|
|
32
|
+
export declare const REMEMBERED_INVALIDATIONS = 1024;
|
|
33
|
+
/** A chunk read in flight: `token` tells the read whether its entry is still the registered one. */
|
|
34
|
+
export interface OpenRead {
|
|
35
|
+
readonly token: object;
|
|
36
|
+
readonly read: Promise<CodecBitmap | null>;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* How many invalidations there have been, and the epoch of each of the latest {@link REMEMBERED_INVALIDATIONS}
|
|
40
|
+
* segments' latest one, by whatever key names a segment to its owner: a cache-key prefix in the engines' scope, a
|
|
41
|
+
* segment key in the `.crbm` source.
|
|
42
|
+
*/
|
|
43
|
+
export declare class InvalidationRecord {
|
|
44
|
+
private count;
|
|
45
|
+
/** The epoch of each remembered segment's latest invalidation, the oldest first. */
|
|
46
|
+
private readonly latest;
|
|
47
|
+
/** The epoch of the newest invalidation the record has forgotten: what a segment it does not remember counts as. */
|
|
48
|
+
private floor;
|
|
49
|
+
/** How many invalidations there have been: a read compares it with the epoch it began under. */
|
|
50
|
+
get epoch(): number;
|
|
51
|
+
/** Record an invalidation of the segment `key` names. */
|
|
52
|
+
invalidate(key: string): void;
|
|
53
|
+
/** Whether the segment `key` names has been invalidated since epoch `since`. */
|
|
54
|
+
invalidatedSince(key: string, since: number): boolean;
|
|
55
|
+
}
|
|
56
|
+
/** The engines' record, by cache-key prefix, and the chunk reads open on all of them. */
|
|
57
|
+
export declare class InvalidationScope extends InvalidationRecord {
|
|
58
|
+
/**
|
|
59
|
+
* The chunk reads open right now, by cache key: a caller that misses the cache and finds its key here awaits that
|
|
60
|
+
* read instead of making its own. An entry lives from the request until it settles, so the map holds at most one
|
|
61
|
+
* promise per distinct key in flight. An invalidation drops the segment's entries, and a read whose entry is gone
|
|
62
|
+
* caches nothing.
|
|
63
|
+
*/
|
|
64
|
+
private readonly openReads;
|
|
65
|
+
/** Record an invalidation of the segment whose cache keys start with `prefix`, and drop its open reads. */
|
|
66
|
+
invalidate(prefix: string): void;
|
|
67
|
+
/** The read of `key` open now, for a caller that misses the cache to await instead of making its own. */
|
|
68
|
+
openRead(key: string): Promise<CodecBitmap | null> | undefined;
|
|
69
|
+
/** Whether a read of `key` is open, without joining it. */
|
|
70
|
+
hasOpenRead(key: string): boolean;
|
|
71
|
+
/** Register `entry` as the open read of `key`. */
|
|
72
|
+
beginRead(key: string, entry: OpenRead): void;
|
|
73
|
+
/** `entry` has settled: forget it, unless an invalidation dropped it and a newer read has taken the key. */
|
|
74
|
+
endRead(key: string, entry: OpenRead): void;
|
|
75
|
+
/** Whether the read `token` names is still the open read of `key`: one an invalidation dropped caches nothing. */
|
|
76
|
+
isOpen(key: string, token: object): boolean;
|
|
77
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The shape of a row's known-generations ledger (`knownGens`), in one place. A registry holds a row to it when it writes
|
|
3
|
+
* and when it reads back, and `checkConsistency` holds a row to it before it reads or walks a ledger a registry of your
|
|
4
|
+
* own handed back, so the two cannot part on what a ledger is. Pure: no I/O, time or randomness.
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Why `value` is not a ledger, or `undefined` when it is one (an absent ledger, `undefined`, passes: whether a row
|
|
8
|
+
* without one is usable is another rule). A ledger is an array of at most `max` entries, each exactly `{ generation,
|
|
9
|
+
* footerCrc }`, the generation a non-negative safe integer and unique, the checksum an integer from 0 to 2^32 - 1. The
|
|
10
|
+
* order is not checked: it is not derivable from the numbers. `show` renders a value for the description, which a
|
|
11
|
+
* caller can truncate.
|
|
12
|
+
*/
|
|
13
|
+
export declare function knownGensShapeFault(value: unknown, max: number, show?: (v: unknown) => string): string | undefined;
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import type { KnownGeneration, RegistryRecord } from './ports.js';
|
|
2
|
+
/**
|
|
3
|
+
* The most generations a load that collects by name records as its window. A `keep` above it collects by listing.
|
|
4
|
+
*/
|
|
5
|
+
export declare const MAX_KEPT_GENERATIONS = 64;
|
|
6
|
+
/**
|
|
7
|
+
* A writer keeps entries that nothing protects and no lease holds only while the ledger holds fewer than this many.
|
|
8
|
+
*/
|
|
9
|
+
export declare const KNOWN_FILL = 128;
|
|
10
|
+
/** The most entries a writer writes, and the most the shared validator accepts at a write. */
|
|
11
|
+
export declare const MAX_KNOWN_GENERATIONS = 192;
|
|
12
|
+
/** The most entries a stored row may hold: twice what a writer writes. */
|
|
13
|
+
export declare const MAX_STORED_KNOWN_GENERATIONS = 384;
|
|
14
|
+
/**
|
|
15
|
+
* The row's ledger when a collection or a rollback may rely on it: present and not empty, on an active row with a
|
|
16
|
+
* pointer, its last entry the pointer, and, where the row's summary is in the clear and describes that generation, its
|
|
17
|
+
* checksum the one the summary's fingerprint carries. A well-formed ledger that disagrees with its row is read but not
|
|
18
|
+
* used (the row counts as one with no ledger), as a summary that names another generation is.
|
|
19
|
+
*/
|
|
20
|
+
export declare function usableKnownGens(record: RegistryRecord | null): readonly KnownGeneration[] | undefined;
|
|
21
|
+
/**
|
|
22
|
+
* The generations the row keeps as its window: the numbers of the usable ledger's entries below the pointer, in the order
|
|
23
|
+
* they were made current, or `undefined` when the ledger is not usable. A collection compares against these. An entry
|
|
24
|
+
* numbered above the pointer is one a rollback left; no collection can take it, so it is no part of the window.
|
|
25
|
+
*/
|
|
26
|
+
export declare function windowOf(record: RegistryRecord | null): readonly number[] | undefined;
|
|
27
|
+
/**
|
|
28
|
+
* Whether the row's ledger is a window a load with a small `keep` can extend: usable, and naming no more than
|
|
29
|
+
* {@link MAX_KEPT_GENERATIONS} generations below the pointer that no live lease holds. A ledger carried through
|
|
30
|
+
* publishes with a larger `keep` (`*Into`, `materializeMany`) names up to 127, and a count by a small `keep` would keep
|
|
31
|
+
* fewer than it names, so the next small-`keep` load collects by listing instead of by name, over the same ledger. A
|
|
32
|
+
* generation held only for a lease is no part of the window and does not count toward the limit.
|
|
33
|
+
*/
|
|
34
|
+
export declare function carriesWindow(record: RegistryRecord | null, leased: ReadonlySet<number>): boolean;
|
|
35
|
+
/**
|
|
36
|
+
* How many generations below its own a publish vouches for at most: what the writer cap leaves beside the new
|
|
37
|
+
* generation and the 64 generations live leases can hold.
|
|
38
|
+
*/
|
|
39
|
+
export declare const KNOWN_SPAN = 127;
|
|
40
|
+
/** Which generations below a new pointer a grace window holds, and which it lets go: both ascending. */
|
|
41
|
+
interface KeptWindow {
|
|
42
|
+
readonly kept: number[];
|
|
43
|
+
readonly evict: number[];
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* The generations below a new pointer that a grace window of `keep` holds, out of `below`, when no ledger says which
|
|
47
|
+
* were made current most recently: the generation the publish superseded first, when it is among them, then the newest
|
|
48
|
+
* of the rest by number. When it is not among them (no generation was current, or its object has gone from the bucket),
|
|
49
|
+
* the newest `keep`.
|
|
50
|
+
*
|
|
51
|
+
* The superseded generation is the one reads were on until the publish, and, after a rollback, the one the operator
|
|
52
|
+
* restored, which a pin taken since reads, and the one to roll back to if the new load is bad too. The generations a
|
|
53
|
+
* rollback moved away from are newer by number, so a window by number alone would keep those and delete it. The window
|
|
54
|
+
* a listing keeps over a row with no usable ledger comes from here, and so does a ledger's first window
|
|
55
|
+
* (`ledger-seed.ts`); once the row has a ledger, its order says which were made current most recently.
|
|
56
|
+
*/
|
|
57
|
+
export declare function keptWindow(below: readonly number[], superseded: number | undefined, keep: number): KeptWindow;
|
|
58
|
+
/**
|
|
59
|
+
* `known`, held to the cap, in the same order. It always keeps the entries numbered in `protect`, then the ones numbered
|
|
60
|
+
* in `leased`, most recently current first, up to {@link MAX_KNOWN_GENERATIONS}, then the rest, most recently current
|
|
61
|
+
* first, while the ledger holds fewer than {@link KNOWN_FILL}. So a protected or leased entry is never dropped for room,
|
|
62
|
+
* and an unprotected one only when the ledger is full. "Most recently current" is the end of the list.
|
|
63
|
+
*/
|
|
64
|
+
export declare function capKnown(known: readonly KnownGeneration[], protect: ReadonlySet<number>, leased?: ReadonlySet<number>): KnownGeneration[];
|
|
65
|
+
/** `known` with `entry` made the most recent: any entry for its generation removed, and the entry added last. */
|
|
66
|
+
export declare function makeCurrent(known: readonly KnownGeneration[], entry: KnownGeneration): KnownGeneration[];
|
|
67
|
+
/** What a publish of a generation writes into the row, and which names it pushes out of the window. */
|
|
68
|
+
export interface KnownAfter {
|
|
69
|
+
/** The ledger to write: the generation published is its last entry. */
|
|
70
|
+
readonly known: readonly KnownGeneration[];
|
|
71
|
+
/** The window as numbers to collect by name, or `undefined` to collect by listing. */
|
|
72
|
+
readonly list: readonly number[] | undefined;
|
|
73
|
+
/** The generations the window no longer keeps, ascending: the names to delete. */
|
|
74
|
+
readonly evict: readonly number[];
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* What a publish of `generation` (its object's footer checksum `footerCrc`) over `prev` writes into the row.
|
|
78
|
+
*
|
|
79
|
+
* The ledger is in the order the generations were made current, so the order says which were current most recently.
|
|
80
|
+
* That ranks the window, and the window is drawn from the entries numbered below the new generation only: collection
|
|
81
|
+
* can only take generations numbered below the pointer. The window is the `keep` most recently current of them (at most
|
|
82
|
+
* {@link KNOWN_SPAN}), so the generation the publish superseded, the ledger's last entry, comes first, and after a
|
|
83
|
+
* rollback the ones the operator restored outrank the ones the rollback moved away from.
|
|
84
|
+
*
|
|
85
|
+
* What it writes, in the order the entries were made current and the new generation last:
|
|
86
|
+
*
|
|
87
|
+
* - the window;
|
|
88
|
+
* - the entries below the new generation that `leased` names (a generation a live lease holds stays known, so it stays a
|
|
89
|
+
* rollback target while the lease lives), whatever `keep` says;
|
|
90
|
+
* - the entries numbered above the new generation, which a rollback left and a load numbered under: they are carried and
|
|
91
|
+
* never evicted, and take no slot of `keep`, while the ledger holds fewer than {@link KNOWN_FILL} entries (see
|
|
92
|
+
* {@link capKnown});
|
|
93
|
+
* - the new generation. An entry already numbered `generation` is a stale one (its object went outside the ledger) and
|
|
94
|
+
* is replaced.
|
|
95
|
+
*
|
|
96
|
+
* `evict` is the entries below the new generation that it leaves out, ascending: never a name at or above the new
|
|
97
|
+
* generation. With a `keep` of at most {@link MAX_KEPT_GENERATIONS} the publish collects by name, so `list` is the
|
|
98
|
+
* generations of the ledger below the new one; above that, or with no `keep`, `list` is `undefined` and `evict` empty, and
|
|
99
|
+
* a load lists. A row with a pointer and no usable ledger writes the new generation alone and lists, unless the caller
|
|
100
|
+
* passes a `seed` (see `ledger-seed.ts`): the window a count keeps, read from the bucket, then the new generation, and
|
|
101
|
+
* the publish collects by that count. A publish that clears leases is an erasure's rewrite, which deletes every older
|
|
102
|
+
* generation, so its ledger is the new generation alone.
|
|
103
|
+
*/
|
|
104
|
+
export declare function nextKnown(prev: RegistryRecord | null, generation: number, footerCrc: number, keep: number | undefined, options?: {
|
|
105
|
+
readonly clearsLeases?: boolean;
|
|
106
|
+
readonly seed?: readonly KnownGeneration[];
|
|
107
|
+
readonly leased?: ReadonlySet<number>;
|
|
108
|
+
}): KnownAfter;
|
|
109
|
+
/**
|
|
110
|
+
* The ledger a rollback onto `target` writes over `prev`: the ledger it found with the target moved to the end, as the
|
|
111
|
+
* most recently current. A rollback never starts a ledger: over a row with no usable ledger it writes none, and the
|
|
112
|
+
* next publish seeds one. A target the ledger did not name
|
|
113
|
+
* is an insert, and the ledger it makes is held to the cap with the target and the generation the rollback leaves
|
|
114
|
+
* protected; a known target's ledger is written as read, unless another build wrote one that is past the writer's cap.
|
|
115
|
+
*/
|
|
116
|
+
export declare function rollbackKnown(prev: RegistryRecord, target: KnownGeneration): readonly KnownGeneration[] | undefined;
|
|
117
|
+
/**
|
|
118
|
+
* The row's ledger with the entries numbered in `doomed` removed, which a write that is about to delete those
|
|
119
|
+
* generations makes first: the entries left keep their order, and the current generation is never removed (the ledger
|
|
120
|
+
* must end with it). Another build's ledger past the writer's cap is held to it, with the current generation protected
|
|
121
|
+
* and the generations the row's leases name kept. `undefined` for a row with no usable ledger, which names nothing to
|
|
122
|
+
* remove.
|
|
123
|
+
*/
|
|
124
|
+
export declare function pruneKnown(record: RegistryRecord, doomed: ReadonlySet<number>): readonly KnownGeneration[] | undefined;
|
|
125
|
+
/**
|
|
126
|
+
* The ledger that puts a row back as it was, for the undo of a rollback: the row's own when it is usable (the
|
|
127
|
+
* generation it left is its last entry), held to the writer's cap when another build wrote one past it, with the
|
|
128
|
+
* generation the undo leaves and the one it moves from protected; `undefined` for a row that had none to put back.
|
|
129
|
+
*/
|
|
130
|
+
export declare function describingKnown(record: RegistryRecord, movedTo: number): readonly KnownGeneration[] | undefined;
|
|
131
|
+
/**
|
|
132
|
+
* Whether the ledger is full: it names at least {@link KNOWN_FILL} generations, which is when a writer may have let an
|
|
133
|
+
* older one go (a publish vouches for at most {@link KNOWN_SPAN} below its own, and an inserting rollback drops an entry
|
|
134
|
+
* once 128 that no lease holds are named). Leased entries do not switch it off: a lease on an entry inside the span
|
|
135
|
+
* leaves the ledger as full as it was. Past the least recently current entry of a full ledger, a rollback can no longer
|
|
136
|
+
* tell a generation the row forgot from one it never knew, so it judges a target numbered below that entry as it did
|
|
137
|
+
* before the ledger existed.
|
|
138
|
+
*/
|
|
139
|
+
export declare function isFullLedger(known: readonly KnownGeneration[]): boolean;
|
|
140
|
+
/**
|
|
141
|
+
* The number a target below the pointer is judged against on a full ledger: the least recently current entry that no
|
|
142
|
+
* lease entry of the row names (live or ended: a rollback has no clock). A leased entry stays in the ledger for its lease,
|
|
143
|
+
* however far the window has moved on since, so it does not mark where the ledger stopped remembering; the first entry no
|
|
144
|
+
* lease holds does. A ledger whose every entry is leased is judged against its first entry.
|
|
145
|
+
*/
|
|
146
|
+
export declare function fallbackBound(record: RegistryRecord, known: readonly KnownGeneration[]): number;
|
|
147
|
+
export {};
|
package/dist/core/leases.d.ts
CHANGED
|
@@ -17,6 +17,13 @@ export declare const MAX_STORED_LEASES = 256;
|
|
|
17
17
|
* plus a few, so that the most a segment can hold, all arriving at once, can all land.
|
|
18
18
|
*/
|
|
19
19
|
export declare const LEASE_CAS_ATTEMPTS: number;
|
|
20
|
+
/**
|
|
21
|
+
* Fresh writes sent after one that got no answer and left the row as it was, by the lease writers, by an erasure's
|
|
22
|
+
* renewal of the row and by a publish: four sends of the row write in all.
|
|
23
|
+
*/
|
|
24
|
+
export declare const UNANSWERED_RESENDS = 3;
|
|
25
|
+
/** The upper bound of the first of those waits, in ms; each next one doubles it. */
|
|
26
|
+
export declare const UNANSWERED_BASE_MS = 500;
|
|
20
27
|
/**
|
|
21
28
|
* Whether an entry holds its generation against a collector reading `now`: it has not ended (`until` plus the margin
|
|
22
29
|
* is ahead), and it does not run further past `now` than any honest holder with a clock inside the margin could have
|
|
@@ -58,7 +65,7 @@ export declare const LEASE_ONLY_RETRIES: number;
|
|
|
58
65
|
*/
|
|
59
66
|
export declare function onlyLeasesDiffer(held: RegistryRecord, now: RegistryRecord, alsoIgnore?: readonly (keyof RegistryRecord)[]): boolean;
|
|
60
67
|
/** A writer's count of the lease-only changes it has waited out, and the wait. One per call of the writer. */
|
|
61
|
-
|
|
68
|
+
interface LeaseChurn {
|
|
62
69
|
/**
|
|
63
70
|
* When `now` is `held` changed only in its leases and the bound is not spent: wait, then read the row again with
|
|
64
71
|
* `reread`, and return what it found. The wait comes before the read, so the writer's next write is made against a row
|
|
@@ -117,3 +124,4 @@ export declare function takeLease(ref: SegmentRef, deps: LeaseDeps, take: LeaseT
|
|
|
117
124
|
* races and the registry's `TransientError` for a write that stayed unanswered over an unchanged row.
|
|
118
125
|
*/
|
|
119
126
|
export declare function releaseLease(ref: SegmentRef, deps: LeaseDeps, holder: string): Promise<void>;
|
|
127
|
+
export {};
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { type ReadRetry } from './retry.js';
|
|
2
|
+
import type { IStorageDriver, KnownGeneration, RegistryRecord, SegmentRef } from './ports.js';
|
|
3
|
+
/**
|
|
4
|
+
* Whether a publish over `row` seeds its ledger: only at a birth, over an active row that has a pointer and no usable
|
|
5
|
+
* ledger. A usable ledger is never seeded again, however wide it is.
|
|
6
|
+
*/
|
|
7
|
+
export declare function wantsSeed(row: RegistryRecord | null): boolean;
|
|
8
|
+
/**
|
|
9
|
+
* The entries a seed names: the listed generations below the new one that a live lease holds and the window does not
|
|
10
|
+
* keep, then the window oldest first by number with the generation the row named as current last. `listed` is the
|
|
11
|
+
* listing the load's numbering made, when it made one; otherwise the bucket is listed once. Every entry's checksum is
|
|
12
|
+
* read from its object's footer (a 104-byte read, under the store's read retry, {@link SEED_READS} at a time), the
|
|
13
|
+
* superseded generation's too: when a clear summary describes it with another checksum, the row and the bucket
|
|
14
|
+
* disagree about that number and it is left out. A generation gone from the bucket, or whose footer fails its own
|
|
15
|
+
* checks, is left out; any other failed read throws, before anything is published.
|
|
16
|
+
*/
|
|
17
|
+
export declare function readSeed(storage: IStorageDriver, ref: SegmentRef, row: RegistryRecord | null, generation: number, keep: number, listed: readonly number[] | undefined, leased: ReadonlySet<number>, readRetry?: ReadRetry): Promise<KnownGeneration[]>;
|
package/dist/core/load.d.ts
CHANGED
|
@@ -1,24 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* `loadSegment` — the loaded store's primary write path, as one call.
|
|
3
|
-
*
|
|
4
|
-
* A load is always the same four steps: take the next generation number, write one immutable object, move the
|
|
5
|
-
* pointer, collect what the move superseded. Inside the library each is its own internal function, and the
|
|
6
|
-
* step a hand-composed load leaves out is the last one, so a store that composed it would accumulate superseded
|
|
7
|
-
* generations it keeps paying for.
|
|
8
|
-
* Composing it here, once, is the point.
|
|
9
|
-
*
|
|
10
|
-
* The other reason it is one call is the **guard**. A load REPLACES a segment: whatever the stream contains is
|
|
11
|
-
* what the segment contains afterwards. That makes an upstream query returning fewer rows than usual — or none —
|
|
12
|
-
* a silent, successful-looking wipe, and nothing downstream can tell it from a legitimate shrink. The guard is
|
|
13
|
-
* the place to say what a plausible result looks like, and it has to sit **between the write and the publish**,
|
|
14
|
-
* because that is the only moment where the new content is known and the old one is still authoritative.
|
|
15
|
-
*
|
|
16
|
-
* Refusing therefore has to clean up after itself. The object is already durable at that point, and it sits
|
|
17
|
-
* ABOVE `currentGen`, where generation collection deliberately never looks (it deletes strictly below the
|
|
18
|
-
* pointer), so a refused load deletes its own object before returning, while the segment's row is unchanged, gone
|
|
19
|
-
* or `destroyed`. Once another write has changed the row, the object's number may name a re-created segment's live
|
|
20
|
-
* object, so it stays, and collection takes it like any other generation once one above it is current.
|
|
21
|
-
*/
|
|
22
1
|
import { type IAuditSink } from './audit.js';
|
|
23
2
|
import { type CodecBitmap, type CodecInterface } from './codec.js';
|
|
24
3
|
import type { Clock, Rng } from './determinism.js';
|
|
@@ -114,11 +93,15 @@ export interface LoadOptions {
|
|
|
114
93
|
* longest pinned job runs, on every writer that loads the segment — see "Generations and `keep`" in the loading
|
|
115
94
|
* guide.
|
|
116
95
|
*
|
|
117
|
-
* The
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
*
|
|
121
|
-
* row
|
|
96
|
+
* The window is the `keep` most recently current generations below the one the load publishes, so the generation the
|
|
97
|
+
* load superseded comes first. After a rollback that is the one the rollback restored, and it stays in the window of
|
|
98
|
+
* every load after it, ahead of the generations the rollback moved away from, for as many loads as `keep` says.
|
|
99
|
+
*
|
|
100
|
+
* The segment's row records the generations a load keeps, up to 64, so a load whose number one existence check found
|
|
101
|
+
* free collects without listing: it deletes the generations its publish pushed out of the window, and lists the
|
|
102
|
+
* segment's objects on every sixteenth generation to take whatever that pass leaves, such as an object a crashed load
|
|
103
|
+
* left below the pointer. A `keep` above 64 collects by listing on every load, and so does the first load of a row with
|
|
104
|
+
* no usable ledger, which starts one from the bucket; so does a load whose check found the current generation's object
|
|
122
105
|
* gone: the guard's read of it, or, when the guard took the size from the row's summary, one zero-byte read made
|
|
123
106
|
* before a `keep` of 1 or more takes a name. A `keep` at least the generation published collects nothing and asks
|
|
124
107
|
* for nothing.
|
|
@@ -140,11 +123,6 @@ export interface LoadOptions {
|
|
|
140
123
|
}
|
|
141
124
|
/** The bounds a guard can refuse a generation for: every {@link LoadRefusal} but a lost race. */
|
|
142
125
|
export type GuardRefusal = Exclude<LoadRefusal, 'superseded'>;
|
|
143
|
-
/**
|
|
144
|
-
* The bound a generation of `cardinality` ids breaks over a current one of `before` (`null`: none, or not read), in the
|
|
145
|
-
* order a load checks them, or `undefined`. Pure: the one judgement a load and a dry run share.
|
|
146
|
-
*/
|
|
147
|
-
export declare function guardRefusal(cardinality: number, before: number | null, guard: LoadGuard | undefined, allowEmpty: boolean | undefined): GuardRefusal | undefined;
|
|
148
126
|
/** What a dry run learns about one write: the size it would replace, and the bound that would refuse it. */
|
|
149
127
|
export interface LoadJudgement {
|
|
150
128
|
/**
|
|
@@ -171,16 +149,19 @@ export type LoadRefusal = 'empty' | 'min-cardinality' | 'min-retained' | 'max-gr
|
|
|
171
149
|
* put refused this one and it wrote nothing (`size: 0`); or the segment's registry row changed while the load was
|
|
172
150
|
* writing: another load published, a retention change, a rollback or an erasure wrote the row, the row was
|
|
173
151
|
* deleted, or a row appeared where the load found none. Once another write has changed the row, the object stays in
|
|
174
|
-
* the bucket, still billed, until a later load's collection deletes it
|
|
175
|
-
*
|
|
176
|
-
*
|
|
177
|
-
*
|
|
152
|
+
* the bucket, still billed, until a later load's collection deletes it or the segment is dropped, because its
|
|
153
|
+
* generation number may by then name another writer's object; until it goes, a rollback onto it needs `allowUnknown`, as onto any generation the row does not know. Once the row is
|
|
154
|
+
* `destroyed` (dropped or crypto-shredded), the object is deleted; once the row is gone, so is an object whose footer
|
|
155
|
+
* matches what this load wrote (a re-created segment's load of the same cleartext ids writes a match), and so is a
|
|
156
|
+
* cleartext one under a row with key material.
|
|
178
157
|
*/
|
|
179
158
|
| 'superseded';
|
|
180
159
|
export interface LoadResult {
|
|
181
160
|
/**
|
|
182
161
|
* The generation written. Present even when refused: a refusal deletes that object while the segment's row is
|
|
183
|
-
* unchanged or `destroyed`,
|
|
162
|
+
* unchanged or `destroyed`, and, once one read of its footer matches what the load wrote, when the row is gone or,
|
|
163
|
+
* for a cleartext object, has key material. Under any other row another write changed, the object stays in the
|
|
164
|
+
* bucket, an orphan a later load's collection takes, and until then a rollback onto it needs `allowUnknown`. A load that lost its
|
|
184
165
|
* generation number to another wrote nothing, and reports `size: 0`.
|
|
185
166
|
*/
|
|
186
167
|
readonly generation: number;
|
package/dist/core/lru.d.ts
CHANGED
|
@@ -1,19 +1,5 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Bounded LRU cache with optional TTL — the cache tier's memory ceiling.
|
|
3
|
-
*
|
|
4
|
-
* Time comes from an injected `Clock` (never `Date.now()`) so the cache is deterministic
|
|
5
|
-
* under simulation.
|
|
6
|
-
*
|
|
7
|
-
* The cache can be bounded by entry **count** (`maxEntries`, always) and, optionally, by an aggregate
|
|
8
|
-
* **byte weight** (`maxBytes`). The byte bound exists for values whose footprint is large and uneven — the
|
|
9
|
-
* storage reader cache, where one entry pins a fully-parsed `.crbm` index that can be ~6 MB for a wide segment
|
|
10
|
-
* yet a few hundred bytes for a small one (a count-only bound lets 1024 wide indices blow a Lambda's
|
|
11
|
-
* memory even though the count is "in bounds"). Weights arrive **after** insertion (a cached reader is a
|
|
12
|
-
* pending promise; its size is known only once it resolves), so weight is reported via {@link setWeight} and
|
|
13
|
-
* eviction re-runs then.
|
|
14
|
-
*/
|
|
15
1
|
import type { Clock } from './determinism.js';
|
|
16
|
-
|
|
2
|
+
interface LruOptions {
|
|
17
3
|
/** Hard ceiling on entries; the least-recently-used is evicted past it. */
|
|
18
4
|
readonly maxEntries: number;
|
|
19
5
|
/**
|
|
@@ -67,3 +53,4 @@ export declare class BoundedLru<K, V> {
|
|
|
67
53
|
/** Current aggregate reported weight (bytes) across all entries. */
|
|
68
54
|
get weightBytes(): number;
|
|
69
55
|
}
|
|
56
|
+
export {};
|
package/dist/core/metrics.d.ts
CHANGED
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
* Design: the library emits **neutral domain events** and stays vendor-agnostic — you (or a ~12-line
|
|
6
6
|
* adapter) map them to OpenTelemetry / Datadog / a log line. The default is a **no-op**, and emission is
|
|
7
7
|
* skipped entirely when no sink is wired (near-zero overhead, no telemetry dependency). Events carry **raw
|
|
8
|
-
* observations** (bytes, counts, ms); turning those into billable units / dollars is
|
|
9
|
-
* which keeps rates
|
|
8
|
+
* observations** (bytes, counts, ms); turning those into billable units / dollars is left to whoever reads them,
|
|
9
|
+
* which keeps rates out of the library (`@cloudbitmaps/tools` prices a workload you give it).
|
|
10
10
|
*
|
|
11
11
|
* Two caveats for sink authors: `onEvent` runs **synchronously on the I/O path**, so keep it cheap and
|
|
12
12
|
* non-blocking (offload batching/network to your own async queue); and `segment`/`namespace` are
|
|
@@ -49,8 +49,10 @@ export type MetricEvent = {
|
|
|
49
49
|
readonly ms: number;
|
|
50
50
|
} | {
|
|
51
51
|
/**
|
|
52
|
-
* One decoded-chunk cache lookup
|
|
53
|
-
*
|
|
52
|
+
* One decoded-chunk cache lookup, or one chunk counted a miss without a lookup (emitted only when a cache is
|
|
53
|
+
* configured). `hit: false` means the cache did not hold the chunk. A combine or `iterate` counts every chunk it
|
|
54
|
+
* needs when it opens its streams: a lookup each, except that on a cache that holds nothing each chunk after the
|
|
55
|
+
* one a stream starts at is counted a miss without one. A caller that
|
|
54
56
|
* missed while another caller's read of the same chunk was open waits on that read and adds no `storage.get`, and a
|
|
55
57
|
* range carries several chunks, so the misses can outnumber the `storage.get` events; the `storage.get` count is
|
|
56
58
|
* the number of requests for chunks. `materializeMany` never looks up the cache, so it emits none.
|
|
@@ -168,9 +170,9 @@ export interface MetricsSnapshot {
|
|
|
168
170
|
}>>;
|
|
169
171
|
}
|
|
170
172
|
/**
|
|
171
|
-
* A ready-made sink that tallies events into a {@link MetricsSnapshot} — handy for tests, quick scripts,
|
|
172
|
-
*
|
|
173
|
-
*
|
|
173
|
+
* A ready-made sink that tallies events into a {@link MetricsSnapshot} — handy for tests, quick scripts, and for
|
|
174
|
+
* reading the request rates a cost model is given. `snapshot()` returns an independent copy; `reset()` zeroes the
|
|
175
|
+
* counters.
|
|
174
176
|
*/
|
|
175
177
|
export declare class CountingMetricsSink implements IMetricsSink {
|
|
176
178
|
private storage;
|
|
@@ -27,14 +27,14 @@ import type { CrbmStorageChunkSource } from './crbm-storage-source.js';
|
|
|
27
27
|
* What a pin holds for one segment: the generation, the version identifying those exact bytes, and the pinned
|
|
28
28
|
* object's fingerprint, so a read can tell that the generation it opens is still that object.
|
|
29
29
|
*
|
|
30
|
-
* `fingerprint` is
|
|
31
|
-
*
|
|
32
|
-
*
|
|
30
|
+
* `fingerprint` is required for a pin of a generation (`null` only when `generation` is `null`): without it a read
|
|
31
|
+
* could not tell a name purged and loaded again from the segment it pinned. `seg.pin()` always records it, and a
|
|
32
|
+
* pin of a generation without a string `version` and `fingerprint` is refused with a `ValidationError`.
|
|
33
33
|
*/
|
|
34
34
|
export interface PinnedAt {
|
|
35
35
|
readonly generation: number | null;
|
|
36
36
|
readonly version: string | null;
|
|
37
|
-
readonly fingerprint
|
|
37
|
+
readonly fingerprint: string | null;
|
|
38
38
|
}
|
|
39
39
|
export declare class PinnedStorageChunkSource implements StorageChunkSource {
|
|
40
40
|
private readonly inner;
|
|
@@ -53,8 +53,6 @@ export declare class PinnedStorageChunkSource implements StorageChunkSource {
|
|
|
53
53
|
* wrong answer this exists to prevent.
|
|
54
54
|
*/
|
|
55
55
|
pins: ReadonlyMap<string, PinnedAt>);
|
|
56
|
-
/** The unpinned segments refresh as the store's own source does; a pinned one never does. */
|
|
57
|
-
get pointerRefreshMs(): number;
|
|
58
56
|
/**
|
|
59
57
|
* The pin for this segment, or `undefined` if it is not pinned here.
|
|
60
58
|
*
|
|
@@ -74,12 +72,14 @@ export declare class PinnedStorageChunkSource implements StorageChunkSource {
|
|
|
74
72
|
sizeOf(ref: SegmentRef): Promise<SegmentSize | null>;
|
|
75
73
|
cardinalities(ref: SegmentRef): Promise<ReadonlyMap<number, number> | null>;
|
|
76
74
|
summary(ref: SegmentRef): Promise<GenerationSummary | null>;
|
|
75
|
+
stat(ref: SegmentRef): Promise<(GenerationSummary & SegmentSize) | null>;
|
|
77
76
|
currentGeneration(ref: SegmentRef): Promise<number | null>;
|
|
78
77
|
/**
|
|
79
78
|
* A pinned segment reports the version captured when it was pinned, marked as a pin's, so its decoded chunks
|
|
80
79
|
* are cached under keys that no live read writes. A live read keys its fetches by the version it resolved when
|
|
81
80
|
* it began, but is served whatever generation the live source holds when each fetch lands — after a publish
|
|
82
|
-
* and a lapsed `cache.genTtlMs`, a reader-cache eviction
|
|
81
|
+
* and a lapsed `cache.genTtlMs`, a reader-cache eviction on a store with no timed refresh, the resolution
|
|
82
|
+
* cache letting the segment's resolution go on a store with one, a sweep that heals the read forward, or an
|
|
83
83
|
* invalidation, a different one — so an entry under a live version can hold another generation's chunk.
|
|
84
84
|
* Invariant 3 lets that live call see it, and later live reads resolve the current version and never look the
|
|
85
85
|
* entry up; a pin sharing the key would be handed it, and return a read that mixed two generations. A pinned
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { RegistryPatch, RegistryRecord, Token } from './ports.js';
|
|
2
|
+
/** The fields a read resolves through: a patch that names any of them renews the row's `pointerId`. */
|
|
3
|
+
export declare const RESOLVED_FIELDS: readonly ["currentGen", "status", "wrappedDeks", "keyId", "summary"];
|
|
4
|
+
/**
|
|
5
|
+
* Whether a compare-and-swap of `patch` renews the row's `pointerId`: whether it names a resolved field, by presence
|
|
6
|
+
* (`'field' in patch`), whatever the value, as every patch rule reads a patch.
|
|
7
|
+
*/
|
|
8
|
+
export declare function renewsPointer(patch: RegistryPatch): boolean;
|
|
9
|
+
/**
|
|
10
|
+
* The patch that renews a row's `pointerId` and changes nothing a read resolves: the pointer, named at the value it
|
|
11
|
+
* has. A same-value pointer keeps the summary and the kept window, which follow the pointer only when it changes value.
|
|
12
|
+
*/
|
|
13
|
+
export declare function renewPointer(row: Pick<RegistryRecord, 'currentGen'>): RegistryPatch;
|
|
14
|
+
/**
|
|
15
|
+
* The row's `pointerId`, refusing a row that has none with {@link UnsupportedError}: a registry written against an
|
|
16
|
+
* earlier port contract returns rows without it, and a reader that cannot tell which write set the row's resolution
|
|
17
|
+
* cannot key a cache on it. A TypeScript registry fails to compile instead; this catches one in JavaScript.
|
|
18
|
+
*/
|
|
19
|
+
export declare function pointerIdOf(row: RegistryRecord): Token;
|