@cloudbitmaps/core 0.19.1 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/README.md +85 -0
  2. package/dist/{chunk-RSLVTRJY.js → chunk-IGLIZBR7.js} +982 -281
  3. package/dist/chunk-IGLIZBR7.js.map +7 -0
  4. package/dist/core/absent-operands.d.ts +8 -0
  5. package/dist/core/audit.d.ts +67 -4
  6. package/dist/core/bit-route.d.ts +2 -1
  7. package/dist/core/blob.d.ts +12 -1
  8. package/dist/core/bulk-load.d.ts +148 -0
  9. package/dist/core/cas-retry.d.ts +27 -0
  10. package/dist/core/check.d.ts +22 -0
  11. package/dist/core/chunk-checks.d.ts +6 -1
  12. package/dist/core/chunk-window.d.ts +11 -4
  13. package/dist/core/combine-expr.d.ts +1 -3
  14. package/dist/core/combine-feed.d.ts +2 -16
  15. package/dist/core/combine-many.d.ts +21 -15
  16. package/dist/core/concurrency.d.ts +8 -0
  17. package/dist/core/consistency.d.ts +76 -18
  18. package/dist/core/crbm/fingerprint.d.ts +23 -0
  19. package/dist/core/crbm/format.d.ts +2 -5
  20. package/dist/core/crbm/frame.d.ts +10 -0
  21. package/dist/core/crbm/plan-reads.d.ts +9 -3
  22. package/dist/core/crbm/reader.d.ts +51 -12
  23. package/dist/core/crbm/varint.d.ts +2 -1
  24. package/dist/core/crbm/writer.d.ts +5 -1
  25. package/dist/core/crbm-open.d.ts +154 -0
  26. package/dist/core/crbm-storage-source.d.ts +243 -490
  27. package/dist/core/crbm-write.d.ts +75 -0
  28. package/dist/core/crypto.d.ts +6 -1
  29. package/dist/core/engine.d.ts +239 -47
  30. package/dist/core/erase-id.d.ts +97 -27
  31. package/dist/core/erasure.d.ts +4 -3
  32. package/dist/core/errors.d.ts +8 -2
  33. package/dist/core/generation-gc.d.ts +131 -16
  34. package/dist/core/generation-list.d.ts +11 -0
  35. package/dist/core/id-window.d.ts +46 -0
  36. package/dist/core/invalidation-scope.d.ts +77 -0
  37. package/dist/core/known-generations-shape.d.ts +13 -0
  38. package/dist/core/known-generations.d.ts +147 -0
  39. package/dist/core/leases.d.ts +9 -1
  40. package/dist/core/ledger-seed.d.ts +17 -0
  41. package/dist/core/load.d.ts +17 -36
  42. package/dist/core/lru.d.ts +2 -15
  43. package/dist/core/metrics.d.ts +9 -7
  44. package/dist/core/order.d.ts +2 -0
  45. package/dist/core/pinned-storage-source.d.ts +7 -7
  46. package/dist/core/pointer-id.d.ts +19 -0
  47. package/dist/core/ports.d.ts +210 -45
  48. package/dist/core/publish.d.ts +207 -0
  49. package/dist/core/reader-defaults.d.ts +14 -3
  50. package/dist/core/registry-scan.d.ts +0 -2
  51. package/dist/core/retention-sweep.d.ts +69 -65
  52. package/dist/core/retention.d.ts +1 -1
  53. package/dist/core/retry.d.ts +12 -19
  54. package/dist/core/rollback.d.ts +62 -32
  55. package/dist/core/summary.d.ts +74 -25
  56. package/dist/core/token.d.ts +13 -8
  57. package/dist/core/validate.d.ts +2 -0
  58. package/dist/driver-kit.d.ts +6 -1
  59. package/dist/driver-kit.js +90 -10
  60. package/dist/driver-kit.js.map +3 -3
  61. package/dist/drivers/_shared/bytes.d.ts +6 -0
  62. package/dist/drivers/_shared/faults.d.ts +15 -0
  63. package/dist/drivers/_shared/object-keys.d.ts +18 -0
  64. package/dist/drivers/_shared/object-registry-keys.d.ts +0 -2
  65. package/dist/drivers/_shared/object-registry.d.ts +3 -1
  66. package/dist/drivers/_shared/options.d.ts +19 -0
  67. package/dist/drivers/_shared/registry.d.ts +31 -19
  68. package/dist/drivers/localfs/exact-case.d.ts +0 -3
  69. package/dist/drivers/localfs/fs-util.d.ts +22 -0
  70. package/dist/drivers/localfs/registry.d.ts +10 -4
  71. package/dist/drivers/localfs/storage.d.ts +16 -8
  72. package/dist/drivers/memory.d.ts +14 -9
  73. package/dist/drivers/retry/retrying-chunk-source.d.ts +1 -5
  74. package/dist/export/index.d.ts +16 -0
  75. package/dist/index.d.ts +2 -5
  76. package/dist/index.js +4303 -2938
  77. package/dist/index.js.map +4 -4
  78. package/package.json +1 -1
  79. package/dist/chunk-RSLVTRJY.js.map +0 -7
  80. package/dist/core/cost.d.ts +0 -351
  81. package/dist/core/kept-generations.d.ts +0 -40
@@ -0,0 +1,77 @@
1
+ /**
2
+ * What the engines reading through one chunk cache share about invalidations: how many there have been, the segment
3
+ * each of the latest was of, and the chunk reads open right now. A store's own engine, each pin's and each combine's
4
+ * with a pinned operand read through one cache, and an invalidation through any of them must reach a read running on
5
+ * another: a read that had its bytes before the invalidation would otherwise write them back into the cache it
6
+ * shares, and later reads would be answered from them, an erased id included, until the cache evicted them.
7
+ *
8
+ * ```
9
+ * engine (store) ──┐
10
+ * engine (pin) ──┼── one InvalidationScope ── epoch · latest invalidation of each segment · open reads
11
+ * engine (combine)─┘ └── one chunk cache
12
+ * ```
13
+ *
14
+ * A stream records the epoch it opened at and asks {@link InvalidationRecord.invalidatedSince} before it caches a
15
+ * chunk, so the scope holds no reference to a stream: a read its caller drops without closing holds nothing here. The
16
+ * `.crbm` source keeps an {@link InvalidationRecord} of its own, by segment, which a pinned stream asks before it hands
17
+ * out a chunk, so only its own segment's invalidation makes it open its object again.
18
+ *
19
+ * Internal to core: not exported from any entry point.
20
+ */
21
+ import type { CodecBitmap } from './codec.js';
22
+ /**
23
+ * How many segments' latest invalidation a record remembers. Past it the oldest is forgotten, and a segment the record
24
+ * does not remember, never-invalidated ones included, counts as invalidated at the newest invalidation it has
25
+ * forgotten, which is always on the safe side. In the engines' scope it means a stream that opened before then caches
26
+ * no more of that segment's chunks, which costs it cache entries, never correctness: so a sweep that invalidates more
27
+ * than this many segments at once (a `retireExpired` or an `eraseSubject` over a large fleet) stops every stream open
28
+ * at the time caching, whatever segment it reads. In the `.crbm` source it means a pinned stream looks its pin up
29
+ * again, which costs a request only where the reader cache has let the pin's reader go. With names at their
30
+ * 256-character limit a record is about 0.6 MB, and about 0.1 MB with 30-character names; a store keeps two.
31
+ */
32
+ export declare const REMEMBERED_INVALIDATIONS = 1024;
33
+ /** A chunk read in flight: `token` tells the read whether its entry is still the registered one. */
34
+ export interface OpenRead {
35
+ readonly token: object;
36
+ readonly read: Promise<CodecBitmap | null>;
37
+ }
38
+ /**
39
+ * How many invalidations there have been, and the epoch of each of the latest {@link REMEMBERED_INVALIDATIONS}
40
+ * segments' latest one, by whatever key names a segment to its owner: a cache-key prefix in the engines' scope, a
41
+ * segment key in the `.crbm` source.
42
+ */
43
+ export declare class InvalidationRecord {
44
+ private count;
45
+ /** The epoch of each remembered segment's latest invalidation, the oldest first. */
46
+ private readonly latest;
47
+ /** The epoch of the newest invalidation the record has forgotten: what a segment it does not remember counts as. */
48
+ private floor;
49
+ /** How many invalidations there have been: a read compares it with the epoch it began under. */
50
+ get epoch(): number;
51
+ /** Record an invalidation of the segment `key` names. */
52
+ invalidate(key: string): void;
53
+ /** Whether the segment `key` names has been invalidated since epoch `since`. */
54
+ invalidatedSince(key: string, since: number): boolean;
55
+ }
56
+ /** The engines' record, by cache-key prefix, and the chunk reads open on all of them. */
57
+ export declare class InvalidationScope extends InvalidationRecord {
58
+ /**
59
+ * The chunk reads open right now, by cache key: a caller that misses the cache and finds its key here awaits that
60
+ * read instead of making its own. An entry lives from the request until it settles, so the map holds at most one
61
+ * promise per distinct key in flight. An invalidation drops the segment's entries, and a read whose entry is gone
62
+ * caches nothing.
63
+ */
64
+ private readonly openReads;
65
+ /** Record an invalidation of the segment whose cache keys start with `prefix`, and drop its open reads. */
66
+ invalidate(prefix: string): void;
67
+ /** The read of `key` open now, for a caller that misses the cache to await instead of making its own. */
68
+ openRead(key: string): Promise<CodecBitmap | null> | undefined;
69
+ /** Whether a read of `key` is open, without joining it. */
70
+ hasOpenRead(key: string): boolean;
71
+ /** Register `entry` as the open read of `key`. */
72
+ beginRead(key: string, entry: OpenRead): void;
73
+ /** `entry` has settled: forget it, unless an invalidation dropped it and a newer read has taken the key. */
74
+ endRead(key: string, entry: OpenRead): void;
75
+ /** Whether the read `token` names is still the open read of `key`: one an invalidation dropped caches nothing. */
76
+ isOpen(key: string, token: object): boolean;
77
+ }
@@ -0,0 +1,13 @@
1
+ /**
2
+ * The shape of a row's known-generations ledger (`knownGens`), in one place. A registry holds a row to it when it writes
3
+ * and when it reads back, and `checkConsistency` holds a row to it before it reads or walks a ledger a registry of your
4
+ * own handed back, so the two cannot part on what a ledger is. Pure: no I/O, time or randomness.
5
+ */
6
+ /**
7
+ * Why `value` is not a ledger, or `undefined` when it is one (an absent ledger, `undefined`, passes: whether a row
8
+ * without one is usable is another rule). A ledger is an array of at most `max` entries, each exactly `{ generation,
9
+ * footerCrc }`, the generation a non-negative safe integer and unique, the checksum an integer from 0 to 2^32 - 1. The
10
+ * order is not checked: it is not derivable from the numbers. `show` renders a value for the description, which a
11
+ * caller can truncate.
12
+ */
13
+ export declare function knownGensShapeFault(value: unknown, max: number, show?: (v: unknown) => string): string | undefined;
@@ -0,0 +1,147 @@
1
+ import type { KnownGeneration, RegistryRecord } from './ports.js';
2
+ /**
3
+ * The most generations a load that collects by name records as its window. A `keep` above it collects by listing.
4
+ */
5
+ export declare const MAX_KEPT_GENERATIONS = 64;
6
+ /**
7
+ * A writer keeps entries that nothing protects and no lease holds only while the ledger holds fewer than this many.
8
+ */
9
+ export declare const KNOWN_FILL = 128;
10
+ /** The most entries a writer writes, and the most the shared validator accepts at a write. */
11
+ export declare const MAX_KNOWN_GENERATIONS = 192;
12
+ /** The most entries a stored row may hold: twice what a writer writes. */
13
+ export declare const MAX_STORED_KNOWN_GENERATIONS = 384;
14
+ /**
15
+ * The row's ledger when a collection or a rollback may rely on it: present and not empty, on an active row with a
16
+ * pointer, its last entry the pointer, and, where the row's summary is in the clear and describes that generation, its
17
+ * checksum the one the summary's fingerprint carries. A well-formed ledger that disagrees with its row is read but not
18
+ * used (the row counts as one with no ledger), as a summary that names another generation is.
19
+ */
20
+ export declare function usableKnownGens(record: RegistryRecord | null): readonly KnownGeneration[] | undefined;
21
+ /**
22
+ * The generations the row keeps as its window: the numbers of the usable ledger's entries below the pointer, in the order
23
+ * they were made current, or `undefined` when the ledger is not usable. A collection compares against these. An entry
24
+ * numbered above the pointer is one a rollback left; no collection can take it, so it is no part of the window.
25
+ */
26
+ export declare function windowOf(record: RegistryRecord | null): readonly number[] | undefined;
27
+ /**
28
+ * Whether the row's ledger is a window a load with a small `keep` can extend: usable, and naming no more than
29
+ * {@link MAX_KEPT_GENERATIONS} generations below the pointer that no live lease holds. A ledger carried through
30
+ * publishes with a larger `keep` (`*Into`, `materializeMany`) names up to 127, and a count by a small `keep` would keep
31
+ * fewer than it names, so the next small-`keep` load collects by listing instead of by name, over the same ledger. A
32
+ * generation held only for a lease is no part of the window and does not count toward the limit.
33
+ */
34
+ export declare function carriesWindow(record: RegistryRecord | null, leased: ReadonlySet<number>): boolean;
35
+ /**
36
+ * How many generations below its own a publish vouches for at most: what the writer cap leaves beside the new
37
+ * generation and the 64 generations live leases can hold.
38
+ */
39
+ export declare const KNOWN_SPAN = 127;
40
+ /** Which generations below a new pointer a grace window holds, and which it lets go: both ascending. */
41
+ interface KeptWindow {
42
+ readonly kept: number[];
43
+ readonly evict: number[];
44
+ }
45
+ /**
46
+ * The generations below a new pointer that a grace window of `keep` holds, out of `below`, when no ledger says which
47
+ * were made current most recently: the generation the publish superseded first, when it is among them, then the newest
48
+ * of the rest by number. When it is not among them (no generation was current, or its object has gone from the bucket),
49
+ * the newest `keep`.
50
+ *
51
+ * The superseded generation is the one reads were on until the publish, and, after a rollback, the one the operator
52
+ * restored, which a pin taken since reads, and the one to roll back to if the new load is bad too. The generations a
53
+ * rollback moved away from are newer by number, so a window by number alone would keep those and delete it. The window
54
+ * a listing keeps over a row with no usable ledger comes from here, and so does a ledger's first window
55
+ * (`ledger-seed.ts`); once the row has a ledger, its order says which were made current most recently.
56
+ */
57
+ export declare function keptWindow(below: readonly number[], superseded: number | undefined, keep: number): KeptWindow;
58
+ /**
59
+ * `known`, held to the cap, in the same order. It always keeps the entries numbered in `protect`, then the ones numbered
60
+ * in `leased`, most recently current first, up to {@link MAX_KNOWN_GENERATIONS}, then the rest, most recently current
61
+ * first, while the ledger holds fewer than {@link KNOWN_FILL}. So a protected or leased entry is never dropped for room,
62
+ * and an unprotected one only when the ledger is full. "Most recently current" is the end of the list.
63
+ */
64
+ export declare function capKnown(known: readonly KnownGeneration[], protect: ReadonlySet<number>, leased?: ReadonlySet<number>): KnownGeneration[];
65
+ /** `known` with `entry` made the most recent: any entry for its generation removed, and the entry added last. */
66
+ export declare function makeCurrent(known: readonly KnownGeneration[], entry: KnownGeneration): KnownGeneration[];
67
+ /** What a publish of a generation writes into the row, and which names it pushes out of the window. */
68
+ export interface KnownAfter {
69
+ /** The ledger to write: the generation published is its last entry. */
70
+ readonly known: readonly KnownGeneration[];
71
+ /** The window as numbers to collect by name, or `undefined` to collect by listing. */
72
+ readonly list: readonly number[] | undefined;
73
+ /** The generations the window no longer keeps, ascending: the names to delete. */
74
+ readonly evict: readonly number[];
75
+ }
76
+ /**
77
+ * What a publish of `generation` (its object's footer checksum `footerCrc`) over `prev` writes into the row.
78
+ *
79
+ * The ledger is in the order the generations were made current, so the order says which were current most recently.
80
+ * That ranks the window, and the window is drawn from the entries numbered below the new generation only: collection
81
+ * can only take generations numbered below the pointer. The window is the `keep` most recently current of them (at most
82
+ * {@link KNOWN_SPAN}), so the generation the publish superseded, the ledger's last entry, comes first, and after a
83
+ * rollback the ones the operator restored outrank the ones the rollback moved away from.
84
+ *
85
+ * What it writes, in the order the entries were made current and the new generation last:
86
+ *
87
+ * - the window;
88
+ * - the entries below the new generation that `leased` names (a generation a live lease holds stays known, so it stays a
89
+ * rollback target while the lease lives), whatever `keep` says;
90
+ * - the entries numbered above the new generation, which a rollback left and a load numbered under: they are carried and
91
+ * never evicted, and take no slot of `keep`, while the ledger holds fewer than {@link KNOWN_FILL} entries (see
92
+ * {@link capKnown});
93
+ * - the new generation. An entry already numbered `generation` is a stale one (its object went outside the ledger) and
94
+ * is replaced.
95
+ *
96
+ * `evict` is the entries below the new generation that it leaves out, ascending: never a name at or above the new
97
+ * generation. With a `keep` of at most {@link MAX_KEPT_GENERATIONS} the publish collects by name, so `list` is the
98
+ * generations of the ledger below the new one; above that, or with no `keep`, `list` is `undefined` and `evict` empty, and
99
+ * a load lists. A row with a pointer and no usable ledger writes the new generation alone and lists, unless the caller
100
+ * passes a `seed` (see `ledger-seed.ts`): the window a count keeps, read from the bucket, then the new generation, and
101
+ * the publish collects by that count. A publish that clears leases is an erasure's rewrite, which deletes every older
102
+ * generation, so its ledger is the new generation alone.
103
+ */
104
+ export declare function nextKnown(prev: RegistryRecord | null, generation: number, footerCrc: number, keep: number | undefined, options?: {
105
+ readonly clearsLeases?: boolean;
106
+ readonly seed?: readonly KnownGeneration[];
107
+ readonly leased?: ReadonlySet<number>;
108
+ }): KnownAfter;
109
+ /**
110
+ * The ledger a rollback onto `target` writes over `prev`: the ledger it found with the target moved to the end, as the
111
+ * most recently current. A rollback never starts a ledger: over a row with no usable ledger it writes none, and the
112
+ * next publish seeds one. A target the ledger did not name
113
+ * is an insert, and the ledger it makes is held to the cap with the target and the generation the rollback leaves
114
+ * protected; a known target's ledger is written as read, unless another build wrote one that is past the writer's cap.
115
+ */
116
+ export declare function rollbackKnown(prev: RegistryRecord, target: KnownGeneration): readonly KnownGeneration[] | undefined;
117
+ /**
118
+ * The row's ledger with the entries numbered in `doomed` removed, which a write that is about to delete those
119
+ * generations makes first: the entries left keep their order, and the current generation is never removed (the ledger
120
+ * must end with it). Another build's ledger past the writer's cap is held to it, with the current generation protected
121
+ * and the generations the row's leases name kept. `undefined` for a row with no usable ledger, which names nothing to
122
+ * remove.
123
+ */
124
+ export declare function pruneKnown(record: RegistryRecord, doomed: ReadonlySet<number>): readonly KnownGeneration[] | undefined;
125
+ /**
126
+ * The ledger that puts a row back as it was, for the undo of a rollback: the row's own when it is usable (the
127
+ * generation it left is its last entry), held to the writer's cap when another build wrote one past it, with the
128
+ * generation the undo leaves and the one it moves from protected; `undefined` for a row that had none to put back.
129
+ */
130
+ export declare function describingKnown(record: RegistryRecord, movedTo: number): readonly KnownGeneration[] | undefined;
131
+ /**
132
+ * Whether the ledger is full: it names at least {@link KNOWN_FILL} generations, which is when a writer may have let an
133
+ * older one go (a publish vouches for at most {@link KNOWN_SPAN} below its own, and an inserting rollback drops an entry
134
+ * once 128 that no lease holds are named). Leased entries do not switch it off: a lease on an entry inside the span
135
+ * leaves the ledger as full as it was. Past the least recently current entry of a full ledger, a rollback can no longer
136
+ * tell a generation the row forgot from one it never knew, so it judges a target numbered below that entry as it did
137
+ * before the ledger existed.
138
+ */
139
+ export declare function isFullLedger(known: readonly KnownGeneration[]): boolean;
140
+ /**
141
+ * The number a target below the pointer is judged against on a full ledger: the least recently current entry that no
142
+ * lease entry of the row names (live or ended: a rollback has no clock). A leased entry stays in the ledger for its lease,
143
+ * however far the window has moved on since, so it does not mark where the ledger stopped remembering; the first entry no
144
+ * lease holds does. A ledger whose every entry is leased is judged against its first entry.
145
+ */
146
+ export declare function fallbackBound(record: RegistryRecord, known: readonly KnownGeneration[]): number;
147
+ export {};
@@ -17,6 +17,13 @@ export declare const MAX_STORED_LEASES = 256;
17
17
  * plus a few, so that the most a segment can hold, all arriving at once, can all land.
18
18
  */
19
19
  export declare const LEASE_CAS_ATTEMPTS: number;
20
+ /**
21
+ * Fresh writes sent after one that got no answer and left the row as it was, by the lease writers, by an erasure's
22
+ * renewal of the row and by a publish: four sends of the row write in all.
23
+ */
24
+ export declare const UNANSWERED_RESENDS = 3;
25
+ /** The upper bound of the first of those waits, in ms; each next one doubles it. */
26
+ export declare const UNANSWERED_BASE_MS = 500;
20
27
  /**
21
28
  * Whether an entry holds its generation against a collector reading `now`: it has not ended (`until` plus the margin
22
29
  * is ahead), and it does not run further past `now` than any honest holder with a clock inside the margin could have
@@ -58,7 +65,7 @@ export declare const LEASE_ONLY_RETRIES: number;
58
65
  */
59
66
  export declare function onlyLeasesDiffer(held: RegistryRecord, now: RegistryRecord, alsoIgnore?: readonly (keyof RegistryRecord)[]): boolean;
60
67
  /** A writer's count of the lease-only changes it has waited out, and the wait. One per call of the writer. */
61
- export interface LeaseChurn {
68
+ interface LeaseChurn {
62
69
  /**
63
70
  * When `now` is `held` changed only in its leases and the bound is not spent: wait, then read the row again with
64
71
  * `reread`, and return what it found. The wait comes before the read, so the writer's next write is made against a row
@@ -117,3 +124,4 @@ export declare function takeLease(ref: SegmentRef, deps: LeaseDeps, take: LeaseT
117
124
  * races and the registry's `TransientError` for a write that stayed unanswered over an unchanged row.
118
125
  */
119
126
  export declare function releaseLease(ref: SegmentRef, deps: LeaseDeps, holder: string): Promise<void>;
127
+ export {};
@@ -0,0 +1,17 @@
1
+ import { type ReadRetry } from './retry.js';
2
+ import type { IStorageDriver, KnownGeneration, RegistryRecord, SegmentRef } from './ports.js';
3
+ /**
4
+ * Whether a publish over `row` seeds its ledger: only at a birth, over an active row that has a pointer and no usable
5
+ * ledger. A usable ledger is never seeded again, however wide it is.
6
+ */
7
+ export declare function wantsSeed(row: RegistryRecord | null): boolean;
8
+ /**
9
+ * The entries a seed names: the listed generations below the new one that a live lease holds and the window does not
10
+ * keep, then the window oldest first by number with the generation the row named as current last. `listed` is the
11
+ * listing the load's numbering made, when it made one; otherwise the bucket is listed once. Every entry's checksum is
12
+ * read from its object's footer (a 104-byte read, under the store's read retry, {@link SEED_READS} at a time), the
13
+ * superseded generation's too: when a clear summary describes it with another checksum, the row and the bucket
14
+ * disagree about that number and it is left out. A generation gone from the bucket, or whose footer fails its own
15
+ * checks, is left out; any other failed read throws, before anything is published.
16
+ */
17
+ export declare function readSeed(storage: IStorageDriver, ref: SegmentRef, row: RegistryRecord | null, generation: number, keep: number, listed: readonly number[] | undefined, leased: ReadonlySet<number>, readRetry?: ReadRetry): Promise<KnownGeneration[]>;
@@ -1,24 +1,3 @@
1
- /**
2
- * `loadSegment` — the loaded store's primary write path, as one call.
3
- *
4
- * A load is always the same four steps: take the next generation number, write one immutable object, move the
5
- * pointer, collect what the move superseded. Inside the library each is its own internal function, and the
6
- * step a hand-composed load leaves out is the last one, so a store that composed it would accumulate superseded
7
- * generations it keeps paying for.
8
- * Composing it here, once, is the point.
9
- *
10
- * The other reason it is one call is the **guard**. A load REPLACES a segment: whatever the stream contains is
11
- * what the segment contains afterwards. That makes an upstream query returning fewer rows than usual — or none —
12
- * a silent, successful-looking wipe, and nothing downstream can tell it from a legitimate shrink. The guard is
13
- * the place to say what a plausible result looks like, and it has to sit **between the write and the publish**,
14
- * because that is the only moment where the new content is known and the old one is still authoritative.
15
- *
16
- * Refusing therefore has to clean up after itself. The object is already durable at that point, and it sits
17
- * ABOVE `currentGen`, where generation collection deliberately never looks (it deletes strictly below the
18
- * pointer), so a refused load deletes its own object before returning, while the segment's row is unchanged, gone
19
- * or `destroyed`. Once another write has changed the row, the object's number may name a re-created segment's live
20
- * object, so it stays, and collection takes it like any other generation once one above it is current.
21
- */
22
1
  import { type IAuditSink } from './audit.js';
23
2
  import { type CodecBitmap, type CodecInterface } from './codec.js';
24
3
  import type { Clock, Rng } from './determinism.js';
@@ -114,11 +93,15 @@ export interface LoadOptions {
114
93
  * longest pinned job runs, on every writer that loads the segment — see "Generations and `keep`" in the loading
115
94
  * guide.
116
95
  *
117
- * The segment's row records the generations a load keeps, up to 64, so a load that found nothing above the pointer
118
- * collects without listing: it deletes the generations its publish pushed out of the window, and lists the segment's
119
- * objects on every sixteenth generation to take whatever that pass leaves, such as an object a crashed load left
120
- * below the pointer. A `keep` above 64 records no list and lists on every load, and so does the first load of a
121
- * row that records none, which then records it; so does a load whose check found the current generation's object
96
+ * The window is the `keep` most recently current generations below the one the load publishes, so the generation the
97
+ * load superseded comes first. After a rollback that is the one the rollback restored, and it stays in the window of
98
+ * every load after it, ahead of the generations the rollback moved away from, for as many loads as `keep` says.
99
+ *
100
+ * The segment's row records the generations a load keeps, up to 64, so a load whose number one existence check found
101
+ * free collects without listing: it deletes the generations its publish pushed out of the window, and lists the
102
+ * segment's objects on every sixteenth generation to take whatever that pass leaves, such as an object a crashed load
103
+ * left below the pointer. A `keep` above 64 collects by listing on every load, and so does the first load of a row with
104
+ * no usable ledger, which starts one from the bucket; so does a load whose check found the current generation's object
122
105
  * gone: the guard's read of it, or, when the guard took the size from the row's summary, one zero-byte read made
123
106
  * before a `keep` of 1 or more takes a name. A `keep` at least the generation published collects nothing and asks
124
107
  * for nothing.
@@ -140,11 +123,6 @@ export interface LoadOptions {
140
123
  }
141
124
  /** The bounds a guard can refuse a generation for: every {@link LoadRefusal} but a lost race. */
142
125
  export type GuardRefusal = Exclude<LoadRefusal, 'superseded'>;
143
- /**
144
- * The bound a generation of `cardinality` ids breaks over a current one of `before` (`null`: none, or not read), in the
145
- * order a load checks them, or `undefined`. Pure: the one judgement a load and a dry run share.
146
- */
147
- export declare function guardRefusal(cardinality: number, before: number | null, guard: LoadGuard | undefined, allowEmpty: boolean | undefined): GuardRefusal | undefined;
148
126
  /** What a dry run learns about one write: the size it would replace, and the bound that would refuse it. */
149
127
  export interface LoadJudgement {
150
128
  /**
@@ -171,16 +149,19 @@ export type LoadRefusal = 'empty' | 'min-cardinality' | 'min-retained' | 'max-gr
171
149
  * put refused this one and it wrote nothing (`size: 0`); or the segment's registry row changed while the load was
172
150
  * writing: another load published, a retention change, a rollback or an erasure wrote the row, the row was
173
151
  * deleted, or a row appeared where the load found none. Once another write has changed the row, the object stays in
174
- * the bucket, still billed, until a later load's collection deletes it (at the latest the listing a load runs
175
- * every sixteenth generation) or the segment is dropped, because its generation number may by then name another
176
- * incarnation's live object; once the row is `destroyed` (dropped or crypto-shredded), the object is deleted, and
177
- * once the row is gone, so is an object its footer proves this load's own.
152
+ * the bucket, still billed, until a later load's collection deletes it or the segment is dropped, because its
153
+ * generation number may by then name another writer's object; until it goes, a rollback onto it needs `allowUnknown`, as onto any generation the row does not know. Once the row is
154
+ * `destroyed` (dropped or crypto-shredded), the object is deleted; once the row is gone, so is an object whose footer
155
+ * matches what this load wrote (a re-created segment's load of the same cleartext ids writes a match), and so is a
156
+ * cleartext one under a row with key material.
178
157
  */
179
158
  | 'superseded';
180
159
  export interface LoadResult {
181
160
  /**
182
161
  * The generation written. Present even when refused: a refusal deletes that object while the segment's row is
183
- * unchanged or `destroyed`, or gone once its footer proves it this load's own, and leaves it in the bucket once another write has changed the row. A load that lost its
162
+ * unchanged or `destroyed`, and, once one read of its footer matches what the load wrote, when the row is gone or,
163
+ * for a cleartext object, has key material. Under any other row another write changed, the object stays in the
164
+ * bucket, an orphan a later load's collection takes, and until then a rollback onto it needs `allowUnknown`. A load that lost its
184
165
  * generation number to another wrote nothing, and reports `size: 0`.
185
166
  */
186
167
  readonly generation: number;
@@ -1,19 +1,5 @@
1
- /**
2
- * Bounded LRU cache with optional TTL — the cache tier's memory ceiling.
3
- *
4
- * Time comes from an injected `Clock` (never `Date.now()`) so the cache is deterministic
5
- * under simulation.
6
- *
7
- * The cache can be bounded by entry **count** (`maxEntries`, always) and, optionally, by an aggregate
8
- * **byte weight** (`maxBytes`). The byte bound exists for values whose footprint is large and uneven — the
9
- * storage reader cache, where one entry pins a fully-parsed `.crbm` index that can be ~6 MB for a wide segment
10
- * yet a few hundred bytes for a small one (a count-only bound lets 1024 wide indices blow a Lambda's
11
- * memory even though the count is "in bounds"). Weights arrive **after** insertion (a cached reader is a
12
- * pending promise; its size is known only once it resolves), so weight is reported via {@link setWeight} and
13
- * eviction re-runs then.
14
- */
15
1
  import type { Clock } from './determinism.js';
16
- export interface LruOptions {
2
+ interface LruOptions {
17
3
  /** Hard ceiling on entries; the least-recently-used is evicted past it. */
18
4
  readonly maxEntries: number;
19
5
  /**
@@ -67,3 +53,4 @@ export declare class BoundedLru<K, V> {
67
53
  /** Current aggregate reported weight (bytes) across all entries. */
68
54
  get weightBytes(): number;
69
55
  }
56
+ export {};
@@ -5,8 +5,8 @@
5
5
  * Design: the library emits **neutral domain events** and stays vendor-agnostic — you (or a ~12-line
6
6
  * adapter) map them to OpenTelemetry / Datadog / a log line. The default is a **no-op**, and emission is
7
7
  * skipped entirely when no sink is wired (near-zero overhead, no telemetry dependency). Events carry **raw
8
- * observations** (bytes, counts, ms); turning those into billable units / dollars is the cost model's job,
9
- * which keeps rates pluggable.
8
+ * observations** (bytes, counts, ms); turning those into billable units / dollars is left to whoever reads them,
9
+ * which keeps rates out of the library (`@cloudbitmaps/tools` prices a workload you give it).
10
10
  *
11
11
  * Two caveats for sink authors: `onEvent` runs **synchronously on the I/O path**, so keep it cheap and
12
12
  * non-blocking (offload batching/network to your own async queue); and `segment`/`namespace` are
@@ -49,8 +49,10 @@ export type MetricEvent = {
49
49
  readonly ms: number;
50
50
  } | {
51
51
  /**
52
- * One decoded-chunk cache lookup (emitted only when a cache is configured). `hit: false` means the lookup found
53
- * no cached chunk. A combine or `iterate` looks up every chunk it needs when it opens its streams. A caller that
52
+ * One decoded-chunk cache lookup, or one chunk counted a miss without a lookup (emitted only when a cache is
53
+ * configured). `hit: false` means the cache did not hold the chunk. A combine or `iterate` counts every chunk it
54
+ * needs when it opens its streams: a lookup each, except that on a cache that holds nothing each chunk after the
55
+ * one a stream starts at is counted a miss without one. A caller that
54
56
  * missed while another caller's read of the same chunk was open waits on that read and adds no `storage.get`, and a
55
57
  * range carries several chunks, so the misses can outnumber the `storage.get` events; the `storage.get` count is
56
58
  * the number of requests for chunks. `materializeMany` never looks up the cache, so it emits none.
@@ -168,9 +170,9 @@ export interface MetricsSnapshot {
168
170
  }>>;
169
171
  }
170
172
  /**
171
- * A ready-made sink that tallies events into a {@link MetricsSnapshot} — handy for tests, quick scripts,
172
- * and as the grounded-request-cost source behind `costReport()`. `snapshot()` returns an independent copy;
173
- * `reset()` zeroes the counters.
173
+ * A ready-made sink that tallies events into a {@link MetricsSnapshot} — handy for tests, quick scripts, and for
174
+ * reading the request rates a cost model is given. `snapshot()` returns an independent copy; `reset()` zeroes the
175
+ * counters.
174
176
  */
175
177
  export declare class CountingMetricsSink implements IMetricsSink {
176
178
  private storage;
@@ -0,0 +1,2 @@
1
+ /** The comparator that sorts numbers smallest first, which `Array.prototype.sort` does not do by default. */
2
+ export declare const ascending: (a: number, b: number) => number;
@@ -27,14 +27,14 @@ import type { CrbmStorageChunkSource } from './crbm-storage-source.js';
27
27
  * What a pin holds for one segment: the generation, the version identifying those exact bytes, and the pinned
28
28
  * object's fingerprint, so a read can tell that the generation it opens is still that object.
29
29
  *
30
- * `fingerprint` is optional so that a pin built by hand, before there was one, still compiles. Nothing checks the
31
- * object such a pin reads, so it cannot tell a name purged and loaded again from the segment it pinned.
32
- * `seg.pin()` always records it.
30
+ * `fingerprint` is required for a pin of a generation (`null` only when `generation` is `null`): without it a read
31
+ * could not tell a name purged and loaded again from the segment it pinned. `seg.pin()` always records it, and a
32
+ * pin of a generation without a string `version` and `fingerprint` is refused with a `ValidationError`.
33
33
  */
34
34
  export interface PinnedAt {
35
35
  readonly generation: number | null;
36
36
  readonly version: string | null;
37
- readonly fingerprint?: string | null;
37
+ readonly fingerprint: string | null;
38
38
  }
39
39
  export declare class PinnedStorageChunkSource implements StorageChunkSource {
40
40
  private readonly inner;
@@ -53,8 +53,6 @@ export declare class PinnedStorageChunkSource implements StorageChunkSource {
53
53
  * wrong answer this exists to prevent.
54
54
  */
55
55
  pins: ReadonlyMap<string, PinnedAt>);
56
- /** The unpinned segments refresh as the store's own source does; a pinned one never does. */
57
- get pointerRefreshMs(): number;
58
56
  /**
59
57
  * The pin for this segment, or `undefined` if it is not pinned here.
60
58
  *
@@ -74,12 +72,14 @@ export declare class PinnedStorageChunkSource implements StorageChunkSource {
74
72
  sizeOf(ref: SegmentRef): Promise<SegmentSize | null>;
75
73
  cardinalities(ref: SegmentRef): Promise<ReadonlyMap<number, number> | null>;
76
74
  summary(ref: SegmentRef): Promise<GenerationSummary | null>;
75
+ stat(ref: SegmentRef): Promise<(GenerationSummary & SegmentSize) | null>;
77
76
  currentGeneration(ref: SegmentRef): Promise<number | null>;
78
77
  /**
79
78
  * A pinned segment reports the version captured when it was pinned, marked as a pin's, so its decoded chunks
80
79
  * are cached under keys that no live read writes. A live read keys its fetches by the version it resolved when
81
80
  * it began, but is served whatever generation the live source holds when each fetch lands — after a publish
82
- * and a lapsed `cache.genTtlMs`, a reader-cache eviction, a sweep that heals the read forward, or an
81
+ * and a lapsed `cache.genTtlMs`, a reader-cache eviction on a store with no timed refresh, the resolution
82
+ * cache letting the segment's resolution go on a store with one, a sweep that heals the read forward, or an
83
83
  * invalidation, a different one — so an entry under a live version can hold another generation's chunk.
84
84
  * Invariant 3 lets that live call see it, and later live reads resolve the current version and never look the
85
85
  * entry up; a pin sharing the key would be handed it, and return a read that mixed two generations. A pinned
@@ -0,0 +1,19 @@
1
+ import type { RegistryPatch, RegistryRecord, Token } from './ports.js';
2
+ /** The fields a read resolves through: a patch that names any of them renews the row's `pointerId`. */
3
+ export declare const RESOLVED_FIELDS: readonly ["currentGen", "status", "wrappedDeks", "keyId", "summary"];
4
+ /**
5
+ * Whether a compare-and-swap of `patch` renews the row's `pointerId`: whether it names a resolved field, by presence
6
+ * (`'field' in patch`), whatever the value, as every patch rule reads a patch.
7
+ */
8
+ export declare function renewsPointer(patch: RegistryPatch): boolean;
9
+ /**
10
+ * The patch that renews a row's `pointerId` and changes nothing a read resolves: the pointer, named at the value it
11
+ * has. A same-value pointer keeps the summary and the kept window, which follow the pointer only when it changes value.
12
+ */
13
+ export declare function renewPointer(row: Pick<RegistryRecord, 'currentGen'>): RegistryPatch;
14
+ /**
15
+ * The row's `pointerId`, refusing a row that has none with {@link UnsupportedError}: a registry written against an
16
+ * earlier port contract returns rows without it, and a reader that cannot tell which write set the row's resolution
17
+ * cannot key a cache on it. A TypeScript registry fails to compile instead; this catches one in JavaScript.
18
+ */
19
+ export declare function pointerIdOf(row: RegistryRecord): Token;