@cloudbitmaps/core 0.21.0 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/README.md +23 -8
  2. package/dist/chunk-7GT632UF.js +641 -0
  3. package/dist/chunk-7GT632UF.js.map +7 -0
  4. package/dist/chunk-CGILDXGP.js +73 -0
  5. package/dist/chunk-CGILDXGP.js.map +7 -0
  6. package/dist/chunk-GKHERVVQ.js +121 -0
  7. package/dist/chunk-GKHERVVQ.js.map +7 -0
  8. package/dist/{chunk-IGLIZBR7.js → chunk-MVUD4R72.js} +253 -765
  9. package/dist/chunk-MVUD4R72.js.map +7 -0
  10. package/dist/core/audit.d.ts +53 -15
  11. package/dist/core/bit-route.d.ts +9 -0
  12. package/dist/core/blob.d.ts +4 -0
  13. package/dist/core/budget.d.ts +8 -3
  14. package/dist/core/check.d.ts +27 -0
  15. package/dist/core/codec.d.ts +1 -1
  16. package/dist/core/combine-expr.d.ts +9 -5
  17. package/dist/core/combine-feed.d.ts +11 -3
  18. package/dist/core/combine-many.d.ts +28 -13
  19. package/dist/core/consistency.d.ts +11 -2
  20. package/dist/core/cooperative.d.ts +7 -0
  21. package/dist/core/crbm-storage-source.d.ts +7 -4
  22. package/dist/core/crypto.d.ts +5 -0
  23. package/dist/core/determinism.d.ts +2 -1
  24. package/dist/core/discover.d.ts +5 -3
  25. package/dist/core/engine.d.ts +4 -4
  26. package/dist/core/erase-id.d.ts +52 -19
  27. package/dist/core/erasure.d.ts +62 -28
  28. package/dist/core/errors.d.ts +83 -9
  29. package/dist/core/leases.d.ts +2 -0
  30. package/dist/core/load-input.d.ts +48 -0
  31. package/dist/core/load.d.ts +71 -16
  32. package/dist/core/metrics.d.ts +22 -1
  33. package/dist/core/pinned-storage-source.d.ts +27 -8
  34. package/dist/core/pointer-id.d.ts +9 -0
  35. package/dist/core/ports.d.ts +107 -13
  36. package/dist/core/preflight.d.ts +80 -0
  37. package/dist/core/registry-scan.d.ts +0 -5
  38. package/dist/core/retention-sweep.d.ts +61 -16
  39. package/dist/core/retention.d.ts +18 -7
  40. package/dist/core/retry.d.ts +5 -2
  41. package/dist/core/rollback.d.ts +23 -12
  42. package/dist/core/validate.d.ts +33 -2
  43. package/dist/driver-kit.d.ts +9 -4
  44. package/dist/driver-kit.js +63 -27
  45. package/dist/driver-kit.js.map +2 -2
  46. package/dist/drivers/_shared/object-registry.d.ts +24 -1
  47. package/dist/drivers/_shared/options.d.ts +15 -7
  48. package/dist/drivers/backends.d.ts +19 -2
  49. package/dist/drivers/crypto.d.ts +18 -3
  50. package/dist/export/index.d.ts +25 -10
  51. package/dist/index.d.ts +19 -17
  52. package/dist/index.js +2457 -2283
  53. package/dist/index.js.map +4 -4
  54. package/dist/node.d.ts +14 -0
  55. package/dist/node.js +1064 -0
  56. package/dist/node.js.map +7 -0
  57. package/package.json +16 -2
  58. package/dist/chunk-IGLIZBR7.js.map +0 -7
  59. package/dist/core/absent-operands.d.ts +0 -8
  60. package/dist/core/bulk-load.d.ts +0 -148
  61. package/dist/core/cas-retry.d.ts +0 -27
  62. package/dist/core/chunk-checks.d.ts +0 -39
  63. package/dist/core/chunk-stream.d.ts +0 -22
  64. package/dist/core/chunk-window.d.ts +0 -37
  65. package/dist/core/combine-ledger.d.ts +0 -43
  66. package/dist/core/crbm/crc32c.d.ts +0 -10
  67. package/dist/core/crbm/fingerprint.d.ts +0 -23
  68. package/dist/core/crbm/format.d.ts +0 -150
  69. package/dist/core/crbm/frame.d.ts +0 -10
  70. package/dist/core/crbm/plan-reads.d.ts +0 -50
  71. package/dist/core/crbm/varint.d.ts +0 -14
  72. package/dist/core/crbm/writer.d.ts +0 -72
  73. package/dist/core/crbm-open.d.ts +0 -154
  74. package/dist/core/crbm-write.d.ts +0 -75
  75. package/dist/core/due-index.d.ts +0 -53
  76. package/dist/core/generation-gc.d.ts +0 -293
  77. package/dist/core/generation-list.d.ts +0 -11
  78. package/dist/core/id-window.d.ts +0 -46
  79. package/dist/core/invalidation-scope.d.ts +0 -77
  80. package/dist/core/item-pull.d.ts +0 -29
  81. package/dist/core/known-generations-shape.d.ts +0 -13
  82. package/dist/core/known-generations.d.ts +0 -147
  83. package/dist/core/ledger-seed.d.ts +0 -17
  84. package/dist/core/metadata.d.ts +0 -42
  85. package/dist/core/order.d.ts +0 -2
  86. package/dist/core/publish.d.ts +0 -207
  87. package/dist/core/reader-defaults.d.ts +0 -29
  88. package/dist/core/summary.d.ts +0 -110
  89. package/dist/core/token.d.ts +0 -41
  90. package/dist/drivers/_shared/entropy.d.ts +0 -5
  91. package/dist/drivers/_shared/registry.d.ts +0 -107
  92. package/dist/drivers/localfs/exact-case.d.ts +0 -34
  93. package/dist/drivers/localfs/fs-util.d.ts +0 -55
  94. package/dist/drivers/localfs/paths.d.ts +0 -40
  95. package/dist/drivers/localfs/registry.d.ts +0 -50
  96. package/dist/drivers/localfs/storage.d.ts +0 -28
  97. package/dist/drivers/memory.d.ts +0 -64
  98. package/dist/testing/fuzz-core.d.ts +0 -15
@@ -1,72 +0,0 @@
1
- import type { CrbmCrypto } from '../crypto.js';
2
- import type { GenerationMetadata } from '../ports.js';
3
- import type { BlobSink } from '../blob.js';
4
- export interface CrbmWriterOptions {
5
- /** Self-describing generation number (also encoded in the object key). */
6
- readonly generation: number;
7
- /**
8
- * Which codec produced the chunk payloads. Defaults to {@link PAYLOAD_CODEC_ROARING_PORTABLE}.
9
- *
10
- * A codec id rather than a roaring-specific one, because `.crbm` is a shared container and a second codec is
11
- * expected before the format freezes.
12
- */
13
- readonly payloadCodecId?: number;
14
- readonly elementWidth?: number;
15
- /** When set, payloads + index are AES-256-GCM-encrypted, bound to `(segment, generation, scope)` via AAD. */
16
- readonly crypto?: CrbmCrypto;
17
- /**
18
- * The generation's metadata, checked against the metadata rules and copied as its canonical JSON when the writer
19
- * is constructed, so a later change to the caller's object never reaches the object. A value that breaks a rule
20
- * throws {@link ValidationError} from the constructor, before a byte is written. None, or the empty object, writes
21
- * format 1.0.
22
- */
23
- readonly metadata?: GenerationMetadata;
24
- }
25
- export declare class CrbmWriter {
26
- private readonly sink;
27
- private readonly options;
28
- private readonly entries;
29
- private offset;
30
- private lastKey;
31
- private totalCardinality;
32
- private preambleWritten;
33
- private finished;
34
- /** The metadata's canonical JSON, copied at construction; `undefined` leaves the footer's extension flag clear. */
35
- private readonly metadata;
36
- /** Bytes written so far, the footer included once {@link finish} has run. */
37
- private written;
38
- /** Where the chunk payloads end: the extension block's start when there is metadata, else the index's. */
39
- private payloadEnd;
40
- private footerCrc32c;
41
- constructor(sink: BlobSink, options: CrbmWriterOptions);
42
- /**
43
- * Append one chunk's payload (the roaring-serialized bytes) with its cardinality. `chunkKey` must be
44
- * strictly greater than the previous one (ascending, no duplicates); `cardinality` is `[1, 65536]`.
45
- */
46
- addChunk(chunkKey: number, payload: Uint8Array, cardinality: number): Promise<void>;
47
- /**
48
- * Write the extension block (when there is metadata), the index region and the fixed footer. After this the object
49
- * is complete and immutable.
50
- */
51
- finish(): Promise<void>;
52
- /**
53
- * The finished object's size and its footer's CRC: what names it, as a reader's fingerprint does. A writer that
54
- * has to tell its own object from another stored later under the same key compares these with the object's
55
- * footer. `tailBytes` is the bytes after the chunk payloads (any extension block, the index and the footer), which
56
- * is what {@link CrbmReader.tailBytes} reports for the object. Throws before {@link finish}.
57
- */
58
- get identity(): {
59
- readonly size: number;
60
- readonly footerCrc: number;
61
- readonly tailBytes: number;
62
- };
63
- private ensurePreamble;
64
- /**
65
- * The extension block: one metadata section (`u8 type ‖ u32 length ‖ bytes`), then the trailer
66
- * (`u32 sectionsLength ‖ u32 crc32c(sections ‖ sectionsLength) ‖ "CRBX"`). Encrypted, the section holds
67
- * `nonce ‖ ciphertext ‖ tag`, sealed with the metadata scope of the associated data.
68
- */
69
- private extensionBlock;
70
- private encodeIndex;
71
- private buildFooter;
72
- }
@@ -1,154 +0,0 @@
1
- import type { BlobReader } from './blob.js';
2
- import type { CrbmCrypto } from './crypto.js';
3
- import type { GenerationMetadata, GenKey, IStorageDriver, TailRead } from './ports.js';
4
- import { CrbmReader } from './crbm/reader.js';
5
- import type { CrbmReaderOptions } from './crbm/reader.js';
6
- import type { SummaryDescription } from './summary.js';
7
- /**
8
- * Adapt one `(driver, key)` pair to the codec's `BlobReader` seam. With `noteTail`, each tail read the reader makes is
9
- * shown to it with the bytes, `size` and `version` the driver answered, after the copy below when one is made.
10
- *
11
- * A tail's bytes reach the reader as a plain `Uint8Array`: the reader keeps a small object's chunk region by slicing
12
- * the tail, and a subclass's `slice` need not copy. A Node `Buffer`'s is a view, so a driver that answered with one
13
- * would leave the reader holding the driver's memory, all of it, as the cache weighs only the region. Such an answer
14
- * is copied here, once ({@link ownedTail}); a plain `Uint8Array` of this realm, which every shipped driver returns, is
15
- * handed through as it is.
16
- */
17
- export declare function storageBlobReader(driver: IStorageDriver, key: GenKey, noteTail?: (tail: TailRead) => void): BlobReader;
18
- /**
19
- * `tail` with its bytes in a plain `Uint8Array` of this realm. Any other `Uint8Array`, a subclass (a Node `Buffer`) or
20
- * one made in another realm (a `vm` context), is copied once, and the copy is built from the answer's fields by name,
21
- * so a `size` or `version` the driver serves from a getter on its own class carries over. The brand is read, not
22
- * `instanceof`, which is false for another realm's arrays. A plain `Uint8Array` of this realm is handed through as it is.
23
- */
24
- export declare function ownedTail(tail: TailRead): TailRead;
25
- /**
26
- * The object under `key`, when one read of its footer matches `fingerprint` ({@link CrbmReader.fingerprint}), with the
27
- * version the driver reported on that read (`undefined` from a driver that reports none). `null` when it is another
28
- * object, when it is gone, and when the read fails in any way. A fingerprint names the object's bytes, not the writer:
29
- * an object another writer stored under the number with the same bytes matches too. A caller that deletes what this
30
- * matches passes the version as the delete's `ifVersion`, so a number taken again between the read and the delete keeps
31
- * the object put under it since.
32
- */
33
- export declare function provenObject(storage: IStorageDriver, key: GenKey, fingerprint: string): Promise<{
34
- readonly version: string | undefined;
35
- } | null>;
36
- /**
37
- * Whether the object under `key` is still the one `fingerprint` names ({@link CrbmReader.fingerprint}), from one read
38
- * of its footer. Unlike {@link provenObject}, a fault is not an answer: `NotFoundError` when the key holds no object,
39
- * and any other failure of the read, thrown as it is.
40
- */
41
- export declare function sameObjectUnder(storage: IStorageDriver, key: GenKey, fingerprint: string): Promise<boolean>;
42
- /**
43
- * Whether the object under `key` says it is encrypted, from one read of its footer, with no key. `noteVersion` is shown
44
- * the version the driver reported on that read, when it reported one: a caller that deletes the object for what its
45
- * footer says passes it as the delete's `ifVersion`.
46
- */
47
- export declare function objectIsEncrypted(storage: IStorageDriver, key: GenKey, noteVersion?: (version: string) => void): Promise<boolean>;
48
- /**
49
- * What one tail read of a rollback target says about it with no key: whether the footer says it is encrypted and its
50
- * CRC, and the remembered tail an open from it ({@link openRollbackTargetFrom}) is answered from.
51
- */
52
- interface RollbackTargetFooter {
53
- readonly encrypted: boolean;
54
- readonly footerCrc: number;
55
- readonly blob: BlobReader;
56
- }
57
- /**
58
- * The footer step of a rollback: one tail read of the object under `key`, which is remembered for the open that may
59
- * follow. A footer that fails its own checks is an {@link IntegrityError}; a missing object is {@link NotFoundError}.
60
- */
61
- export declare function readRollbackTargetFooter(storage: IStorageDriver, key: GenKey): Promise<RollbackTargetFooter>;
62
- /**
63
- * The reader {@link RollbackTargetFooter}'s object opens as, from its remembered tail, when the object and the key
64
- * agree (an encrypted object opened with `crypto`, a cleartext one with none); `undefined` when they disagree, which
65
- * the caller refuses as a target the row cannot use. An object that is another generation's, and an index or metadata
66
- * that do not open, are an {@link IntegrityError}.
67
- */
68
- export declare function openRollbackTargetFrom(footer: RollbackTargetFooter, key: GenKey, crypto: CrbmCrypto | undefined): Promise<CrbmReader | undefined>;
69
- /**
70
- * The footer CRC of the object under `key` ({@link CrbmReader.fingerprint} carries it), from one read of a footer's
71
- * worth, so it reads an encrypted object's CRC as it does a cleartext one's, with no key. {@link NotFoundError} when
72
- * the key holds no object; a footer that fails its own checks is an {@link IntegrityError}.
73
- */
74
- export declare function footerCrcUnder(storage: IStorageDriver, key: GenKey): Promise<number>;
75
- /**
76
- * Whether the object under `key` is the one `fingerprint` names, from one read of its footer: false when it is another
77
- * object or gone. Unlike {@link provenObject}, a read that cannot tell throws, for a caller that must not take "could
78
- * not look" for "not this object".
79
- */
80
- export declare function provesOwnObject(storage: IStorageDriver, key: GenKey, fingerprint: string): Promise<boolean>;
81
- /**
82
- * Open a {@link CrbmReader} on one generation over the storage driver's range/tail reads (decrypting if `crypto`).
83
- * The reader the write paths use to re-read what they wrote, and the erasure rewrite uses to stream the old
84
- * generation; the engine's read path goes through {@link CrbmStorageChunkSource} instead, which caches these. With
85
- * `named`, the fingerprint the row's summary records, the object must be that one: another is {@link NotFoundError}.
86
- * The version of the object it opened, from the tail read the open made, is what {@link objectVersionOf} answers.
87
- */
88
- export declare function openGenerationReader(storage: IStorageDriver, key: GenKey, crypto: CrbmCrypto | undefined, options?: Omit<CrbmReaderOptions, 'crypto'>, named?: string): Promise<CrbmReader>;
89
- /**
90
- * {@link openGenerationReader}, with `noteVersion` shown the version {@link objectVersionOf} would answer as soon as the
91
- * tail read that carries it returns, before the open goes on: a caller whose open then fails, on an object sealed under
92
- * another key, say, still knows which object it met, and can condition a delete of it on that read.
93
- */
94
- export declare function openGenerationReaderNoting(storage: IStorageDriver, key: GenKey, crypto: CrbmCrypto | undefined, noteVersion: (version: string) => void): Promise<CrbmReader>;
95
- /**
96
- * The version of the object `reader` was opened on, as the driver reported it on the tail read the open made, for a
97
- * delete conditioned on that object ({@link StorageDeleteOptions.ifVersion}); `undefined` for a reader
98
- * {@link openGenerationReader} did not open, or a driver that reports none. Held beside the reader rather than on it, so
99
- * nothing is added to the reader's public surface, and no read the engine makes pays for it.
100
- */
101
- export declare function objectVersionOf(reader: CrbmReader): string | undefined;
102
- /**
103
- * {@link openGenerationReader}, of the object `summary.fingerprint` names: `undefined` when the object under the key is
104
- * another one, found from its footer before its index is opened, so an object sealed under another key is told apart as
105
- * well. The tail read is sized from the summary's `tailBytes` ({@link sizedTailBytes}), so a wide generation's index
106
- * comes in that read.
107
- */
108
- export declare function openNamedGeneration(storage: IStorageDriver, key: GenKey, crypto: CrbmCrypto | undefined, summary: Pick<SummaryDescription, 'fingerprint' | 'tailBytes'>, noteTail?: (tail: TailRead) => void): Promise<CrbmReader | undefined>;
109
- /**
110
- * Open a reader on `key`, refusing an object whose footer names another generation. Every writer stamps the key's
111
- * number, and the chunk cache, the load guard and the erasure rewrite all trust it, so an object that disagrees
112
- * was written under another key or altered. Every open, the live read's, a pin's and the write paths', comes here.
113
- * With `named`, the object must also be the one that fingerprint names: another one is {@link NotFoundError}, a move.
114
- * With `early`, the tail read {@link startTailRead} began, the open takes that read as its own instead of making one.
115
- */
116
- export declare function openChecked(storage: IStorageDriver, key: GenKey, options: CrbmReaderOptions, keepChunkBytesUpTo?: number, named?: string, noteTail?: (tail: TailRead) => void, early?: Promise<TailRead>): Promise<CrbmReader>;
117
- /**
118
- * The size of the tail read an open makes when its row's summary says how many bytes of the object's end the open
119
- * needs (`tailBytes`: any extension block, the index and the footer), so that one read reaches the index of a wide
120
- * generation and no range read follows it. With no summary it is the source's own size ({@link openTailBytes}).
121
- *
122
- * The number came from a registry the library may not have written, so it only ever sizes a read: never below the
123
- * source's own tail, at most the object less its preamble (the size its fingerprint names), and at most the source's
124
- * `maxIndexBytes` (or the default) plus an extension block, its trailer and the footer, the most any open reads past
125
- * its own tail. A hint that is not a whole number of bytes above zero is ignored, and so is a result that is not one. A
126
- * wrong hint is a longer read or one more range read, never a different answer: the reader checks every byte it takes.
127
- */
128
- export declare function sizedTailBytes(summary: Pick<SummaryDescription, 'fingerprint' | 'tailBytes'> | undefined, options: {
129
- readonly tailBytes?: number;
130
- readonly maxIndexBytes?: number;
131
- }): number;
132
- /**
133
- * Send the tail read an open of `key` makes, now, and hand back its answer for {@link openChecked}'s `early`: the open
134
- * then waits on the one request instead of sending its own, so work the open waits on first, such as a keystore call,
135
- * runs beside the request and not before it. Nothing is parsed here: the bytes are untrusted until the open reads them.
136
- * The returned promise already has a handler attached, so a caller that never reaches the open (its key failed) leaves
137
- * no unhandled rejection; the open that does reach it still sees the rejection.
138
- */
139
- export declare function startTailRead(storage: IStorageDriver, key: GenKey, options: CrbmReaderOptions): Promise<TailRead>;
140
- /**
141
- * Re-open a freshly written generation and assert it round-trips exactly what was streamed into it: the same
142
- * per-chunk key set *and* the same total cardinality (on top of the codec's own per-chunk CRC + footer checks), and the
143
- * metadata it was given. `expected` is the streaming writer's tally (the stream is consumed, so it can't be
144
- * re-iterated) — the key-set comparison catches a dropped/extra chunk that a cardinality-only check could miss when
145
- * two errors cancel out. The count and the metadata are what the generation's row summary is about to claim, so the
146
- * summary is held to the object it describes before it is published.
147
- * Throws {@link IntegrityError}: the object is on disk but must not be published.
148
- */
149
- export declare function verifyGeneration(storage: IStorageDriver, key: GenKey, expected: {
150
- readonly chunkKeys: readonly number[];
151
- readonly cardinality: number;
152
- readonly metadata?: GenerationMetadata | undefined;
153
- }, crypto: CrbmCrypto | undefined): Promise<void>;
154
- export {};
@@ -1,75 +0,0 @@
1
- import type { Yielder } from './cooperative.js';
2
- import type { CrbmCrypto } from './crypto.js';
3
- import type { GenerationMetadata, GenKey, IStorageDriver } from './ports.js';
4
- import type { CodecBitmap, EncodedChunk } from './codec.js';
5
- /**
6
- * Write one immutable generation from in-memory bitmaps. Chunks are
7
- * sorted ascending and empty bitmaps skipped (empty chunks are never stored). Returns the driver's
8
- * `{ size, sha256 }` for the written object. Pass `options.crypto` to AES-256-GCM-encrypt the generation
9
- * (built from the segment's DEK, with associated data bound to `(segment, generation)`).
10
- */
11
- export declare function writeCrbmGeneration(driver: IStorageDriver, key: GenKey, chunks: Iterable<{
12
- chunkKey: number;
13
- bitmap: CodecBitmap;
14
- }>, options?: {
15
- crypto?: CrbmCrypto;
16
- clock?: Yielder;
17
- metadata?: GenerationMetadata;
18
- }): Promise<{
19
- size: number;
20
- sha256: string;
21
- fingerprint: string;
22
- tailBytes: number;
23
- }>;
24
- /** Each non-empty bitmap as the chunk it stores, in the order given. */
25
- export declare function encodeEach(chunks: Iterable<{
26
- chunkKey: number;
27
- bitmap: CodecBitmap;
28
- }>): Generator<EncodedChunk>;
29
- /**
30
- * Write one immutable generation from chunks already encoded, which must ascend by key (the writer refuses one
31
- * that does not). Returns the driver's `{ size, sha256 }` and a tally of what was written.
32
- */
33
- export declare function writeEncodedChunks(driver: IStorageDriver, key: GenKey, chunks: Iterable<EncodedChunk>, options: {
34
- crypto?: CrbmCrypto;
35
- clock?: Yielder;
36
- metadata?: GenerationMetadata;
37
- }): Promise<{
38
- size: number;
39
- sha256: string;
40
- chunkCount: number;
41
- cardinality: number;
42
- fingerprint: string;
43
- /** The bytes after the chunk payloads ({@link CrbmReader.tailBytes}). */
44
- tailBytes: number;
45
- }>;
46
- /** What {@link writeCrbmGenerationStream} wrote: the driver's `{ size, sha256 }` + a tally of the generation. */
47
- interface StreamWriteResult {
48
- readonly size: number;
49
- readonly sha256: string;
50
- /** Non-empty chunk keys written, in ascending order (for the post-write verify, since the stream is consumed). */
51
- readonly chunkKeys: number[];
52
- /** Total ids written. */
53
- readonly cardinality: number;
54
- /** The fingerprint of what this call wrote ({@link CrbmReader.fingerprint}), which tells it from another object under its key. */
55
- readonly fingerprint: string;
56
- /** The bytes after the chunk payloads ({@link CrbmReader.tailBytes}). */
57
- readonly tailBytes: number;
58
- }
59
- /**
60
- * Streaming variant of {@link writeCrbmGeneration} for a **constant-memory** rewrite (the erasure rewrite, a
61
- * sorted load): consumes an **already-ascending** async stream of `{ chunkKey, bitmap }`, feeding each to the
62
- * codec and freeing it, instead of materializing the whole generation. Paired with a streaming storage sink (S3
63
- * multipart / LocalFs temp file), the writer itself holds ~one chunk + one part. Returns a tally so the caller can verify
64
- * the re-opened object without re-iterating the (now-consumed) stream. Input **must** be ascending by `chunkKey`
65
- * — the codec rejects an out-of-order chunk; empty bitmaps are skipped.
66
- */
67
- export declare function writeCrbmGenerationStream(driver: IStorageDriver, key: GenKey, chunks: AsyncIterable<{
68
- chunkKey: number;
69
- bitmap: CodecBitmap;
70
- }>, options?: {
71
- crypto?: CrbmCrypto;
72
- clock?: Yielder;
73
- metadata?: GenerationMetadata;
74
- }): Promise<StreamWriteResult>;
75
- export {};
@@ -1,53 +0,0 @@
1
- import { MAX_NAME_LENGTH } from './validate.js';
2
- import type { RegistryRecord, SegmentRef } from './ports.js';
3
- /** Bucket width. One day: small enough that a cycle reads little, coarse enough that the index stays tiny. */
4
- export declare const DUE_BUCKET_MS = 86400000;
5
- /**
6
- * The name-length cap, and an index row encodes **both** of the original ref's parts into one name — so a ref
7
- * whose encoding would exceed it is simply not indexed (see {@link canIndex}).
8
- *
9
- * Re-exported from the validator rather than restated. The two were separate constants with the same value
10
- * until the validator started measuring the **encoded** length; `canIndex` went on counting raw characters and
11
- * began answering `true` for refs the registry then refused, turning a documented graceful degradation into a
12
- * guaranteed throw on the retention fast path.
13
- */
14
- export { MAX_NAME_LENGTH };
15
- /** Which bucket an expiry falls in. Total and reversible; no calendar, no timezone, no ambient time. */
16
- export declare function dueBucket(expiresAt: number): number;
17
- /** The namespace holding one bucket's pointers. */
18
- export declare function dueNamespace(bucket: number): string;
19
- /** Is this record a due-index pointer rather than a segment? */
20
- export declare function isDueIndexRow(record: Pick<RegistryRecord, 'namespace'>): boolean;
21
- /**
22
- * Encode a ref into one index-row name, unambiguously.
23
- *
24
- * `${namespaceLength}.${namespace}${segment}` — a decimal length, a dot, then the two parts concatenated. The
25
- * length prefix is what makes it reversible: a name may contain any character, the dot included, so no
26
- * separator character could ever be unambiguous on its own. Reading the digits
27
- * up to the first dot tells the parser exactly where the namespace ends.
28
- */
29
- export declare function encodeDueName(ref: SegmentRef): string;
30
- /**
31
- * Can this ref be indexed at all? False when the encoded name would exceed the 256-character name cap —
32
- * possible only for a ref whose namespace and segment are together near the limit.
33
- *
34
- * **Not indexable is not "not retired".** The repair scan still sees the segment's own row, so the consequence
35
- * is that it expires on the repair cadence instead of the fast one. Callers must not treat `false` as an error.
36
- */
37
- export declare function canIndex(ref: SegmentRef): boolean;
38
- /** The registry ref of the pointer for `ref` in `bucket`. Throws if the ref cannot be encoded — check first. */
39
- export declare function dueIndexRef(bucket: number, ref: SegmentRef): SegmentRef;
40
- /**
41
- * Recover the original ref from an index row's name. `null` for anything we did not write — a foreign row in
42
- * the reserved namespace is ignored rather than acted upon.
43
- */
44
- export declare function decodeDueName(name: string): SegmentRef | null;
45
- /**
46
- * Which buckets a cycle must read at `now`: everything from `since` up to and including the current one.
47
- *
48
- * Past buckets are included because a sweep that did not run — a scaled-to-zero worker, a failed deploy, a
49
- * paused schedule — must not leave its buckets stranded. `lookbackBuckets` bounds how far back a first run
50
- * reaches, so a long outage costs a bounded number of list calls per cycle rather than one per day since the
51
- * epoch; anything older is the repair scan's job.
52
- */
53
- export declare function dueBucketsAt(now: number, lookbackBuckets: number): number[];
@@ -1,293 +0,0 @@
1
- import { NotFoundError } from './errors.js';
2
- import type { GenKey, IStorageDriver, IRegistryDriver, KnownGeneration, RegistryRecord, SegmentRef, Token } from './ports.js';
3
- /** The two ports generation bookkeeping needs: the objects, and the pointer that says which one is current. */
4
- interface GenerationDeps {
5
- readonly storage: IStorageDriver;
6
- readonly registry: IRegistryDriver;
7
- /** What a delete loop waits on before it reads the row again ({@link watchDeletes}). Absent, it does not wait. */
8
- readonly clock?: {
9
- readonly sleep?: (ms: number) => Promise<void>;
10
- };
11
- }
12
- /**
13
- * The waits, in milliseconds, before each further read of the row when a delete loop finds the pointer on a generation
14
- * it deleted, with no object under it ({@link watchDeletes}). A rollback that moved the pointer there checks the bucket
15
- * after its move, and when the check runs after the delete it finds the object gone and puts the pointer back: one
16
- * listing of the segment's objects (one page while the segment holds under a thousand) and one compare-and-swap, two
17
- * round trips, three when a lease write lands first. A rollback client farther from the store than the deleting
18
- * client takes that much longer, so the loop reads the row once at once and, with a clock that can wait, three more
19
- * times over 1.75 s in all, which covers clients a few hundred milliseconds away. Paid only when a read finds the
20
- * pointer on a missing generation the loop deleted, never on a pass where nothing raced.
21
- */
22
- export declare const DELETE_SETTLE_WAITS_MS: readonly number[];
23
- /** What a delete loop has deleted, held to every read of the segment's row it makes after a delete. */
24
- interface DeleteWatch {
25
- /** Note a delete that landed. */
26
- deleted(generation: number): void;
27
- /** Every generation noted so far. */
28
- readonly gone: ReadonlySet<number>;
29
- /**
30
- * `row`, a read of the row made after `held`, the row a delete was decided on, held to what was deleted: the deleted
31
- * generation it leaves the pointer on, with no object under it once the row is read again, or `undefined`.
32
- */
33
- check(held: RegistryRecord, row: RegistryRecord | null): Promise<number | undefined>;
34
- /** The same for one more read of the row, made only when a delete landed since the last {@link check}. */
35
- settle(held: RegistryRecord): Promise<number | undefined>;
36
- }
37
- /**
38
- * Hold a delete loop's reads of the row to what it deleted.
39
- *
40
- * Every loop that deletes generations reads the row before each delete, and one round trip remains between that read
41
- * and the delete. A rollback that lands inside it onto the generation being deleted passes its own check (a listing made
42
- * before the delete) and leaves the pointer on an object that is gone. Nothing can put the object back, so the loop
43
- * detects it instead: every read of the row it makes after a delete, and one more after its last delete when it made no
44
- * read since, is held to what it deleted.
45
- *
46
- * A read names a deleted generation when its row is active, of the incarnation `held` was read from, and its pointer
47
- * names one. A tombstone never does, since no rollback moves a `destroyed` row's pointer, and neither does a row purged
48
- * and made again, whose numbers are its own. Then:
49
- *
50
- * - The bucket is asked, with one zero-byte read, a metadata request on every shipped driver. A number can be taken
51
- * again once its object is deleted (invariant 2): after a rollback below it, a load can number it afresh and
52
- * publish, and then the object there is that load's, and nothing is missing. Where the storage driver ignores a
53
- * delete's condition, the loop's own delete can take that object too, and the read finds it missing, which it is.
54
- * - A missing object is not yet the answer. A rollback whose own check ran after the delete finds the object gone and
55
- * puts the pointer back, and the loop's read can fall between its move and that undo. So the row is read again, at
56
- * once and, with a clock that can wait, after each of {@link DELETE_SETTLE_WAITS_MS}, each read judged as the first
57
- * was, the bucket asked again; the answer is the generation the last of them names, only if every one names a
58
- * deleted generation with no object under it.
59
- *
60
- * A read of the bucket that fails in any other way than "not found" throws: it says nothing about the object.
61
- */
62
- export declare function watchDeletes(deps: GenerationDeps, ref: SegmentRef): DeleteWatch;
63
- /** Who deleted the generation a {@link pointerOnDeletedError} names, which leads its message. */
64
- type Deleter = {
65
- readonly kind: 'collection';
66
- } | {
67
- readonly kind: 'erasure';
68
- } | {
69
- readonly kind: 'load';
70
- readonly published: number;
71
- } | {
72
- readonly kind: 'refused-load';
73
- readonly refusal: string;
74
- };
75
- /**
76
- * The error for a pointer a delete loop found on a generation it deleted ({@link watchDeletes}). The writer that moved
77
- * the pointer there is a rollback, or, where the storage driver ignores a delete's condition, a load that took the
78
- * number again and published. A rollback slower than the loop's further reads can still put the pointer back after
79
- * them, so the message says to look before acting.
80
- */
81
- export declare function pointerOnDeletedError(ref: SegmentRef, deleter: Deleter, generation: number): NotFoundError;
82
- /**
83
- * The generation number a writer should use for the segment's **next** object: one above the highest generation
84
- * the registry points at *or* that is present in Storage — whichever is higher. The erasure rewrite numbers this
85
- * way; a load numbers with {@link nextLoadGeneration}, which lists as this does when its existence check finds the
86
- * number taken or cannot answer.
87
- *
88
- * Both are consulted because they can disagree. A load that wrote its object and crashed before publishing leaves
89
- * an object *above* `currentGen`; a writer that consulted only the pointer would pick that same number and hit the
90
- * write-once conflict on every retry. Skipping past it costs one `list` and keeps the retry trivial — the crashed
91
- * object is an orphan {@link gcOrphanGenerations} collects once the pointer moves on. Reusing such an object
92
- * instead of skipping it (verify-and-publish) is `load()`'s job, not a numbering helper's.
93
- *
94
- * A segment with no row and no objects starts at 0. An admin/write-path helper, never on the read path.
95
- */
96
- export declare function nextGeneration(ref: SegmentRef, deps: GenerationDeps): Promise<number>;
97
- /** The number a load takes, and whether one existence check proved it free (so nothing is above the pointer). */
98
- interface LoadNumber {
99
- readonly generation: number;
100
- /**
101
- * `true` when the number is `currentGen + 1` and the check found no object holding it. Collection by name rests
102
- * on it: the row named `generation - 1` as current, so every number the window evicts is below it, and nothing
103
- * the listing would have kept is a name the pass takes. `false` when the listing numbered the load.
104
- */
105
- readonly checked: boolean;
106
- /** The generations that listing found, when the load listed: a seed of the ledger reads them instead of listing again. */
107
- readonly listed?: readonly number[];
108
- }
109
- /**
110
- * The generation number a load takes, from the row it has already read: `currentGen + 1` (0 with no row or no
111
- * pointer yet) when no object holds that number, which one existence check proves (a zero-byte `getTail`, one
112
- * metadata request on every shipped driver); otherwise one above the pointer and above everything listed, as
113
- * {@link nextGeneration} numbers. The check finding the number taken (a crashed load's orphan, the object of a
114
- * load still in flight, or the generations a rollback left above the pointer) and the check failing in any way
115
- * other than "not found" both take the listing, so the listing stays the authority whenever the check cannot
116
- * prove the number free. A failed check falls back silently: nothing records it, and what it costs is the listing,
117
- * one more PUT-class request on S3, so a fault that makes every check fail shows only in the bill.
118
- *
119
- * A load can therefore take a number **below** an object already in the bucket: an orphan at `currentGen + 2`
120
- * with `currentGen + 1` free is numbered under, not past. That is as safe as a listing's number. A load never
121
- * takes a number an object holds, and a put that races onto one fails write-once, so the load reports
122
- * `superseded`. The orphan above stays unpublished; the load whose check meets it numbers past it by the listing,
123
- * and collection takes it once a generation above it is current. Like a listing's, the number can be one whose
124
- * object was deleted (an erasure removes the generations above a rolled-back pointer that held the id), which is
125
- * why nothing identifies a generation by its number alone: the reader cache and the chunk cache key on the number and
126
- * the row's `pointerId`, which every write that takes a number again renews, and an open of a row with a usable
127
- * summary holds the object to the fingerprint the summary records; a pin keys on its object's fingerprint.
128
- */
129
- export declare function nextLoadGeneration(ref: SegmentRef, deps: GenerationDeps, row: RegistryRecord | null): Promise<LoadNumber>;
130
- /**
131
- * How often a load that could collect by name lists instead: every generation divisible by this. A name-only pass
132
- * takes the names its publish pushed out of the row's window and never looks at the rest, so what it misses (a crashed
133
- * or refused load's object, a delete that failed, extras a fault left after a `keep` that shrank, an object a rollback
134
- * or an erasure left stranded below the pointer) waits for a listing. The number is a generation's, not a clock's, so the choice needs
135
- * no state and no time, and a segment that loads cleanly lists once in this many loads at most. The bound is in
136
- * generation numbers: a rollback moves the pointer down, and the loads that then take the numbers above it count
137
- * from there.
138
- */
139
- export declare const LIST_COLLECTION_CADENCE = 16;
140
- /**
141
- * Garbage-collect superseded Storage generations for a segment: everything strictly below `currentGen`, keeping
142
- * the most recent `keep` of them as a grace window, so a read still fetching from a just-superseded
143
- * generation need not re-resolve mid-call (**invariant 4**). It is a window, not a lock: a read whose
144
- * generation is swept anyway re-resolves and retries once rather than failing (see `withFreshSnapshot`), so
145
- * `keep` trades storage for round trips. Generations ≥ `currentGen` are never touched. Returns the generations
146
- * deleted.
147
- *
148
- * **Except on a `destroyed` segment, where EVERY generation is garbage** and the grace window is meaningless.
149
- * A tombstoned segment resolves no generation for any reader, pinned or not — a pin cannot be taken on one, and
150
- * a pin taken while it was still active re-checks the status on every open — and nothing else in the library
151
- * would ever collect them, so without this those objects are billed forever.
152
- *
153
- * **Throws {@link NotFoundError} when a rollback moved the pointer onto a generation it deleted**: the rollback landed
154
- * between the pass's re-proof of the row and that delete, so the pointer names an object that is gone. The pass finds
155
- * it at its next re-proof, or, after its last delete, by reading the row once more. A deleted object cannot be
156
- * restored: the answer is a rollback to a generation still in the bucket.
157
- *
158
- * **Throws {@link WriteConflictError} when it cannot act on the row it read**: the row is gone when it is re-read,
159
- * after the listing or before a delete; on an active segment, the pointer has fallen below the cutoff before a
160
- * delete; on a `destroyed` segment, the row's token has changed. Not to be confused with the empty array it still
161
- * returns when there was genuinely nothing to collect (no row at all, or no pointer yet), which is why an empty
162
- * array is not a receipt. The row is read before the listing and acted on after it, so a name that was purged and
163
- * re-created in that window is a *different* segment wearing the same name, and its live object must not be
164
- * collected on the strength of the old row: on an active segment the lower of the two pointers is what keeps it
165
- * safe, and on a `destroyed` one the token. Refusing is safe — the objects are not going anywhere and the next
166
- * pass reads a consistent row — but it has to be **distinguishable** from "there was nothing to collect", which
167
- * an empty array is not: `eraseIdFromSegment` reads the returned list as the physical half of its erasure
168
- * receipt, and would otherwise report `erased: true` over bytes still in the bucket. Re-run it.
169
- *
170
- * That state is reachable in practice: a `dropSegment` whose Storage sweep threw part-way, or a load that had read
171
- * the row before the tombstone landed, wrote its object afterwards, and stopped before its refusal deleted it. `dropSegment` re-sweeps
172
- * and reports whatever it could not reclaim in `generationsRemaining`, but a drop that was never re-run leaves a
173
- * residual, and this is what eventually collects it from the retention sweep.
174
- *
175
- * `keep: 0` is how a subject-erasure rewrite makes a bit **physically** gone on return: the generation that held
176
- * it is collected the moment the rewrite is current. A live read still fetching from it heals forward to the
177
- * rewrite, and a pin of it (`seg.pin()`) fails with `NotFoundError` for any chunk it has yet to read.
178
- */
179
- export declare function gcOrphanGenerations(ref: SegmentRef, deps: GenerationDeps, options?: {
180
- keep?: number;
181
- }): Promise<number[]>;
182
- /**
183
- * {@link gcOrphanGenerations}, reporting a rollback onto a generation it deleted in `missing` instead of throwing it: the
184
- * pass stopped there. For a caller that records what it deleted before it raises, as an erasure does. With `skipKnown` the
185
- * pass also spares every generation the row names when it reads the row before a delete (an erasure's passes on an active
186
- * row, which remove from the row what they mean to delete first).
187
- */
188
- export declare function collectBelowPointer(ref: SegmentRef, deps: GenerationDeps, options?: {
189
- keep?: number;
190
- skipKnown?: boolean;
191
- }): Promise<{
192
- readonly deleted: number[];
193
- readonly missing?: number;
194
- readonly knownSpared?: number[];
195
- }>;
196
- /**
197
- * What makes a collection spare leased generations: the clock that says whether a lease has ended. Only a load's
198
- * collection passes one, and a load with no clock passes none: it cannot tell a live lease from an ended one, so it
199
- * reads none. A pass without a guard never reads the row's leases, which is what keeps erasure, shred and drop deleting
200
- * what a lease names.
201
- */
202
- interface LeaseGuard {
203
- readonly now: () => number;
204
- }
205
- /** What a load's publish recorded in the row, as `publishGeneration` reports it. */
206
- export interface PublishedKept {
207
- /** The window the publish recorded, as numbers, or `undefined` when it recorded none (the row then does not know which are kept). */
208
- readonly list: readonly number[] | undefined;
209
- /** The ledger the publish wrote, in the order the generations were made current, when it wrote one. */
210
- readonly known?: readonly KnownGeneration[];
211
- /** The names the publish pushed out of the window, ascending. */
212
- readonly evict: readonly number[];
213
- /** The token of the row the publish wrote, when it is known: a write settled by reading the row has none. */
214
- readonly token: Token | undefined;
215
- }
216
- /**
217
- * Collect after a load's publish. The row's ledger records the generations the load keeps, so the pass deletes by name
218
- * what the publish pushed out of that window ({@link deleteEvicted}) and lists the segment ({@link gcOrphanGenerations}'s
219
- * pass) only when it cannot rely on the row or must catch what a name does not:
220
- *
221
- * - the publish recorded no window to collect by name (a `keep` above {@link MAX_KEPT_GENERATIONS}, a publish that
222
- * seeded its ledger from the bucket, a row with no usable ledger, or a publish whose write was settled by reading the
223
- * row): the pass keeps `keep` of the generations present below the pointer, the ledger's own most recently current
224
- * names first and then the newest others by number, or, over a row with no usable ledger, `previous` first and then
225
- * the newest by number ({@link keptWindow}), and writes nothing: the publish already wrote the ledger, which is what
226
- * the next load derives its window from. It spares every name the row knows when it re-reads the row before a
227
- * delete, which a publish or a rollback that landed since can have added to;
228
- * - the guard found the current generation's object gone, or, for a `keep` of 1 or more when the guard took the size
229
- * from the row's summary and opened nothing, one zero-byte read of it does not find it (`proveCurrent`): the
230
- * window then counts a generation that is not there, so the same pass keeps what is;
231
- * - a number the listing chose (something sits above the pointer, or the existence check could not answer), a
232
- * caller that asks for a listing, and every {@link LIST_COLLECTION_CADENCE}th generation: the pass deletes every
233
- * generation below the pointer the row does not name, which collects what a name-only pass leaves.
234
- *
235
- * It makes no request when `keep` is at least the generation published, since no more generations than that exist
236
- * below it. A load onto a `destroyed` row is refused at its publish and never reaches here; a drop that lands after
237
- * the publish leaves this pass collecting the tombstone's generations, and the drop's own sweep takes the rest.
238
- *
239
- * Any fault of the collection is raised, as every fault of a collection after a landed publish is.
240
- *
241
- * So is a rollback onto a generation the pass deleted, which the pass finds by holding every read of the row it makes
242
- * after a delete to what it deleted, and by reading the row once more after its last delete
243
- * ({@link watchDeletes}): it throws {@link NotFoundError}, saying the load took effect at `generation` and the
244
- * generation the pointer names is missing. That read is the one request this adds to a load whose pass deleted
245
- * anything.
246
- */
247
- export declare function collectAfterLoad(ref: SegmentRef, deps: GenerationDeps, options: {
248
- generation: number;
249
- keep: number;
250
- /** The load's number was proved free by one existence check, and no caller asked for a listing. */
251
- byName: boolean;
252
- /** The guard opened the current generation's object and found it gone. */
253
- currentGone?: boolean;
254
- kept: PublishedKept | undefined;
255
- /**
256
- * The object the load's row named as current, which the caller did not open. Before it collects by name, the pass
257
- * looks for this object with one zero-byte read, and lists instead unless that finds it: a name-only pass is
258
- * safe only while the object the row named is in the bucket.
259
- */
260
- proveCurrent?: GenKey;
261
- /** Spare the generations the row's live leases hold. Absent, the pass ignores leases. */
262
- leases?: LeaseGuard;
263
- /**
264
- * The generation the row named as current when the load read it, which its publish superseded. A window kept by
265
- * listing holds it first, while the pointer is still the load's. Absent: the load found no pointer.
266
- */
267
- previous?: number;
268
- }): Promise<number[]>;
269
- /**
270
- * Delete, with no listing, the generations a publish of `generation` pushed out of its row's window, `evict`, taken
271
- * from the list the publish was written against. Returns the names asked for, whether or not an object was there: a
272
- * delete of an absent object succeeds on every backend and says nothing, so a generation a concurrent pass, an erasure
273
- * or a lifecycle rule had already taken is listed all the same. A list is not a receipt (invariant 4), and neither is
274
- * this.
275
- *
276
- * Each delete re-reads the row first (invariant 4) and stops the pass, deleting nothing more and raising nothing, unless
277
- * the row still has a pointer at or above `generation` (a rollback, or a name purged and re-created with fewer loads,
278
- * has moved it below: the publish already landed, nothing here protects it or needs the result), the name is below
279
- * that pointer, and the row's own list is usable and does not name it. The last condition is what keeps a late delete
280
- * from taking a generation that a later publish, after a rollback, put in the window. A pointer that has moved above
281
- * `generation` does not stop it: a publish landing in the meantime leaves routine collection working. A fault, a
282
- * registry read or a delete that throws, propagates, with the pointer at `generation`.
283
- *
284
- * A rollback onto a name it deleted is not a stop: every read of the row after a delete is held to what it deleted, and
285
- * so is one more read after its last delete when it made none since ({@link watchDeletes}). One that names a
286
- * deleted generation throws {@link NotFoundError}, saying the load took effect at `generation`.
287
- */
288
- export declare function deleteEvicted(ref: SegmentRef, deps: GenerationDeps, options: {
289
- generation: number;
290
- evict: readonly number[];
291
- leases?: LeaseGuard;
292
- }): Promise<number[]>;
293
- export {};