@cloudbitmaps/core 0.20.0 → 0.21.0

package/dist/core/crbm/format.d.ts

@@ -37,7 +37,6 @@ export declare const FOOTER: {
 export declare const FOOTER_CRC_COVERAGE: 96;
 /** `flags` bit positions. */
 export declare const FLAG_ENCRYPTED: number;
-export declare const FLAG_INDEX_COMPRESSED: number;
 export declare const FLAG_LITTLE_ENDIAN: number;
 /**
  * The object carries an extension block just before its index (a generation's metadata). Set only when there is

package/dist/core/crbm/plan-reads.d.ts

@@ -34,11 +34,17 @@ export interface ChunkRegion {
 }
 /**
  * Plan the reads for `extents`: the needed chunks of one object, ascending by offset (the index lists chunks in key
- * order, which is object order), each at least a byte long, none overlapping another. The reads come back
- * ascending and disjoint, each covering one or more of the chunks, every chunk in exactly one read.
+ * order, which is object order), each at least a byte long, none overlapping another. The reads come out ascending
+ * and disjoint, each covering one or more of the chunks, every chunk in exactly one read.
+ *
+ * Lazy: the plan is a walk, so a consumer that takes the first reads pulls only the extents those reads cover (and
+ * the one after, which closes the last of them), and a plan over a very wide object costs nothing until it is read.
+ * Each extent is checked as the walk reaches it, before the read that would carry it is handed out, so no read ever
+ * covers an extent that was not checked; a plan taken to its end has checked them all. The reads are exactly the ones
+ * a plan made all at once would hold, in the same order.
  *
  * Throws {@link IntegrityError} for an extent that is not a pair of safe integers, or is outside `region`, or out of
  * order, or overlapping the one before it:
  * the index a plan is made from is untrusted bytes, and a read is never planned outside the chunk region.
  */
-export declare function planChunkReads(extents: readonly ChunkExtent[], region: ChunkRegion): readonly PlannedRead[];
+export declare function planChunkReads(extents: Iterable<ChunkExtent>, region: ChunkRegion): Generator<PlannedRead, void, undefined>;

package/dist/core/crbm/reader.d.ts

@@ -7,7 +7,7 @@ import type { ChunkRead, GenerationMetadata, ReadChunksOptions } from '../ports.
  * `cardinalityMinusOne` stores `cardinality - 1` because a chunk's cardinality runs `[1, 65536]` and 65,536 does
  * not fit a `u16`; an offset is a `f64` because an object past 4 GiB puts a payload beyond a `u32`.
  */
-export interface ParsedIndex {
+interface ParsedIndex {
     readonly keys: Uint16Array;
     readonly cardinalityMinusOne: Uint16Array;
     readonly lengths: Uint32Array;
@@ -29,7 +29,12 @@ export declare const RETAINED_BYTES_PER_INDEX_ENTRY = 20;
  */
 export declare function indexCapacity(indexLength: number): number;
 export interface CrbmReaderOptions {
-    /** Speculative tail size in bytes (default 256 KB; clamped up to at least the footer size). */
+    /**
+     * How many bytes the first tail read asks for, a whole number (default 256 KiB; clamped up to at least the footer
+     * size). Any value of at least an object's `tailBytes` opens it in that one read. Where an open keeps a small object's
+     * chunks (a `CrbmStorageChunkSource`'s does), it keeps them only when this read covers the whole object, so a value
+     * below the default can leave them to reads of their own.
+     */
     readonly tailBytes?: number;
     /** Hard cap on a single chunk payload length (default: the 1 MiB decode cap, plus the 28-byte nonce and tag on an encrypted object). The engine decodes under `maxBitmapBytes`; raise this with it. */
     readonly maxPayloadBytes?: number;
@@ -49,6 +54,20 @@ export interface CrbmReaderOptions {
      */
     readonly lineage?: unknown;
 }
+/**
+ * Refuses a `size` that is not a whole byte count its own tail fits in. The size is the tier's word too: one that
+ * is not would turn off every bounds check an open makes against it.
+ */
+export declare function checkSize(tail: Uint8Array, size: number): void;
+/**
+ * What the footer of the object behind `blob` says with no key and no index, from one tail read of a footer's worth:
+ * whether it is encrypted, and its CRC (the one in {@link CrbmReader.fingerprint}). A footer that fails its own checks
+ * throws, as an open would.
+ */
+export declare function readFooter(blob: BlobReader): Promise<{
+    readonly encrypted: boolean;
+    readonly footerCrc: number;
+}>;
 /**
  * Whether the object behind `blob` says it is encrypted, from one tail read of a footer's worth, with no key and no
  * index; a footer that fails its own checks throws, as an open would. For a caller that must not open an object with
@@ -61,6 +80,14 @@ export declare function footerSaysEncrypted(blob: BlobReader): Promise<boolean>;
  * object and that region is at most `keepChunkBytesUpTo` bytes. Not exported from a package entry.
  */
 export declare function openCrbmReaderKeeping(blob: BlobReader, options: CrbmReaderOptions, keepChunkBytesUpTo: number | undefined): Promise<CrbmReader>;
+/**
+ * How many bytes the first tail read of an open asks for: the caller's `tailBytes` or the default, and a footer's worth
+ * at least. Not exported from a package entry: {@link CrbmReader.open} and a caller that sends that read ahead of it
+ * ask for the same size.
+ */
+export declare function openTailBytes(options: {
+    readonly tailBytes?: number;
+}): number;
 export declare class CrbmReader {
     private readonly blob;
     private readonly objectSize;
@@ -102,6 +129,11 @@ export declare class CrbmReader {
     get fingerprint(): string;
     /** Total object bytes (from the one-GET tail read): the `sizeBytes` a segment's `stat()` reports. */
     get sizeBytes(): number;
+    /**
+     * The bytes of the object after its chunk payloads: any extension block, the index and the footer. A tail read of
+     * this many bytes covers all of them.
+     */
+    get tailBytes(): number;
     /**
      * Retained JS heap of this reader's parsed index — the weight the storage reader cache bounds on (a wide
      * segment's parsed index, not its payloads, dominates the reader's footprint): the exact byte length of the
@@ -134,6 +166,16 @@ export declare class CrbmReader {
     count(): number;
     /** The slot holding `chunkKey` (binary search over the ascending keys), or -1 when this generation has none. */
     private slotOf;
+    /**
+     * {@link slotOf} for a key at or after the one that held slot `at`, which a walk over ascending keys has: the slot
+     * itself or the next are tried first, so a read of nearly every chunk costs no search.
+     */
+    private slotFrom;
+    /**
+     * The extents of the chunks `chunkKeys` name that this generation holds, in key order, each once (a key given twice
+     * is read once). A walk, so nothing is looked up until the plan pulls it.
+     */
+    private neededExtents;
     has(chunkKey: number): boolean;
     /**
      * Range-GET one chunk's payload, verifying its CRC32C before returning. Returns `null` if the chunk is
@@ -168,10 +210,13 @@ export declare class CrbmReader {
      * as {@link getChunk} answers for an absent key. A key given twice is read once and answered at both positions with
      * the same view.
      *
-     * The whole key list is planned once, up front, against this one generation's index. The ranges then run through a
-     * window of `options.concurrency` reads ({@link MAX_RANGES_IN_FLIGHT} by default) that counts a range until the
-     * consumer has taken it, so a stream never holds more than that many ranges, in flight or landed, however many keys
-     * it was given or how slowly it is read; with `options.ramp` the window opens 1, 2, 4 … wide. A range is at most 1 MiB
+     * The key list is planned against this one generation's index ({@link planChunkReads}), as a walk the window pulls
+     * from only as it opens a range, so a stream that stops early has planned little: the same ranges, in the same
+     * order, as a plan of the whole list would make, and each chunk's extent is checked before the range that carries
+     * it is requested. The ranges run through a window of `options.concurrency` reads ({@link MAX_RANGES_IN_FLIGHT} by
+     * default) that counts a range until the consumer has taken it, so a stream never holds more than that many ranges,
+     * in flight or landed, however many keys it was given or how slowly it is read; with `options.ramp` the window opens
+     * 1, 2, 4 … wide. A range is at most 1 MiB
      * unless it is one chunk, which is at most the payload cap, so the bound is that many times the per-chunk cap
      * (1 MiB, and 28 bytes more for an encrypted chunk). Nothing is read until the first item is asked for, and a
      * consumer that stops launches no more: ranges already in flight finish and are dropped, and a failure of one of
@@ -223,3 +268,4 @@ export type ChunkStream = AsyncGenerator<Omit<ChunkRead, 'version'>> & {
  */
 export declare function parseExtension(sections: Uint8Array, crypto: CrbmCrypto | undefined): GenerationMetadata | undefined;
 export declare function parseIndex(indexBytes: Uint8Array, payloadEnd: number, maxPayloadBytes: number, minPayloadBytes?: number): ParsedIndex;
+export {};

package/dist/core/crbm/varint.d.ts

@@ -1,6 +1,6 @@
 /** Append the LEB128 encoding of a u32 `value` to `out`. */
 export declare function writeVarint(out: number[], value: number): void;
-export interface VarintRead {
+interface VarintRead {
     readonly value: number;
     /** Offset just past the bytes consumed. */
     readonly next: number;
@@ -11,3 +11,4 @@ export interface VarintRead {
  * cleanly, never loop or overflow.
  */
 export declare function readVarint(bytes: Uint8Array, offset: number): VarintRead;
+export {};

package/dist/core/crbm/writer.d.ts

@@ -35,6 +35,8 @@ export declare class CrbmWriter {
     private readonly metadata;
     /** Bytes written so far, the footer included once {@link finish} has run. */
     private written;
+    /** Where the chunk payloads end: the extension block's start when there is metadata, else the index's. */
+    private payloadEnd;
     private footerCrc32c;
     constructor(sink: BlobSink, options: CrbmWriterOptions);
     /**
@@ -50,11 +52,13 @@ export declare class CrbmWriter {
     /**
      * The finished object's size and its footer's CRC: what names it, as a reader's fingerprint does. A writer that
      * has to tell its own object from another stored later under the same key compares these with the object's
-     * footer. Throws before {@link finish}.
+     * footer. `tailBytes` is the bytes after the chunk payloads (any extension block, the index and the footer), which
+     * is what {@link CrbmReader.tailBytes} reports for the object. Throws before {@link finish}.
      */
     get identity(): {
         readonly size: number;
         readonly footerCrc: number;
+        readonly tailBytes: number;
     };
     private ensurePreamble;
     /**

package/dist/core/crbm-open.d.ts

@@ -0,0 +1,154 @@
+import type { BlobReader } from './blob.js';
+import type { CrbmCrypto } from './crypto.js';
+import type { GenerationMetadata, GenKey, IStorageDriver, TailRead } from './ports.js';
+import { CrbmReader } from './crbm/reader.js';
+import type { CrbmReaderOptions } from './crbm/reader.js';
+import type { SummaryDescription } from './summary.js';
+/**
+ * Adapt one `(driver, key)` pair to the codec's `BlobReader` seam. With `noteTail`, each tail read the reader makes is
+ * shown to it with the bytes, `size` and `version` the driver answered, after the copy below when one is made.
+ *
+ * A tail's bytes reach the reader as a plain `Uint8Array`: the reader keeps a small object's chunk region by slicing
+ * the tail, and a subclass's `slice` need not copy. A Node `Buffer`'s is a view, so a driver that answered with one
+ * would leave the reader holding the driver's memory, all of it, as the cache weighs only the region. Such an answer
+ * is copied here, once ({@link ownedTail}); a plain `Uint8Array` of this realm, which every shipped driver returns, is
+ * handed through as it is.
+ */
+export declare function storageBlobReader(driver: IStorageDriver, key: GenKey, noteTail?: (tail: TailRead) => void): BlobReader;
+/**
+ * `tail` with its bytes in a plain `Uint8Array` of this realm. Any other `Uint8Array`, a subclass (a Node `Buffer`) or
+ * one made in another realm (a `vm` context), is copied once, and the copy is built from the answer's fields by name,
+ * so a `size` or `version` the driver serves from a getter on its own class carries over. The brand is read, not
+ * `instanceof`, which is false for another realm's arrays. A plain `Uint8Array` of this realm is handed through as it is.
+ */
+export declare function ownedTail(tail: TailRead): TailRead;
+/**
+ * The object under `key`, when one read of its footer matches `fingerprint` ({@link CrbmReader.fingerprint}), with the
+ * version the driver reported on that read (`undefined` from a driver that reports none). `null` when it is another
+ * object, when it is gone, and when the read fails in any way. A fingerprint names the object's bytes, not the writer:
+ * an object another writer stored under the number with the same bytes matches too. A caller that deletes what this
+ * matches passes the version as the delete's `ifVersion`, so a number taken again between the read and the delete keeps
+ * the object put under it since.
+ */
+export declare function provenObject(storage: IStorageDriver, key: GenKey, fingerprint: string): Promise<{
+    readonly version: string | undefined;
+} | null>;
+/**
+ * Whether the object under `key` is still the one `fingerprint` names ({@link CrbmReader.fingerprint}), from one read
+ * of its footer. Unlike {@link provenObject}, a fault is not an answer: `NotFoundError` when the key holds no object,
+ * and any other failure of the read, thrown as it is.
+ */
+export declare function sameObjectUnder(storage: IStorageDriver, key: GenKey, fingerprint: string): Promise<boolean>;
+/**
+ * Whether the object under `key` says it is encrypted, from one read of its footer, with no key. `noteVersion` is shown
+ * the version the driver reported on that read, when it reported one: a caller that deletes the object for what its
+ * footer says passes it as the delete's `ifVersion`.
+ */
+export declare function objectIsEncrypted(storage: IStorageDriver, key: GenKey, noteVersion?: (version: string) => void): Promise<boolean>;
+/**
+ * What one tail read of a rollback target says about it with no key: whether the footer says it is encrypted and its
+ * CRC, and the remembered tail an open from it ({@link openRollbackTargetFrom}) is answered from.
+ */
+interface RollbackTargetFooter {
+    readonly encrypted: boolean;
+    readonly footerCrc: number;
+    readonly blob: BlobReader;
+}
+/**
+ * The footer step of a rollback: one tail read of the object under `key`, which is remembered for the open that may
+ * follow. A footer that fails its own checks is an {@link IntegrityError}; a missing object is {@link NotFoundError}.
+ */
+export declare function readRollbackTargetFooter(storage: IStorageDriver, key: GenKey): Promise<RollbackTargetFooter>;
+/**
+ * The reader {@link RollbackTargetFooter}'s object opens as, from its remembered tail, when the object and the key
+ * agree (an encrypted object opened with `crypto`, a cleartext one with none); `undefined` when they disagree, which
+ * the caller refuses as a target the row cannot use. An object that is another generation's, and an index or metadata
+ * that do not open, are an {@link IntegrityError}.
+ */
+export declare function openRollbackTargetFrom(footer: RollbackTargetFooter, key: GenKey, crypto: CrbmCrypto | undefined): Promise<CrbmReader | undefined>;
+/**
+ * The footer CRC of the object under `key` ({@link CrbmReader.fingerprint} carries it), from one read of a footer's
+ * worth, so it reads an encrypted object's CRC as it does a cleartext one's, with no key. {@link NotFoundError} when
+ * the key holds no object; a footer that fails its own checks is an {@link IntegrityError}.
+ */
+export declare function footerCrcUnder(storage: IStorageDriver, key: GenKey): Promise<number>;
+/**
+ * Whether the object under `key` is the one `fingerprint` names, from one read of its footer: false when it is another
+ * object or gone. Unlike {@link provenObject}, a read that cannot tell throws, for a caller that must not take "could
+ * not look" for "not this object".
+ */
+export declare function provesOwnObject(storage: IStorageDriver, key: GenKey, fingerprint: string): Promise<boolean>;
+/**
+ * Open a {@link CrbmReader} on one generation over the storage driver's range/tail reads (decrypting if `crypto`).
+ * The reader the write paths use to re-read what they wrote, and the erasure rewrite uses to stream the old
+ * generation; the engine's read path goes through {@link CrbmStorageChunkSource} instead, which caches these. With
+ * `named`, the fingerprint the row's summary records, the object must be that one: another is {@link NotFoundError}.
+ * The version of the object it opened, from the tail read the open made, is what {@link objectVersionOf} answers.
+ */
+export declare function openGenerationReader(storage: IStorageDriver, key: GenKey, crypto: CrbmCrypto | undefined, options?: Omit<CrbmReaderOptions, 'crypto'>, named?: string): Promise<CrbmReader>;
+/**
+ * {@link openGenerationReader}, with `noteVersion` shown the version {@link objectVersionOf} would answer as soon as the
+ * tail read that carries it returns, before the open goes on: a caller whose open then fails, on an object sealed under
+ * another key, say, still knows which object it met, and can condition a delete of it on that read.
+ */
+export declare function openGenerationReaderNoting(storage: IStorageDriver, key: GenKey, crypto: CrbmCrypto | undefined, noteVersion: (version: string) => void): Promise<CrbmReader>;
+/**
+ * The version of the object `reader` was opened on, as the driver reported it on the tail read the open made, for a
+ * delete conditioned on that object ({@link StorageDeleteOptions.ifVersion}); `undefined` for a reader
+ * {@link openGenerationReader} did not open, or a driver that reports none. Held beside the reader rather than on it, so
+ * nothing is added to the reader's public surface, and no read the engine makes pays for it.
+ */
+export declare function objectVersionOf(reader: CrbmReader): string | undefined;
+/**
+ * {@link openGenerationReader}, of the object `summary.fingerprint` names: `undefined` when the object under the key is
+ * another one, found from its footer before its index is opened, so an object sealed under another key is told apart as
+ * well. The tail read is sized from the summary's `tailBytes` ({@link sizedTailBytes}), so a wide generation's index
+ * comes in that read.
+ */
+export declare function openNamedGeneration(storage: IStorageDriver, key: GenKey, crypto: CrbmCrypto | undefined, summary: Pick<SummaryDescription, 'fingerprint' | 'tailBytes'>, noteTail?: (tail: TailRead) => void): Promise<CrbmReader | undefined>;
+/**
+ * Open a reader on `key`, refusing an object whose footer names another generation. Every writer stamps the key's
+ * number, and the chunk cache, the load guard and the erasure rewrite all trust it, so an object that disagrees
+ * was written under another key or altered. Every open, the live read's, a pin's and the write paths', comes here.
+ * With `named`, the object must also be the one that fingerprint names: another one is {@link NotFoundError}, a move.
+ * With `early`, the tail read {@link startTailRead} began, the open takes that read as its own instead of making one.
+ */
+export declare function openChecked(storage: IStorageDriver, key: GenKey, options: CrbmReaderOptions, keepChunkBytesUpTo?: number, named?: string, noteTail?: (tail: TailRead) => void, early?: Promise<TailRead>): Promise<CrbmReader>;
+/**
+ * The size of the tail read an open makes when its row's summary says how many bytes of the object's end the open
+ * needs (`tailBytes`: any extension block, the index and the footer), so that one read reaches the index of a wide
+ * generation and no range read follows it. With no summary it is the source's own size ({@link openTailBytes}).
+ *
+ * The number came from a registry the library may not have written, so it only ever sizes a read: never below the
+ * source's own tail, at most the object less its preamble (the size its fingerprint names), and at most the source's
+ * `maxIndexBytes` (or the default) plus an extension block, its trailer and the footer, the most any open reads past
+ * its own tail. A hint that is not a whole number of bytes above zero is ignored, and so is a result that is not one. A
+ * wrong hint is a longer read or one more range read, never a different answer: the reader checks every byte it takes.
+ */
+export declare function sizedTailBytes(summary: Pick<SummaryDescription, 'fingerprint' | 'tailBytes'> | undefined, options: {
+    readonly tailBytes?: number;
+    readonly maxIndexBytes?: number;
+}): number;
+/**
+ * Send the tail read an open of `key` makes, now, and hand back its answer for {@link openChecked}'s `early`: the open
+ * then waits on the one request instead of sending its own, so work the open waits on first, such as a keystore call,
+ * runs beside the request and not before it. Nothing is parsed here: the bytes are untrusted until the open reads them.
+ * The returned promise already has a handler attached, so a caller that never reaches the open (its key failed) leaves
+ * no unhandled rejection; the open that does reach it still sees the rejection.
+ */
+export declare function startTailRead(storage: IStorageDriver, key: GenKey, options: CrbmReaderOptions): Promise<TailRead>;
+/**
+ * Re-open a freshly written generation and assert it round-trips exactly what was streamed into it: the same
+ * per-chunk key set *and* the same total cardinality (on top of the codec's own per-chunk CRC + footer checks), and the
+ * metadata it was given. `expected` is the streaming writer's tally (the stream is consumed, so it can't be
+ * re-iterated) — the key-set comparison catches a dropped/extra chunk that a cardinality-only check could miss when
+ * two errors cancel out. The count and the metadata are what the generation's row summary is about to claim, so the
+ * summary is held to the object it describes before it is published.
+ * Throws {@link IntegrityError}: the object is on disk but must not be published.
+ */
+export declare function verifyGeneration(storage: IStorageDriver, key: GenKey, expected: {
+    readonly chunkKeys: readonly number[];
+    readonly cardinality: number;
+    readonly metadata?: GenerationMetadata | undefined;
+}, crypto: CrbmCrypto | undefined): Promise<void>;
+export {};