@cloudbitmaps/core 0.12.0 → 0.14.0

package/dist/core/chunk-window.d.ts

@@ -0,0 +1,24 @@
+/**
+ * An ordered window of reads: up to `max` reads open ahead of the one being consumed, handed back in key order.
+ * With `ramp`, the window opens 1, 2, 4 … wide up to `max` instead of `max` at once, so a consumer that stops
+ * after a few chunks has fetched a handful, not a full window.
+ *
+ * `T` is whatever one read yields (a decoded chunk, or the raw bytes of one). Each read is wrapped to resolve and
+ * never reject, so a read nobody consumes (the consumer stopped, or an earlier chunk failed) cannot raise an
+ * unhandled rejection; its error surfaces from {@link take} only if the consumer reaches that chunk. Memory is
+ * bounded by the window: at most `max` results are held ahead.
+ *
+ * Internal to core: not exported from any entry point.
+ */
+export declare class ChunkWindow<T> {
+    private readonly keys;
+    private readonly fetch;
+    private readonly max;
+    private readonly ramp;
+    private readonly open;
+    private launched;
+    private taken;
+    constructor(keys: readonly number[], fetch: (chunkKey: number) => Promise<T>, max: number, ramp: boolean);
+    /** The next read's result in key order, or the error that read raised. */
+    take(): Promise<T>;
+}

package/dist/core/codec.d.ts

@@ -52,6 +52,17 @@ export interface CodecBitmap {
     /** Ascending iterator over the set values. */
     [Symbol.iterator](): IterableIterator<number>;
     toArray(): number[];
+    /**
+     * The set's values as one ascending `Uint32Array` the caller owns: what a batch read converts a chunk with, in one
+     * native step instead of one iterator step per value.
+     *
+     * **Optional.** A codec without it is read through its iterator, which is correct and slower.
+     *
+     * **It must return a NEW array on every call, which the caller owns:** the engine adds the chunk's base to each
+     * value in place and hands the array on, so an array that is shared, cached or a view of the bitmap's own storage
+     * would be corrupted, and would corrupt the bitmap. It holds exactly what the iterator yields, ascending.
+     */
+    toUint32Array?(): Uint32Array;
     /**
      * The largest value in the set, or `undefined` when empty.
      *

package/dist/core/crbm/format.d.ts

@@ -104,8 +104,11 @@ export declare const DEFAULT_TAIL_BYTES: number;
  * above the ~650 KB full-span worst case, well below a denial-of-wallet allocation.
  */
 export declare const DEFAULT_MAX_INDEX_BYTES: number;
-/** Hard cap on a single chunk payload, defending the native deserializer against oversized input. */
-export declare const DEFAULT_MAX_PAYLOAD_BYTES: number;
+/**
+ * Largest serialized bitmap the engine decodes (1 MiB, per chunk). A well-formed 16-bit chunk serializes to at most
+ * about 8.2 KiB, so no legitimate chunk is near it; both the read engine and the erase path decode under this one cap.
+ */
+export declare const DEFAULT_MAX_BITMAP_BYTES: number;
 /** A chunk's cardinality is in `[1, 65536]` (empty chunks are never written). */
 export declare const MAX_CHUNK_CARDINALITY = 65536;
 /**
@@ -142,3 +145,9 @@ export declare const CRC32C_BYTES = 4;
  */
 export declare const AEAD_NONCE_BYTES = 12;
 export declare const AEAD_TAG_BYTES = 16;
+/**
+ * Hard cap on one encrypted chunk's stored payload (a cleartext one is capped at the decode cap): the decode cap plus the AEAD framing, so an
+ * entry the engine would refuse to decode is refused when the object is opened, before any payload is read. No writer
+ * exceeds it: a stored payload is a serialized 16-bit chunk (at most about 8.2 KiB) plus, when encrypted, that framing.
+ */
+export declare const DEFAULT_MAX_PAYLOAD_BYTES: number;

package/dist/core/crbm/reader.d.ts

@@ -31,7 +31,7 @@ export declare function indexCapacity(indexLength: number): number;
 export interface CrbmReaderOptions {
     /** Speculative tail size in bytes (default 256 KB; clamped up to at least the footer size). */
     readonly tailBytes?: number;
-    /** Hard cap on a single chunk payload length (default 16 MB). */
+    /** Hard cap on a single chunk payload length (default: the 1 MiB decode cap, plus the 28-byte nonce and tag on an encrypted object). The engine decodes under `maxBitmapBytes`; raise this with it. */
     readonly maxPayloadBytes?: number;
     /** Hard cap on the whole index region fetched/parsed from one object (default 8 MB). */
     readonly maxIndexBytes?: number;

package/dist/core/crbm-storage-source.d.ts

@@ -426,7 +426,7 @@ export interface StreamWriteResult {
  * Streaming variant of {@link writeCrbmGeneration} for a **constant-memory** rewrite (the erasure rewrite, a
  * sorted load): consumes an **already-ascending** async stream of `{ chunkKey, bitmap }`, feeding each to the
  * codec and freeing it, instead of materializing the whole generation. Paired with a streaming storage sink (S3
- * multipart / LocalFs temp file), peak memory is ~one chunk + one part. Returns a tally so the caller can verify
+ * multipart / LocalFs temp file), the writer itself holds ~one chunk + one part. Returns a tally so the caller can verify
  * the re-opened object without re-iterating the (now-consumed) stream. Input **must** be ascending by `chunkKey`
  * — the codec rejects an out-of-order chunk; empty bitmaps are skipped.
  */

package/dist/core/engine.d.ts

@@ -8,6 +8,7 @@ export interface EngineDeps {
     readonly storage: StorageChunkSource;
     /** Optional cache of decoded (immutable) Storage chunks. */
     readonly cache?: BoundedLru<string, CodecBitmap>;
+    /** Per-chunk decode ceiling (invariant 5); defaults to 1 MiB. The `.crbm` reader refuses an entry above its own `maxPayloadBytes` (1 MiB, plus 28 bytes when encrypted) at open, so raise that on the chunk source too. */
     readonly maxBitmapBytes?: number;
     /**
      * The bitmap codec — **required**. `core/` is codec-agnostic: it can have no default, because the concrete codec
@@ -47,7 +48,13 @@ export interface IdRange {
 }
 /** Options common to the chunk-aligned combines. */
 export interface CombineOptions extends IdRange {
-    /** Max chunk keys resolved concurrently — bounds the Storage footprint. A positive integer. */
+    /**
+     * Max chunk keys resolved concurrently — bounds the Storage footprint (about `concurrency × operands` chunks in
+     * flight). A positive integer; default 32. The window opens 8 keys wide (or `concurrency`, if lower) and widens
+     * as keys are taken, so a read that stops early fetches no more than a few keys ahead; one that runs to the end
+     * spends nearly all of its round trips at the full width. A read that stops early has requested up to
+     * `concurrency` keys per operand past the last one it used.
+     */
     readonly concurrency?: number;
     /** Override the store's per-op budget for this call (`false` lifts it). */
     readonly budget?: BudgetOption;
@@ -70,6 +77,12 @@ export declare class SegmentEngine {
     private readonly metricsOn;
     /** Resolved per-op budget (null = disabled); undefined deps ⇒ the generous default. */
     private readonly budget;
+    /**
+     * The chunk reads open right now, by cache key: a caller that misses the cache and finds its key here awaits that
+     * read instead of making its own. An entry lives from the request until it settles, so the map holds at most one
+     * promise per distinct key in flight.
+     */
+    private readonly openReads;
     constructor(deps: EngineDeps);
     /** Membership: one chunk lookup — the cache, else one Storage fetch of that chunk. */
     has(seg: SegmentRef, id: number): Promise<boolean>;
@@ -104,7 +117,7 @@ export declare class SegmentEngine {
     /** The chunks of `seg` at `chunkKeys`, read through a window of {@link DEFAULT_INTERSECT_CONCURRENCY}. */
     private chunkWindow;
     /**
-     * Every id, ascending, reading ahead through a window of up to 8 chunk fetches that opens 1, 2, 4, 8 wide, so a
+     * Every id, ascending, reading ahead through a window of up to 32 chunk fetches that opens 1, 2, 4 … 32 wide, so a
      * read that stops early has fetched only a handful of chunks past the one it stopped in; with `range`, only the ids in `(after, through]`, fetching only the
      * chunks the range overlaps (see {@link IdRange}).
      *
@@ -115,6 +128,13 @@ export declare class SegmentEngine {
     iterate(seg: SegmentRef, range?: IdRange): AsyncGenerator<number>;
     private iterateAll;
     private iterateRange;
+    /**
+     * {@link iterate}, one `Uint32Array` per chunk instead of one id at a time: the same chunks, in the same order, with
+     * the same ids (a range cut at its edges the same way), and no array for a chunk that holds none of them. The
+     * reads, the budget charge and the window are the per-id read's. Each array is at most 65,536 ids (256 KiB) and is
+     * the caller's to keep.
+     */
+    iterateBatches(seg: SegmentRef, range?: IdRange): AsyncGenerator<Uint32Array>;
     /**
      * `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
      *
@@ -163,7 +183,20 @@ export declare class SegmentEngine {
      * the cost scales with the segment being filtered rather than with the size of the suppression list.
      */
     andNot(seg: SegmentRef, excludes: readonly SegmentRef[], options?: Omit<CombineOptions, 'exclude'>): AsyncGenerator<number>;
+    /** {@link intersect}, one `Uint32Array` per chunk — see {@link iterateBatches}. */
+    intersectBatches(segs: readonly SegmentRef[], options?: CombineOptions): AsyncGenerator<Uint32Array>;
+    /** {@link union}, one `Uint32Array` per chunk — see {@link iterateBatches}. */
+    unionBatches(segs: readonly SegmentRef[], options?: CombineOptions): AsyncGenerator<Uint32Array>;
+    /** {@link andNot}, one `Uint32Array` per chunk — see {@link iterateBatches}. */
+    andNotBatches(seg: SegmentRef, excludes: readonly SegmentRef[], options?: Omit<CombineOptions, 'exclude'>): AsyncGenerator<Uint32Array>;
+    /**
+     * Everything a combine decides before it reads a chunk: validation, each operand's keys and generation, the
+     * absent-operand refusal, key alignment, the budget and the metric. `null` when the range is empty. The per-id
+     * read and the batch read share it, so the two cannot disagree about what is read or charged.
+     */
+    private combinePlan;
     private combine;
+    private combineBatches;
     /**
      * The AND (or OR) of one chunk key across all operands, minus the excludes that hold it, or `null` if empty.
      * Operand chunks are fetched **in parallel** (the spec's parallel byte-range reads).
@@ -255,7 +288,15 @@ export declare class SegmentEngine {
      * `gen === undefined` ⇒ the source can't report a generation (it only ever serves one) ⇒ the key stays
      * generation-free. Superseded-generation entries age out under the LRU ceiling — no active purge.
      *
-     * The returned instance is **shared** (it may be the cached one): callers read it or clone it, never mutate it.
+     * Concurrent misses of one key share **one** request, with or without a cache: the first caller starts the read,
+     * later ones await it, and the entry goes when the read settles, so a failure is every waiting caller's and the
+     * next call reads again. The key carries the version exactly as the cache's does, so two versions of one chunk
+     * are two reads, never one.
+     *
+     * The returned instance is **shared** (it may be the cached one, and every caller that joined a read gets the same
+     * one): callers read it or clone it, never mutate it.
      */
     private storageChunk;
+    /** One request for one chunk, decoded, range-checked and cached. */
+    private fetchChunk;
 }

package/dist/core/erase-id.d.ts

@@ -6,8 +6,8 @@
  * is what every other write in this library is: a new generation. The current generation is streamed chunk by
  * chunk through the ascending writer — every chunk decoded, range-checked and re-encoded, the one holding the id
  * with that bit cleared — then published fenced on the generation it streamed, and that generation is collected immediately
- * (`keep: 0`), so the bit is **physically gone from the bucket when this returns**. Constant memory: one chunk in
- * flight, never the whole segment.
+ * (`keep: 0`), so the bit is **physically gone from the bucket when this returns**. Bounded memory: a window of 32
+ * chunk reads ahead of the writer, never the whole segment.
  *
  * The rewrite is the same generation without one id, so it keeps everything else: the new object carries the source's
  * metadata as it is, and the row's summary of it, built from what was written, counts one id fewer and holds the same
@@ -103,7 +103,7 @@ export interface EraseIdDeps {
      * source is used if it has one, and otherwise each wait is its bound.
      */
     readonly rng?: Rng;
-    /** Per-chunk decode ceiling (invariant 5); defaults to 1 MiB. */
+    /** Per-chunk decode ceiling (invariant 5); defaults to 1 MiB. The `.crbm` reader refuses an entry above its own `maxPayloadBytes` (1 MiB, plus 28 bytes when encrypted) at open, so raise that on the chunk source too. */
     readonly maxBitmapBytes?: number;
     /**
      * The store's read retry, for the reads the rewrite makes along the way: the generation it rewrites and each of its

package/dist/core/metrics.d.ts

@@ -26,7 +26,7 @@ export type MetricEvent = {
     readonly kind: 'storage.get';
     readonly namespace?: string;
     readonly segment: string;
-    /** Bytes returned (0 if the chunk was absent — a GET still happened). */
+    /** One event per request, however many callers were waiting on it. Bytes returned (0 if the chunk was absent — a GET still happened). */
     readonly bytes: number;
     /**
      * Elapsed wall time of the read — includes any transient-retry backoff on the storage call. From the
@@ -34,6 +34,12 @@ export type MetricEvent = {
      */
     readonly ms: number;
 } | {
+    /**
+     * One decoded-chunk cache lookup (emitted only when a cache is configured). `hit: false` means the lookup found
+     * no cached chunk. A caller that missed while another caller's read of the same chunk was open waits on that
+     * read and adds no `storage.get`, so the misses can outnumber the `storage.get` events; the `storage.get`
+     * count is the number of requests.
+     */
     readonly kind: 'cache';
     readonly hit: boolean;
 } | {