@cloudbitmaps/core 0.9.0 → 0.11.0

package/dist/core/metrics.d.ts

@@ -0,0 +1,136 @@
+/**
+ * Metrics seam — an injected sink the engine pushes typed events to, exactly like the `Clock`/driver seams
+ * keep `core/` pure and storage-agnostic.
+ *
+ * Design: the library emits **neutral domain events** and stays vendor-agnostic — you (or a ~12-line
+ * adapter) map them to OpenTelemetry / Datadog / a log line. The default is a **no-op**, and emission is
+ * skipped entirely when no sink is wired (near-zero overhead, no telemetry dependency). Events carry **raw
+ * observations** (bytes, counts, ms); turning those into billable units / dollars is the cost model's job,
+ * which keeps rates pluggable.
+ *
+ * Two caveats for sink authors: `onEvent` runs **synchronously on the I/O path**, so keep it cheap and
+ * non-blocking (offload batching/network to your own async queue); and `segment`/`namespace` are
+ * caller-controlled strings that may be PII and are **unbounded-cardinality** — do not map them to
+ * per-series metric labels unless they're known low-cardinality and PII-free. See the observability
+ * guide.
+ */
+/** The segment operations that emit an `op` latency event (timed with the injected clock, at the facade). */
+export type MetricOpName = 'has' | 'count' | 'intersectInto' | 'unionInto' | 'andNotInto';
+/**
+ * One observability event. A discriminated union on `kind` — new variants can be added over time without
+ * breaking consumers (they switch on the kinds they care about). `namespace` is the segment's namespace
+ * (absent ⇒ the default namespace). Segment names are identifiers, never bitmap contents (threat model:
+ * never log PII / bitmap bits).
+ */
+export type MetricEvent = {
+    readonly kind: 'storage.get';
+    readonly namespace?: string;
+    readonly segment: string;
+    /** Bytes returned (0 if the chunk was absent — a GET still happened). */
+    readonly bytes: number;
+    /**
+     * Elapsed wall time of the read — includes any transient-retry backoff on the storage call. From the
+     * injected clock, clamped to ≥ 0 (0 under the no-wait test clock).
+     */
+    readonly ms: number;
+} | {
+    readonly kind: 'cache';
+    readonly hit: boolean;
+} | {
+    readonly kind: 'retry';
+    /** Infrastructure-fault backoff (throttling, 5xx, a dropped connection) — the one kind of retry the store does. */
+    readonly reason: 'transient';
+    /** 1-based number of the attempt about to be retried. */
+    readonly attempt: number;
+    readonly delayMs: number;
+} | {
+    readonly kind: 'intersect';
+    /**
+     * Which chunk-aligned combine this was. **Optional** — absent means `'intersect'`.
+     *
+     * For `'union'` over its *include* operands alone, `skippedChunks` is 0 by construction — union reads
+     * every chunk of every operand, so there is nothing to prune. It can still be non-zero when a union
+     * carries `exclude` operands, because a suppression list may hold keys no include does and those are
+     * legitimately never fetched. So: 0 is the expected shape for a plain union, not a guarantee.
+     */
+    readonly op?: 'intersect' | 'union' | 'andNot';
+    /** Operand count, including any `exclude` (suppression) operands. */
+    readonly operands: number;
+    /**
+     * Distinct chunk **keys** selected to fetch — for `'intersect'`, those present in every operand.
+     *
+     * **Keys, not requests.** The actual storage GETs are roughly `fetchedChunks × operands`, plus one per
+     * `exclude` that holds a given key, which is why this number is smaller than what the per-op budget
+     * charges for the same call. The two are different units by design; if you are reconciling a bill, the
+     * budget's accounting is the one that models requests.
+     */
+    readonly fetchedChunks: number;
+    /**
+     * **Distinct** chunk-keys pruned — never fetched (the chunk-skipping saving). Counts distinct keys,
+     * not per-operand GETs, so it under-states the true GET saving when 3+ operands partially overlap.
+     *
+     * On a read bounded by `after` / `through`, this and `fetchedChunks` count only the keys inside the range:
+     * the keys outside it are never considered, so they are not counted as pruned either. Both count the keys the
+     * call would fetch if read to the end, so a page that stops early reports every key up to `through`, or to the
+     * end of the segment when `through` is left out.
+     */
+    readonly skippedChunks: number;
+} | {
+    readonly kind: 'op';
+    readonly name: MetricOpName;
+    readonly ms: number;
+};
+/**
+ * The sink you plug in. One method, synchronous, fire-and-forget. It must never throw back into the
+ * caller — `CloudRoaring` wraps a user sink with {@link safeMetrics} so a buggy sink can't break I/O.
+ */
+export interface IMetricsSink {
+    onEvent(event: MetricEvent): void;
+}
+/** The default sink: discards everything, zero overhead. Used whenever no sink is wired. */
+export declare const NOOP_METRICS: IMetricsSink;
+/**
+ * Wrap a sink so a throwing/buggy `onEvent` can never break the data path — observability is strictly
+ * best-effort. `CloudRoaring` applies this to the user-supplied sink at construction.
+ */
+export declare function safeMetrics(sink: IMetricsSink): IMetricsSink;
+/** Accumulated totals — the shape returned by {@link CountingMetricsSink.snapshot}. */
+export interface MetricsSnapshot {
+    readonly storage: {
+        readonly gets: number;
+        readonly bytes: number;
+        readonly totalMs: number;
+    };
+    readonly cache: {
+        readonly hits: number;
+        readonly misses: number;
+    };
+    readonly retries: {
+        readonly transient: number;
+    };
+    readonly intersect: {
+        readonly calls: number;
+        readonly fetchedChunks: number;
+        readonly skippedChunks: number;
+    };
+    readonly ops: Readonly<Record<MetricOpName, {
+        readonly count: number;
+        readonly totalMs: number;
+    }>>;
+}
+/**
+ * A ready-made sink that tallies events into a {@link MetricsSnapshot} — handy for tests, quick scripts,
+ * and as the grounded-request-cost source behind `costReport()`. `snapshot()` returns an independent copy;
+ * `reset()` zeroes the counters.
+ */
+export declare class CountingMetricsSink implements IMetricsSink {
+    private storage;
+    private cache;
+    private retries;
+    private intersect;
+    private ops;
+    private static zeroOps;
+    onEvent(event: MetricEvent): void;
+    snapshot(): MetricsSnapshot;
+    reset(): void;
+}

package/dist/core/name-codec.d.ts

@@ -0,0 +1,69 @@
+/**
+ * Name → physical-key encoding. The library validates *length*, and encodes everything else.
+ *
+ * Lives in `core/` rather than beside the drivers because it is pure string logic — no I/O, no SDK, no
+ * `node:` builtin — and because `validate.ts` needs it to measure a name against the key budget. Core may
+ * not import a driver (invariant 7), and a driver may import core, so this is the only layer both can see.
+ *
+ * A name is **any non-empty string**, and each physical boundary escapes what *it* cannot take literally. An
+ * allowlist of characters would reject names for the storage layer's convenience, and the storage layer's
+ * convenience is the library's job, not the caller's. It would also let through the names that are actually
+ * dangerous: `con`, `nul` and `com1` are Windows device names made only of letters and digits, which pass any
+ * allowlist and fail only on a user's machine. A grammar can be both too narrow and too permissive at once,
+ * which is the argument against having one.
+ *
+ * Percent-encoding, because it is the one escape everybody already reads. `%` escapes itself as `%25` and is
+ * encoded **first**, which is what makes the transform injective: every `%` in an encoded string starts an
+ * escape, so decoding is unambiguous and two distinct names can never collide on one key.
+ *
+ * There are two alphabets, because the boundaries differ:
+ *
+ * - **Object keys** (S3, GCS, Azure Blob) take almost anything. They need `/` escaped so a name cannot
+ *   invent hierarchy or break a key parser that splits on it, and control characters because they are not
+ *   legal in the XML an S3 LIST returns. Every other character outside `[A-Za-z0-9._:-]` is escaped as well.
+ * - **Filesystem paths** need all of that plus `:` (an NTFS alternate-data-stream separator, where a write can
+ *   *succeed* while `readdir` never lists the result), plus three hazards that are about the component as a
+ *   whole rather than its characters — see {@link encodeNameForPath}.
+ *
+ * **The key alphabet is fixed for good.** Adding a character to it would change what `encodeNameForKey` emits
+ * for every name containing that character, silently moving those segments to keys nothing looks up. A test pins
+ * it, one printable ASCII character at a time.
+ *
+ * **A plain name encodes to itself on the OBJECT-KEY alphabet** — one made only of `[A-Za-z0-9._:-]` that
+ * starts with a letter or digit — so S3, GCS and Azure keys read as the names that made them. That is the
+ * property to preserve if this file is ever edited, and a property test asserts it over every plain name.
+ *
+ * The **path** alphabet escapes more. Besides `:`, two classes of plain name are escaped on a path: a Windows
+ * device-name stem (`con`, `nul`, `com1`, `con.backup`) and a trailing dot (`a.`), since both silently alias
+ * or fail on Windows.
+ */
+/**
+ * Encode a name for use inside an **object key** (S3, GCS, Azure Blob).
+ *
+ * Leaves `[A-Za-z0-9._:-]` literal, except a leading `_` and a name that is only `.` or `..`, so a plain name
+ * reads as itself, and escapes everything else — including `/` (which would invent hierarchy and break a parser
+ * that splits on it) and control characters (not legal in S3's XML responses).
+ */
+export declare function encodeNameForKey(name: string): string;
+/** Inverse of {@link encodeNameForKey}. */
+export declare function decodeNameFromKey(encoded: string): string;
+/**
+ * Encode a name for use as a **filesystem path component**.
+ *
+ * Everything {@link encodeNameForKey} escapes, plus `:`, plus three hazards that are properties of the whole
+ * component rather than of any character in it. All three are silent corruption rather than errors, which is
+ * why they are handled here instead of being left to the OS:
+ *
+ * 1. **`.` and `..`** are traversal. `join(root, '..')` leaves the storage root.
+ * 2. **Windows reserved device names** — `CON`, `PRN`, `AUX`, `NUL`, `COM1`–`COM9`, `LPT1`–`LPT9`, case
+ *    insensitive and *with or without an extension*, so `con.0.crbm` is reserved too. Opening one addresses a
+ *    device rather than a file, so `store.segment('con')` would break only on a user's Windows box.
+ * 3. **A trailing `.` or space**, which Windows silently strips — so `a.` and `a` would become the same
+ *    directory, quietly merging two segments.
+ *
+ * Each is fixed by escaping the one character that defuses it, which keeps the encoding injective: a name that
+ * needed defusing produces a `%XX` no other name produces.
+ */
+export declare function encodeNameForPath(name: string): string;
+/** Inverse of {@link encodeNameForPath}. */
+export declare function decodeNameFromPath(encoded: string): string;

package/dist/core/pinned-storage-source.d.ts

@@ -0,0 +1,90 @@
+/**
+ * A {@link StorageChunkSource} view that holds **one** segment at **one** generation, and passes everything else
+ * through to the live source.
+ *
+ * A pin exists so a long job describes a single instant. An ordinary handle re-resolves on `cache.genTtlMs`, so a
+ * publish part-way through an export, a send or a reconciliation means the second half of the job describes a
+ * different instant than the first — and nothing in the result says so.
+ *
+ * **A pin is a generation number and the object it named, not a captured reader.** The reader behind it can live
+ * in the shared bounded LRU and be evicted freely: re-opening at the same number reopens the same object, or, if
+ * the name was purged and loaded again since, finds another one, which the pin's fingerprint refuses. Holding the
+ * reader instead put every live pin outside the library's memory ceiling.
+ *
+ * **Other segments are not pinned, and that is the contract, not an omission.** `snap.intersect([other])` reads
+ * `snap` at its pinned generation and `other` at whatever is current. Pinning the whole query means pinning
+ * each segment. The alternative shapes are both worse: refusing the combine outright makes a pin useless to the
+ * export and reconciliation jobs it exists for, and silently pinning the operands would quietly change what
+ * `other` means to a caller who did not ask.
+ *
+ * What this must never do is the inverse — read a pinned handle's segment **live** because the call was made
+ * on some other handle. That is why the routing is by segment identity here rather than by which object the
+ * caller happened to start from.
+ */
+import type { ChunkRef, StorageChunkSource, SegmentRef, SegmentSize } from './ports.js';
+import type { CrbmStorageChunkSource } from './crbm-storage-source.js';
+/**
+ * What a pin holds for one segment: the generation, the version identifying those exact bytes, and the pinned
+ * object's fingerprint, so a read can tell that the generation it opens is still that object.
+ *
+ * `fingerprint` is optional so that a pin built by hand, before there was one, still compiles. Nothing checks the
+ * object such a pin reads, so it cannot tell a name purged and loaded again from the segment it pinned.
+ * `seg.pin()` always records it.
+ */
+export interface PinnedAt {
+    readonly generation: number | null;
+    readonly version: string | null;
+    readonly fingerprint?: string | null;
+}
+export declare class PinnedStorageChunkSource implements StorageChunkSource {
+    private readonly inner;
+    /**
+     * The pinned segments, keyed by {@link segmentKey}. A **set** rather than one entry because a combine may
+     * involve several pinned handles, and each must be read at its own pin — including when the call was made
+     * on a different handle. Reading a pinned handle live because the caller started elsewhere is the silent
+     * wrong answer this exists to prevent.
+     */
+    private readonly pins;
+    constructor(inner: CrbmStorageChunkSource, 
+    /**
+     * The pinned segments, keyed by {@link segmentKey}. A **set** rather than one entry because a combine may
+     * involve several pinned handles, and each must be read at its own pin — including when the call was made
+     * on a different handle. Reading a pinned handle live because the caller started elsewhere is the silent
+     * wrong answer this exists to prevent.
+     */
+    pins: ReadonlyMap<string, PinnedAt>);
+    /** The unpinned segments refresh as the store's own source does; a pinned one never does. */
+    get pointerRefreshMs(): number;
+    /**
+     * The pin for this segment, or `undefined` if it is not pinned here.
+     *
+     * A pin whose `generation` is `null` is a segment that had none when it was pinned: that handle reads empty
+     * for its lifetime, which is exactly what an unpinned read of it does. That is **not** the same as "this
+     * source cannot pin" — a source that genuinely cannot is a capability gap and fails fast at `pin()`.
+     */
+    private pinFor;
+    getChunk(ref: ChunkRef): Promise<Uint8Array | null>;
+    listChunkKeys(ref: SegmentRef): Promise<number[]>;
+    sizeOf(ref: SegmentRef): Promise<SegmentSize | null>;
+    cardinalities(ref: SegmentRef): Promise<ReadonlyMap<number, number> | null>;
+    currentGeneration(ref: SegmentRef): Promise<number | null>;
+    /**
+     * A pinned segment reports the version captured when it was pinned, marked as a pin's, so its decoded chunks
+     * are cached under keys that no live read writes. A live read keys its fetches by the version it resolved when
+     * it began, but is served whatever generation the live source holds when each fetch lands — after a publish
+     * and a lapsed `cache.genTtlMs`, a reader-cache eviction, a sweep that heals the read forward, or an
+     * invalidation, a different one — so an entry under a live version can hold another generation's chunk.
+     * Invariant 3 lets that live call see it, and later live reads resolve the current version and never look the
+     * entry up; a pin sharing the key would be handed it, and return a read that mixed two generations. A pinned
+     * read fetches exactly its own generation, so the entries it fills are always that generation's.
+     *
+     * The pin still shares the store's chunk cache, and its memory ceiling; what it gives up is a hit on a chunk a
+     * live read of the same version cached, which costs a pin one GET per such chunk. Live reads make no call for it,
+     * though a pin's entries share the cache's bound with theirs, so under a small `cache.maxChunks` each can evict
+     * the other. The version, not the bare generation, is still what the key carries: sharing on a generation-only
+     * key was how a pinned read could resurrect an id that `eraseIdFromSegment` had reported physically gone.
+     */
+    currentVersion(ref: SegmentRef): Promise<string | null>;
+    exists(ref: SegmentRef): Promise<boolean>;
+    invalidate(ref: SegmentRef): void;
+}