@cloudbitmaps/core 0.9.0 → 0.11.0

package/dist/drivers/localfs/fs-util.d.ts

@@ -0,0 +1,21 @@
+/** True when `err` is a Node system error with the given `code` (e.g. `'ENOENT'`, `'EEXIST'`). */
+export declare const isCode: (err: unknown, code: string) => boolean;
+export declare const isFsTransient: (err: unknown) => boolean;
+/**
+ * Reclassify a transient filesystem fault as a retryable {@link TransientError}, so the store's read retry can
+ * ride it out and a write's caller can tell it from a deterministic failure; everything else — including the
+ * driver's own typed errors — propagates unchanged.
+ */
+export declare const mapFsError: (err: unknown) => unknown;
+/**
+ * `O_NOFOLLOW` where the platform supports it (absent on Windows → 0). OR it into an `open` flag set so a
+ * symlink **at the final path component** is refused rather than followed outside the storage root.
+ * Containment of symlinked *directory* components assumes the root itself is not attacker-writable.
+ */
+export declare const O_NOFOLLOW: number;
+/**
+ * Best-effort parent-directory fsync, so a just-published `rename`/`link` survives a crash. Without it the
+ * directory entry can still be in the page cache when power goes: the generation's bytes are durable and
+ * the name pointing at them is not, which is the one way a published generation can come back missing.
+ */
+export declare function fsyncDir(dir: string): Promise<void>;

package/dist/drivers/localfs/paths.d.ts

@@ -0,0 +1,33 @@
+import type { GenKey, SegmentRef } from '../../core/ports.js';
+/** Directory holding all of a namespace's segment objects. */
+export declare function segmentsDir(root: string, ref: SegmentRef): string;
+/** Absolute path of one `.crbm` generation object. */
+export declare function storageObjectPath(root: string, key: GenKey): string;
+/** Filename pattern for a segment's generations: `<segment>.<gen>.crbm`. */
+export declare function storageObjectFilename(segment: string, generation: number): string;
+/** Parse a generation number out of a `<segment>.<gen>.crbm` filename, or `null` if it doesn't match. */
+export declare function parseGeneration(segment: string, filename: string): number | null;
+/** Directory holding a namespace's registry rows (one file per segment). */
+export declare function registryDir(root: string, namespace: string | undefined): string;
+/** Absolute path of one segment's registry row file: `<ns>/registry/<segment>.reg`. */
+export declare function registryRowPath(root: string, ref: SegmentRef): string;
+/**
+ * Parse a segment name out of a `<segment>.reg` filename, or `null` if it doesn't match — including any
+ * file whose stem isn't a valid segment name. A strict parser means a stray or planted `.reg` file makes
+ * `list()` *skip* it, never abort the whole enumeration on a boundary throw.
+ */
+export declare function parseRegistryRow(filename: string): string | null;
+/**
+ * Parse a namespace out of a directory name under the storage root, or `null` if it was not written by us.
+ *
+ * The inverse of the `namespacePathPart(ns)` that {@link segmentsDir} and {@link registryDir}
+ * write. A fleet-wide scan enumerates these directories, so getting it wrong does not fail one segment — it
+ * aborts the whole enumeration, taking the consistency check, the retention sweep and subject erasure with it.
+ *
+ * Like {@link parseRegistryRow} this requires the encoding to **round-trip**, so a planted literal `tenant:acme`
+ * directory sitting next to the driver's own `tenant%3Aacme` is skipped rather than reported as a namespace
+ * whose rows then resolve to the other directory. `_default` maps back to "no namespace".
+ */
+export declare function parseNamespaceDir(entry: string): {
+    namespace: string | undefined;
+} | null;

package/dist/drivers/localfs/registry.d.ts

@@ -0,0 +1,28 @@
+import type { IRegistryDriver, NewRegistryRecord, RegCaps, RegistryPatch, RegistryRecord, SegmentRef, Token } from '../../core/ports.js';
+/** Entries in the process-wide row-lock map; the map is empty whenever no registry operation is in flight. */
+export declare function localFsRowLockCount(): number;
+export interface LocalFsRegistryDriverOptions {
+    /** Injected clock for `createdAt`/`updatedAt`; defaults to `Date.now`. */
+    readonly now?: () => number;
+}
+export declare class LocalFsRegistryDriver implements IRegistryDriver {
+    private readonly root;
+    private readonly now;
+    constructor(root: string, options?: LocalFsRegistryDriverOptions);
+    capabilities(): RegCaps;
+    get(ref: SegmentRef): Promise<RegistryRecord | null>;
+    create(ref: SegmentRef, record: NewRegistryRecord): Promise<{
+        token: Token;
+    }>;
+    compareAndSwap(ref: SegmentRef, expected: Token, patch: RegistryPatch): Promise<{
+        token: Token;
+    }>;
+    list(namespace?: string): AsyncIterable<RegistryRecord>;
+    delete(ref: SegmentRef, expected?: Token): Promise<void>;
+    /** Namespaces to scan: just the one requested, or every namespace dir under the root. */
+    private namespaceDirs;
+    private readRow;
+    private writeRow;
+    /** Serialize callbacks for a row so read-modify-write is atomic across every instance in the process. */
+    private withRowLock;
+}

package/dist/drivers/localfs/storage.d.ts

@@ -0,0 +1,19 @@
+import type { BlobSink } from '../../core/blob.js';
+import type { StorageCaps, GenKey, IStorageDriver, SegmentRef } from '../../core/ports.js';
+export declare class LocalFsStorageDriver implements IStorageDriver {
+    private readonly root;
+    constructor(root: string);
+    capabilities(): StorageCaps;
+    putImmutable(key: GenKey, write: (sink: BlobSink) => Promise<void>): Promise<{
+        size: number;
+        sha256: string;
+    }>;
+    getRange(key: GenKey, offset: number, length: number): Promise<Uint8Array>;
+    getTail(key: GenKey, maxBytes: number): Promise<{
+        bytes: Uint8Array;
+        size: number;
+    }>;
+    delete(key: GenKey): Promise<void>;
+    list(ref: SegmentRef): AsyncIterable<GenKey>;
+    private openRead;
+}

package/dist/drivers/memory.d.ts

@@ -0,0 +1,49 @@
+import type { BlobSink } from '../core/blob.js';
+import type { StorageCaps, GenKey, IStorageDriver, IRegistryDriver, NewRegistryRecord, RegCaps, RegistryPatch, RegistryRecord, SegmentRef, Token } from '../core/ports.js';
+export interface MemoryRegistryDriverOptions {
+    /** Injected clock for `createdAt`/`updatedAt`; defaults to `Date.now` (drivers may use ambient time). */
+    readonly now?: () => number;
+}
+/**
+ * In-memory {@link IRegistryDriver} — one record per segment under OCC. The token is a single global,
+ * monotonic, never-reused counter (so a record recreated after `delete` always gets a strictly-greater
+ * token → ABA-safe even though `delete` removes the row physically).
+ */
+export declare class MemoryRegistryDriver implements IRegistryDriver {
+    private readonly rows;
+    private readonly now;
+    private seq;
+    constructor(options?: MemoryRegistryDriverOptions);
+    private nextToken;
+    capabilities(): RegCaps;
+    get(ref: SegmentRef): Promise<RegistryRecord | null>;
+    create(ref: SegmentRef, record: NewRegistryRecord): Promise<{
+        token: Token;
+    }>;
+    compareAndSwap(ref: SegmentRef, expected: Token, patch: RegistryPatch): Promise<{
+        token: Token;
+    }>;
+    list(namespace?: string): AsyncIterable<RegistryRecord>;
+    delete(ref: SegmentRef, expected?: Token): Promise<void>;
+}
+/**
+ * In-memory {@link IStorageDriver} — write-once immutable generation objects as opaque bytes. A "dumb byte
+ * mover" (understands neither roaring nor `.crbm`), so it's a faithful storage backend for tests and a zero-setup
+ * local storage tier. Mirrors the LocalFs/S3 contract (write-once, range/tail reads).
+ */
+export declare class MemoryStorageDriver implements IStorageDriver {
+    private readonly objects;
+    capabilities(): StorageCaps;
+    putImmutable(key: GenKey, write: (sink: BlobSink) => Promise<void>): Promise<{
+        size: number;
+        sha256: string;
+    }>;
+    getRange(key: GenKey, offset: number, length: number): Promise<Uint8Array>;
+    getTail(key: GenKey, maxBytes: number): Promise<{
+        bytes: Uint8Array;
+        size: number;
+    }>;
+    delete(key: GenKey): Promise<void>;
+    list(ref: SegmentRef): AsyncIterable<GenKey>;
+    private require;
+}

package/dist/drivers/retry/retrying-chunk-source.d.ts

@@ -0,0 +1,59 @@
+/**
+ * The store's read retry.
+ *
+ * {@link RetryingStorageChunkSource} wraps a {@link StorageChunkSource} so every read it serves retries
+ * **transient** faults with bounded, jittered backoff, through the one shared `core/retry` primitive, so every
+ * backend (S3, GCS, Azure Blob, LocalFS, …) inherits the same policy instead of each rolling its own. `CloudRoaring`
+ * wraps the source it reads segment data through in one — a `StorageChunkSource` passed as `storage` included, so
+ * wrapping that one first multiplies each read's attempts — and a pinned handle's source too, unless it is built
+ * with `retry: false`.
+ * Pure composition over the port (no SDK, no I/O of its own); the driver under the source is responsible for
+ * *classifying* its transient faults (raising {@link TransientError}), and this wrapper decides *whether and when*
+ * to retry.
+ *
+ * **Reads only, on purpose.** A read changes nothing, so running it again is safe. A write is not: a conditional put
+ * or compare-and-swap that lands and then loses its response would, replayed, find its own write already there and
+ * report it as a conflict. So nothing here wraps a write, and the store's writes and admin calls report a transient
+ * fault to their caller, who re-runs the call or checks what landed.
+ *
+ * What is **not** retried: every deterministic error (`ValidationError`/`IntegrityError`/`NotFoundError`/…).
+ * Default classifier: {@link isTransientError}.
+ */
+import type { Clock, Rng } from '../../core/determinism.js';
+import type { RetryPolicy } from '../../core/retry.js';
+import type { ChunkRef, StorageChunkSource, SegmentRef, SegmentSize } from '../../core/ports.js';
+export interface RetryingOptions {
+    readonly clock: Clock;
+    readonly rng: Rng;
+    /** Defaults to {@link DEFAULT_RETRY_POLICY}. */
+    readonly policy?: RetryPolicy;
+    /** Override which errors are retryable. Default: {@link isTransientError} (any `TransientError`). */
+    readonly isRetryable?: (err: unknown) => boolean;
+    /** Observability hook fired before each backoff wait. */
+    readonly onRetry?: (info: {
+        attempt: number;
+        delayMs: number;
+        err: unknown;
+    }) => void;
+}
+/** Wrap a storage chunk source so its reads retry transient faults. */
+export declare class RetryingStorageChunkSource implements StorageChunkSource {
+    private readonly inner;
+    private readonly policy;
+    private readonly deps;
+    /** Present only when the inner source supports it — so capability detection stays honest. */
+    readonly sizeOf?: (ref: SegmentRef) => Promise<SegmentSize | null>;
+    readonly cardinalities?: (ref: SegmentRef) => Promise<ReadonlyMap<number, number> | null>;
+    readonly currentGeneration?: (ref: SegmentRef) => Promise<number | null>;
+    readonly invalidate?: (ref: SegmentRef) => void;
+    readonly exists?: (ref: SegmentRef) => Promise<boolean>;
+    readonly currentVersion?: (ref: SegmentRef) => Promise<string | null>;
+    /**
+     * Forwarded, not retried: a property, with no request behind it. Without it the grounded cost report priced a
+     * retrying store's refresh at the default TTL whatever the store was configured with.
+     */
+    get pointerRefreshMs(): number | undefined;
+    constructor(inner: StorageChunkSource, opts: RetryingOptions);
+    getChunk(ref: ChunkRef): Promise<Uint8Array | null>;
+    listChunkKeys(ref: SegmentRef): Promise<number[]>;
+}

package/dist/export/index.d.ts

@@ -0,0 +1,93 @@
+import type { CodecInterface } from '../core/codec.js';
+import type { IRegistryDriver, SegmentRef } from '../core/ports.js';
+/** Output format for {@link runExport} / `CloudRoaring.exportSegments`. */
+export type ExportFormat = 'roaring' | 'ndjson';
+/** A writer for one segment's output — inject a filesystem file, an S3 upload, stdout, or a test buffer. */
+export interface ExportWriter {
+    /** Append bytes (called one or more times; for `ndjson`, batched line chunks; for `roaring`, once). */
+    write(bytes: Uint8Array): Promise<void> | void;
+    /** **Commit** the output — called once, only on success (all bytes written). Finalize/flush/rename here. */
+    close(): Promise<void> | void;
+    /**
+     * **Discard** a partial output — called instead of `close()` if a fault occurs mid-segment, so a truncated file
+     * never masquerades as complete (e.g. delete the temp file). Optional; sinks with nothing to undo can omit it.
+     */
+    abort?(): Promise<void> | void;
+}
+/** A destination the export writes each segment to. `ext` is `.roaring` or `.ndjson`. */
+export interface ExportSink {
+    open(ref: SegmentRef, ext: string): Promise<ExportWriter> | ExportWriter;
+}
+export interface ExportOptions {
+    /** Output format; defaults to `'roaring'`. */
+    readonly format?: ExportFormat;
+    /**
+     * Bitmap codec that builds the exported bitmap for the `'roaring'` format. Optional in the type; a **flavor**'s
+     * store passes its own (see `requireCodec`). Not needed for `'ndjson'`, which streams plain ids.
+     */
+    readonly codec?: CodecInterface;
+    /** Scope the export to one namespace; omit to export every registered segment. */
+    readonly namespace?: string;
+    /** `ndjson` flush threshold in ~bytes (ids are ASCII, so ≈ chars); defaults to 64 KiB. */
+    readonly ndjsonBatchBytes?: number;
+}
+/** Per-segment result recorded in the {@link ExportManifest}. */
+export interface ExportedSegment {
+    readonly segment: string;
+    readonly namespace?: string;
+    /** Ids written (the segment's effective cardinality at export time). */
+    readonly count: number;
+    /** Bytes written to the sink for this segment. */
+    readonly bytes: number;
+}
+/**
+ * A segment that could **not** be exported — recorded rather than aborting the whole run (fault isolation), so a
+ * single corrupt/undecryptable segment never blocks the rest of the dump.
+ */
+export interface ExportFailure {
+    readonly segment: string;
+    readonly namespace?: string;
+    /** Why it failed (an error message) — e.g. a corrupt storage object, or an encrypted segment with no keystore. */
+    readonly error: string;
+}
+/** What {@link runExport} produced — a self-describing summary the CLI persists as `manifest.json`. */
+export interface ExportManifest {
+    /** Schema version of this manifest shape (currently `1`) — insurance for evolving the format post-1.0. */
+    readonly version: number;
+    readonly format: ExportFormat;
+    readonly totalSegments: number;
+    readonly totalIds: number;
+    readonly segments: readonly ExportedSegment[];
+    /**
+     * Segments that could not be read (empty when everything succeeded). Its presence in the manifest is the point:
+     * "a manifest exists" ⇒ the run finished, **not** that every segment succeeded — always check this list.
+     */
+    readonly failed: readonly ExportFailure[];
+}
+/** The minimal read surface the export needs from a store — a `CloudRoaring` satisfies this structurally. */
+export interface SegmentReader {
+    segment(name: string, options?: {
+        namespace?: string;
+    }): {
+        iterate(): AsyncIterable<number>;
+    };
+}
+/**
+ * Export every registered segment's current generation through `sink`. Enumerates via `registry`, reads via
+ * `reader` (the store). Returns a manifest. `CloudRoaring.exportSegments` is the
+ * public entry — it calls this with its own store + registry (so enumeration and the read path provably share one
+ * registry).
+ *
+ * **Fault isolation:** a per-segment read/write fault aborts that segment's partial (so a truncated file never
+ * looks complete), records it in `failed`, and the run continues — one bad segment never blocks the rest. A
+ * `close()` fault is likewise recorded (never triggering `abort()` — they're mutually exclusive).
+ *
+ * Re-running overwrites the segments it re-exports but does **not** prune files for segments that have since
+ * disappeared — export to a fresh directory for a clean dump. For a *current* image, run against a
+ * freshly-constructed store: a store's storage source re-resolves each segment's generation on a short TTL, or, with
+ * no registry, with `cache.genTtlMs: 0`, or built without a clock, only when an eviction, a sweep or an invalidation
+ * makes it, so a long-lived store may export a view one load behind, or more. Each segment is read live, not pinned:
+ * a publish while a long segment exports can re-resolve it part-way, and its file then holds chunks of two
+ * generations, each whole and verified. For one generation per file, export from a quiet window.
+ */
+export declare function runExport(reader: SegmentReader, registry: IRegistryDriver, sink: ExportSink, options?: ExportOptions): Promise<ExportManifest>;