@soulcraft/cor 3.0.0-rc.1

package/dist/utils/columnStoreTypes.d.ts

@@ -0,0 +1,161 @@
+/**
+ * @module utils/columnStoreTypes
+ * @description Cor-local type definitions for the native column store bridge.
+ *
+ * These mirror `@soulcraft/brainy`'s internal `indexes/columnStore/types`, which
+ * is NOT a published export of brainy 7.24.0. Cor defines them locally so the
+ * native `ColumnStoreProvider` implementation compiles against published brainy
+ * while staying byte-compatible with brainy's `.cidx` segments and manifests.
+ *
+ * The `ValueType` enum values are load-bearing: they are written into segment
+ * headers and manifests and MUST match brainy's enum exactly (Number=0, Float=1,
+ * String=2, Boolean=3). Keep in lockstep if brainy ever adds a value type.
+ */
+import type { StorageAdapter } from '@soulcraft/brainy';
+import type { RoaringBitmap32 } from '../native/NativeRoaringBitmap32.js';
+/**
+ * @description Discriminator for a column's value encoding, identical to
+ * brainy's `ValueType`. Numbers are load-bearing (persisted in headers/manifests).
+ */
+export declare enum ValueType {
+    /** i64 little-endian — timestamps, integer counts. */
+    Number = 0,
+    /** f64 little-endian — prices, scores, ratings. */
+    Float = 1,
+    /** u32-length-prefixed UTF-8 — status, category, names. */
+    String = 2,
+    /** i64 (0 or 1) — flags, toggles. */
+    Boolean = 3
+}
+/**
+ * @description Metadata for a single persisted segment within a field manifest.
+ * Matches brainy's `SegmentMeta` shape so manifests are interchangeable.
+ */
+export interface SegmentMeta {
+    /** Segment id, unique within the field (monotonic counter). */
+    id: number;
+    /** LSM compaction level. 0 = freshest (direct from the tail buffer). */
+    level: number;
+    /** Number of entries in the segment (including tombstoned). */
+    count: number;
+    /** Minimum value in the sorted column (zone-map pruning). */
+    minValue: number | string;
+    /** Maximum value in the sorted column. */
+    maxValue: number | string;
+    /** Relative file name within the field directory (e.g. `L0-000001.cidx`). */
+    file: string;
+}
+/**
+ * @description Per-field manifest payload, persisted as JSON at
+ * `_column_index/{field}/MANIFEST.json`. Matches brainy's `ManifestData`.
+ */
+export interface ManifestData {
+    /** Format version, bumped on every save. */
+    version: number;
+    /** Field name this manifest covers. */
+    field: string;
+    /** Value type for all segments in this field. */
+    valueType: ValueType;
+    /** Whether this is a multi-value field (one entity → many entries). */
+    multiValue: boolean;
+    /** All known segments, ordered by level then id. */
+    segments: SegmentMeta[];
+    /** Monotonically increasing segment id counter. */
+    nextSegmentId: number;
+    /** Whether the initial build from existing entities has completed. */
+    buildComplete: boolean;
+}
+/**
+ * @description The slice of brainy's `EntityIdMapper` that callers of the column
+ * store actually use. Cor's metadata index builds one of these backed by the
+ * native id mapper. Only `getUuid` is exercised by brainy's `orderBy` path; the
+ * rest mirror the brainy mapper surface that other call sites touch.
+ */
+export interface EntityIdMapperLike {
+    /** UUID for an int id, or undefined if unknown. */
+    getUuid(intId: number): string | undefined;
+    /** Int id for a UUID without assigning, or undefined if unknown. */
+    getInt(uuid: string): number | undefined;
+    /** Int id for a UUID, assigning a new one if absent. */
+    getOrAssign(uuid: string): number;
+    /** Resolve an iterable of int ids to UUIDs (unknown ids skipped). */
+    intsIterableToUuids(ints: Iterable<number>): string[];
+    /**
+     * Optional: reconstruct the UUID↔int mapping from the restored canonical
+     * entities. Brainy 8.0 `restore()` feature-detects this (`typeof
+     * idMapper.rebuild === 'function'`) and awaits it BEFORE rebuilding the
+     * metadata/vector/graph indexes, so a native adjacency keyed on entity ints
+     * resolves verb endpoints against the snapshot's assignments rather than
+     * stale/missing ints. Absent on the JS mapper (whose ints are re-derived by
+     * the metadata rebuild itself).
+     */
+    rebuild?(): Promise<void>;
+}
+/**
+ * @description The provider contract cor implements and registers with brainy
+ * as the column store. Structurally identical to brainy's `ColumnStoreProvider`;
+ * cor's `RoaringBitmap32` is binary-compatible with brainy's, so the bitmap
+ * return/argument types interoperate at runtime.
+ */
+export interface ColumnStoreProvider {
+    /**
+     * Initialize: discover fields, load manifests + segments, re-apply tombstones.
+     * @param storage - Storage adapter for segment/manifest I/O.
+     * @param idMapper - Shared entity id mapper for UUID ↔ int conversion.
+     */
+    init(storage: StorageAdapter, idMapper: EntityIdMapperLike): Promise<void>;
+    /**
+     * Index an entity's field values (synchronous; buffered until flush).
+     * @param entityIntId - The entity's u32 int id.
+     * @param fields - Map of field name → value (array = multi-value).
+     */
+    addEntity(entityIntId: number, fields: Record<string, unknown>): void;
+    /**
+     * Tombstone an entity across all indexed fields (synchronous).
+     * @param entityIntId - The entity's u32 int id.
+     */
+    removeEntity(entityIntId: number): void;
+    /**
+     * Point filter: entity int ids where `field` equals `value`.
+     * @param field - Field to filter.
+     * @param value - Exact value to match.
+     */
+    filter(field: string, value: unknown): Promise<RoaringBitmap32>;
+    /**
+     * Range filter: entity int ids where `field` is within `[min, max]`.
+     * @param field - Field to filter.
+     * @param min - Lower bound (undefined = open).
+     * @param max - Upper bound (undefined = open).
+     */
+    rangeQuery(field: string, min?: unknown, max?: unknown): Promise<RoaringBitmap32>;
+    /**
+     * Sort top-K entity int ids by `field` in `order`.
+     * @param field - Field to sort by.
+     * @param order - 'asc' or 'desc'.
+     * @param k - Maximum results.
+     */
+    sortTopK(field: string, order: 'asc' | 'desc', k: number): Promise<number[]>;
+    /**
+     * Filtered sort top-K: `sortTopK` restricted to `filterBitmap`.
+     * @param filterBitmap - Candidate entity int ids.
+     * @param field - Field to sort by.
+     * @param order - 'asc' or 'desc'.
+     * @param k - Maximum results.
+     */
+    filteredSortTopK(filterBitmap: RoaringBitmap32, field: string, order: 'asc' | 'desc', k: number): Promise<number[]>;
+    /**
+     * All distinct values for a field, as ascending strings.
+     * @param field - Field name.
+     */
+    getFilterValues(field: string): Promise<string[]>;
+    /**
+     * Whether a field has any indexed data.
+     * @param field - Field name.
+     */
+    hasField(field: string): boolean;
+    /** Flush all tail buffers to L0 segments and save manifests. */
+    flush(): Promise<void>;
+    /** Flush and release all resources. */
+    close(): Promise<void>;
+}
+//# sourceMappingURL=columnStoreTypes.d.ts.map

package/dist/utils/columnStoreTypes.js

@@ -0,0 +1,29 @@
+/**
+ * @module utils/columnStoreTypes
+ * @description Cor-local type definitions for the native column store bridge.
+ *
+ * These mirror `@soulcraft/brainy`'s internal `indexes/columnStore/types`, which
+ * is NOT a published export of brainy 7.24.0. Cor defines them locally so the
+ * native `ColumnStoreProvider` implementation compiles against published brainy
+ * while staying byte-compatible with brainy's `.cidx` segments and manifests.
+ *
+ * The `ValueType` enum values are load-bearing: they are written into segment
+ * headers and manifests and MUST match brainy's enum exactly (Number=0, Float=1,
+ * String=2, Boolean=3). Keep in lockstep if brainy ever adds a value type.
+ */
+/**
+ * @description Discriminator for a column's value encoding, identical to
+ * brainy's `ValueType`. Numbers are load-bearing (persisted in headers/manifests).
+ */
+export var ValueType;
+(function (ValueType) {
+    /** i64 little-endian — timestamps, integer counts. */
+    ValueType[ValueType["Number"] = 0] = "Number";
+    /** f64 little-endian — prices, scores, ratings. */
+    ValueType[ValueType["Float"] = 1] = "Float";
+    /** u32-length-prefixed UTF-8 — status, category, names. */
+    ValueType[ValueType["String"] = 2] = "String";
+    /** i64 (0 or 1) — flags, toggles. */
+    ValueType[ValueType["Boolean"] = 3] = "Boolean";
+})(ValueType || (ValueType = {}));
+//# sourceMappingURL=columnStoreTypes.js.map

package/dist/utils/nativeBinaryEntityIdMapper.d.ts

@@ -0,0 +1,211 @@
+/**
+ * @module utils/nativeBinaryEntityIdMapper
+ * @description TypeScript wrapper around cor's native binary
+ * `BinaryIdMapper`. Implements brainy's `EntityIdMapperProvider` so the
+ * mmap-backed billion-scale mapper is a drop-in for the existing
+ * JSON-persisted one.
+ *
+ * ## When this engages
+ *
+ * The cor plugin registers this wrapper as the `'entityIdMapper'`
+ * provider when the storage adapter exposes `getBinaryBlobPath()` (i.e.
+ * filesystem-backed storage with cor's 2.4.0 #2 mmap-vector layer).
+ * Cloud-storage adapters fall back to the JSON variant
+ * (`NativeEntityIdMapperWrapper`) since they have no local-path concept.
+ *
+ * ## UUID format conversion
+ *
+ * Brainy passes UUIDs as strings (typically the canonical 36-char
+ * `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`). The native side works in
+ * 16-byte Buffers. This wrapper converts at the boundary. Non-canonical
+ * UUID strings (any other 32-hex-digit form) are also accepted.
+ *
+ * ## Concurrency
+ *
+ * `getOrAssign` is atomic across concurrent callers for the same UUID
+ * (256 sharded per-UUID mutexes in the native layer). Lookups are
+ * lock-free. The wrapper holds no JS-side mutable state besides the
+ * native handle.
+ *
+ * ## IdSpace (Piece 10)
+ *
+ * The wrapper supports two entity-int wire widths:
+ *
+ * - `'u32'` (default): cortex 2.x compatible — JS `number` throughout,
+ *   capped at 4.29 B entities. Persists in the legacy `int_to_uuid.bin`
+ *   v1 header.
+ * - `'u64'`: opt-in via `idSpace: 'u64'`. The native layer's `number`
+ *   surface throws in this mode, so the wrapper transparently routes
+ *   the `EntityIdMapperProvider` methods through the BigInt napi
+ *   siblings and converts BigInt → number at the boundary. Entity ints
+ *   above `Number.MAX_SAFE_INTEGER` (2^53 - 1 = ~9 PB of entities)
+ *   throw a clear `EntityIdSpaceExceeded` error so callers get a loud
+ *   failure rather than a silent precision loss.
+ *
+ * The `getOrAssignBig` / `getIntBig` / `getUuidBig` / `sizeBig`
+ * sibling methods are available on both modes and always return
+ * `bigint` losslessly — use them in u64-aware code paths that need
+ * the full u64 range.
+ */
+import type { StorageAdapter } from '@soulcraft/brainy';
+import type { EntityIdMapperProvider } from '../providerContracts.js';
+export interface NativeBinaryEntityIdMapperOptions {
+    /** Storage adapter — required for binary blob path resolution. */
+    storage: StorageAdapter;
+    /**
+     * Override the relative path under storage for the uuid_to_int file.
+     * Default `_id_mapper/uuid_to_int.mkv`.
+     */
+    uuidToIntKey?: string;
+    /**
+     * Override the relative path under storage for the int_to_uuid file.
+     * Default `_id_mapper/int_to_uuid.bin`.
+     */
+    intToUuidKey?: string;
+    /** Sparse file size for int_to_uuid. Default 32 GB. */
+    intToUuidSize?: bigint;
+    /** Sparse file size for uuid_to_int. Default 32 GB. */
+    uuidToIntSize?: bigint;
+    /** Bucket capacity in the MmapKv. Default 16. */
+    bucketCapacity?: number;
+    /** Maximum extendible-hash directory depth. Default 28. */
+    maxGlobalDepth?: number;
+    /**
+     * Entity-int wire width. **`'u64'` is the default** for cor 3.0 —
+     * supports `u64::MAX - 1` entities with no measurable perf cost
+     * (audit: 4 extra bytes per int_to_uuid slot, no impact on the hot
+     * paths per `findPerfGates.test.ts`). `'u32'` is available for the
+     * brainy 7.x-compatible cap at 4.29 B entities but is no longer the
+     * default — cor 3.0's design floor of 1 B+ entities makes u32
+     * the wrong default for production.
+     *
+     * The `EntityIdMapperProvider` `number`-typed methods still work in
+     * U64 mode by routing through the BigInt napi siblings; entity ints
+     * above `Number.MAX_SAFE_INTEGER` (2^53 - 1) throw an explicit
+     * `EntityIdSpaceExceeded` error.
+     *
+     * Ignored on open when the underlying file's header disagrees: the
+     * on-disk format wins and any mismatch is surfaced as a hard error.
+     */
+    idSpace?: 'u32' | 'u64';
+}
+/**
+ * Thrown when a U64-mode mapper allocates or returns an entity int
+ * above `Number.MAX_SAFE_INTEGER` (2^53 - 1). At this point a JS
+ * `number` can no longer represent the value losslessly; callers must
+ * switch to the BigInt sibling methods (`getOrAssignBig`,
+ * `getIntBig`, `getUuidBig`).
+ */
+export declare class EntityIdSpaceExceeded extends Error {
+    /** The u64 entity int that exceeded the safe-integer ceiling. */
+    readonly value: bigint;
+    /** The method that was called (`'getOrAssign'`, `'getInt'`, etc.). */
+    readonly method: string;
+    constructor(method: string, value: bigint);
+}
+/**
+ * Drop-in `EntityIdMapperProvider` backed by the native `BinaryIdMapper`.
+ *
+ * @example
+ * ```typescript
+ * const mapper = new NativeBinaryEntityIdMapperWrapper({ storage })
+ * await mapper.init()
+ * const intId = mapper.getOrAssign('12345678-1234-5678-1234-567812345678')
+ * const uuid = mapper.getUuid(intId)
+ * ```
+ */
+export declare class NativeBinaryEntityIdMapperWrapper implements EntityIdMapperProvider {
+    private storage;
+    private uuidToIntKey;
+    private intToUuidKey;
+    private intToUuidSize;
+    private uuidToIntSize;
+    private bucketCapacity;
+    private maxGlobalDepth;
+    private requestedIdSpace;
+    /**
+     * The actual IdSpace of the open mapper, sourced from the native
+     * binding's `idSpace()` reflection after `init()`. The on-disk
+     * header wins over `requestedIdSpace` (which may be ignored at
+     * `openExisting` time).
+     */
+    private resolvedIdSpace;
+    private native;
+    private initialized;
+    constructor(options: NativeBinaryEntityIdMapperOptions);
+    init(): Promise<void>;
+    /**
+     * Report the mapper's actual IdSpace mode. Returns `'u32'` before
+     * `init()` (the default the wrapper assumes); after init, returns the
+     * mode reported by the native binding (which is authoritative).
+     */
+    getIdSpace(): 'u32' | 'u64';
+    /**
+     * Allocate or retrieve the entity int for `uuid`. Returns a JS
+     * `number`. In U64 mode, routes through the BigInt sibling and
+     * throws {@link EntityIdSpaceExceeded} if the allocated int exceeds
+     * `Number.MAX_SAFE_INTEGER` — at that point the caller MUST switch
+     * to `getOrAssignBig` for the full u64 range.
+     */
+    getOrAssign(uuid: string): number;
+    /**
+     * Look up the UUID for `intId`. Accepts a JS `number` — in U64 mode
+     * this is a lossy conversion above 2^53; use {@link getUuidBig} for
+     * the full u64 range.
+     */
+    getUuid(intId: number): string | undefined;
+    /**
+     * Look up the entity int for `uuid`. Returns a JS `number`. In U64
+     * mode throws {@link EntityIdSpaceExceeded} if the int exceeds
+     * `Number.MAX_SAFE_INTEGER`.
+     */
+    getInt(uuid: string): number | undefined;
+    remove(uuid: string): boolean;
+    flush(): Promise<void>;
+    clear(): Promise<void>;
+    /**
+     * Materialise every live int id into a JS `number[]`. **U32 mode
+     * only.** U64 mode throws — the native binding refuses to allocate
+     * a giant JS array at the scale a U64 brain implies. Iterate via
+     * the BigInt sibling iterator (TBD — a follow-up surfaces it on the
+     * wrapper) for U64 brains.
+     */
+    getAllIntIds(): number[];
+    intsIterableToUuids(ints: Iterable<number>): string[];
+    /**
+     * @description Batch-resolve entity ints to their UUIDs in one native
+     * call (ADR-006 Q5) — the provider path brainy uses to hydrate a
+     * `Subgraph`'s `nodes` column. One boundary crossing instead of N
+     * per-int `getUuidBig` calls when a viz/export renders many rows. An
+     * unmapped int yields `''` so the result stays index-aligned with the
+     * input.
+     * @param ints - The entity ints to resolve (a `BigInt64Array`, the
+     *   `Subgraph.nodes` column).
+     * @returns UUID strings index-aligned with `ints` (`''` for unmapped).
+     */
+    entityIntsToUuids(ints: BigInt64Array): string[];
+    get size(): number;
+    /**
+     * Allocate or retrieve the entity int for `uuid` as a `bigint`.
+     * Lossless across the full u64 range; safe to call in either mode.
+     */
+    getOrAssignBig(uuid: string): bigint;
+    /** Look up the entity int for `uuid` as a `bigint`. */
+    getIntBig(uuid: string): bigint | undefined;
+    /** Look up the UUID for `int` (passed as a `bigint`). */
+    getUuidBig(int: bigint): string | undefined;
+    /** Live (non-tombstone) entry count as a `bigint`. */
+    sizeBig(): bigint;
+    /** Largest int ever assigned + 1, as a `bigint`. */
+    nextIntBig(): bigint;
+    /**
+     * Encode a UUID string into a 16-byte Buffer. Accepts canonical
+     * 36-char form (with hyphens) or any 32-hex-digit form. Throws on
+     * malformed input.
+     */
+    private encode;
+    /** Decode a 16-byte Buffer back to canonical UUID string. */
+    private decode;
+    private ensure;
+}
+//# sourceMappingURL=nativeBinaryEntityIdMapper.d.ts.map