@soulcraft/cor 3.0.0-rc.1

package/dist/utils/ColumnManifest.d.ts

@@ -0,0 +1,97 @@
+/**
+ * @module utils/ColumnManifest
+ * @description Cor-local per-field manifest for the native column store.
+ *
+ * Replicates `@soulcraft/brainy`'s internal `ColumnManifest` (not a published
+ * export of brainy 7.24.0) so cor produces interchangeable manifest files.
+ * Stored as JSON at `_column_index/{field}/MANIFEST.json`; the manifest is the
+ * source of truth for which segments exist for a field. On startup the column
+ * store loads each field's manifest and discovers segments from it.
+ *
+ * Manifest JSON is written/read via the storage adapter's object-path
+ * primitives (`writeObjectToPath`/`readObjectFromPath`) — the same channel
+ * brainy uses for workspace-global metadata such as `_cow/`. Segment payloads
+ * themselves are NOT stored here; they are raw mmap-able binary blobs persisted
+ * separately by `NativeColumnStore`.
+ */
+import type { SegmentMeta } from './columnStoreTypes.js';
+import { ValueType } from './columnStoreTypes.js';
+/**
+ * @description The object-path JSON surface a manifest needs from storage.
+ * Present on every brainy filesystem/cloud adapter; the published
+ * `StorageAdapter` type omits it, so it is declared locally.
+ */
+interface ManifestStorage {
+    writeObjectToPath(path: string, value: unknown): Promise<void>;
+    readObjectFromPath(path: string): Promise<any>;
+}
+/**
+ * Manages a single field's segment manifest.
+ *
+ * @example
+ * const manifest = new ColumnManifest('createdAt', '_column_index')
+ * await manifest.load(storage)
+ * const id = manifest.nextSegmentId()
+ * manifest.addSegment({ id, level: 0, count: 0, minValue: 0, maxValue: 0, file: 'L0-000001.cidx' })
+ * await manifest.save(storage)
+ */
+export declare class ColumnManifest {
+    /** Field this manifest covers. */
+    readonly fieldName: string;
+    /** Base storage path for the column index (e.g. `_column_index`). */
+    readonly basePath: string;
+    /** Manifest data — loaded from storage or initialized fresh. */
+    private data;
+    /**
+     * @param fieldName - The field name (e.g. `createdAt`, `name`, `score`).
+     * @param basePath - Base path for segment files (default `_column_index`).
+     */
+    constructor(fieldName: string, basePath?: string);
+    /**
+     * @description Load the manifest from storage. If not found, stays fresh.
+     * @param storage - Storage adapter with object-path support.
+     */
+    load(storage: ManifestStorage): Promise<void>;
+    /**
+     * @description Save the manifest to storage atomically, bumping the version.
+     * @param storage - Storage adapter with object-path support.
+     */
+    save(storage: ManifestStorage): Promise<void>;
+    /**
+     * @description Register a new segment, keeping segments ordered by level then
+     * id for consistent iteration.
+     * @param meta - Segment metadata to record.
+     */
+    addSegment(meta: SegmentMeta): void;
+    /**
+     * @description Allocate the next segment id (monotonically increasing).
+     * @returns The next available segment id.
+     */
+    nextSegmentId(): number;
+    /**
+     * @description Roll back the most recent {@link nextSegmentId} allocation when
+     * the corresponding segment was not actually written, keeping the counter
+     * gap-free. Never goes below 1.
+     */
+    rollbackSegmentId(): void;
+    /**
+     * @description All segments across all levels.
+     * @returns The segment list (ordered by level then id).
+     */
+    getAllSegments(): SegmentMeta[];
+    /**
+     * @description Whether the manifest has no segments.
+     * @returns True if empty.
+     */
+    isEmpty(): boolean;
+    /** Value type for this field (persisted in the manifest). */
+    get valueType(): ValueType;
+    set valueType(type: ValueType);
+    /**
+     * @description Storage path for this field's manifest JSON.
+     * @returns Path like `_column_index/createdAt/MANIFEST.json`.
+     */
+    manifestPath(): string;
+}
+export {};
+//# sourceMappingURL=ColumnManifest.d.ts.map

package/dist/utils/ColumnManifest.js

@@ -0,0 +1,129 @@
+/**
+ * @module utils/ColumnManifest
+ * @description Cor-local per-field manifest for the native column store.
+ *
+ * Replicates `@soulcraft/brainy`'s internal `ColumnManifest` (not a published
+ * export of brainy 7.24.0) so cor produces interchangeable manifest files.
+ * Stored as JSON at `_column_index/{field}/MANIFEST.json`; the manifest is the
+ * source of truth for which segments exist for a field. On startup the column
+ * store loads each field's manifest and discovers segments from it.
+ *
+ * Manifest JSON is written/read via the storage adapter's object-path
+ * primitives (`writeObjectToPath`/`readObjectFromPath`) — the same channel
+ * brainy uses for workspace-global metadata such as `_cow/`. Segment payloads
+ * themselves are NOT stored here; they are raw mmap-able binary blobs persisted
+ * separately by `NativeColumnStore`.
+ */
+import { ValueType } from './columnStoreTypes.js';
+/**
+ * Manages a single field's segment manifest.
+ *
+ * @example
+ * const manifest = new ColumnManifest('createdAt', '_column_index')
+ * await manifest.load(storage)
+ * const id = manifest.nextSegmentId()
+ * manifest.addSegment({ id, level: 0, count: 0, minValue: 0, maxValue: 0, file: 'L0-000001.cidx' })
+ * await manifest.save(storage)
+ */
+export class ColumnManifest {
+    /** Field this manifest covers. */
+    fieldName;
+    /** Base storage path for the column index (e.g. `_column_index`). */
+    basePath;
+    /** Manifest data — loaded from storage or initialized fresh. */
+    data;
+    /**
+     * @param fieldName - The field name (e.g. `createdAt`, `name`, `score`).
+     * @param basePath - Base path for segment files (default `_column_index`).
+     */
+    constructor(fieldName, basePath = '_column_index') {
+        this.fieldName = fieldName;
+        this.basePath = basePath;
+        this.data = {
+            version: 0,
+            field: fieldName,
+            valueType: ValueType.Number,
+            multiValue: false,
+            segments: [],
+            nextSegmentId: 1,
+            buildComplete: false
+        };
+    }
+    /**
+     * @description Load the manifest from storage. If not found, stays fresh.
+     * @param storage - Storage adapter with object-path support.
+     */
+    async load(storage) {
+        try {
+            const data = await storage.readObjectFromPath(this.manifestPath());
+            if (data && data.field === this.fieldName) {
+                this.data = data;
+            }
+        }
+        catch {
+            // Not found — start fresh (data is already initialized in the constructor).
+        }
+    }
+    /**
+     * @description Save the manifest to storage atomically, bumping the version.
+     * @param storage - Storage adapter with object-path support.
+     */
+    async save(storage) {
+        this.data.version++;
+        await storage.writeObjectToPath(this.manifestPath(), this.data);
+    }
+    /**
+     * @description Register a new segment, keeping segments ordered by level then
+     * id for consistent iteration.
+     * @param meta - Segment metadata to record.
+     */
+    addSegment(meta) {
+        this.data.segments.push(meta);
+        this.data.segments.sort((a, b) => (a.level !== b.level ? a.level - b.level : a.id - b.id));
+    }
+    /**
+     * @description Allocate the next segment id (monotonically increasing).
+     * @returns The next available segment id.
+     */
+    nextSegmentId() {
+        return this.data.nextSegmentId++;
+    }
+    /**
+     * @description Roll back the most recent {@link nextSegmentId} allocation when
+     * the corresponding segment was not actually written, keeping the counter
+     * gap-free. Never goes below 1.
+     */
+    rollbackSegmentId() {
+        if (this.data.nextSegmentId > 1)
+            this.data.nextSegmentId--;
+    }
+    /**
+     * @description All segments across all levels.
+     * @returns The segment list (ordered by level then id).
+     */
+    getAllSegments() {
+        return this.data.segments;
+    }
+    /**
+     * @description Whether the manifest has no segments.
+     * @returns True if empty.
+     */
+    isEmpty() {
+        return this.data.segments.length === 0;
+    }
+    /** Value type for this field (persisted in the manifest). */
+    get valueType() {
+        return this.data.valueType;
+    }
+    set valueType(type) {
+        this.data.valueType = type;
+    }
+    /**
+     * @description Storage path for this field's manifest JSON.
+     * @returns Path like `_column_index/createdAt/MANIFEST.json`.
+     */
+    manifestPath() {
+        return `${this.basePath}/${this.fieldName}/MANIFEST.json`;
+    }
+}
+//# sourceMappingURL=ColumnManifest.js.map

package/dist/utils/NativeColumnStore.d.ts

@@ -0,0 +1,284 @@
+/**
+ * @module utils/NativeColumnStore
+ * @description Rust-accelerated implementation of brainy's `ColumnStoreProvider`.
+ *
+ * Brainy's unified column store keeps per-field sorted columns for filtering,
+ * range queries, and `orderBy` sorting at billion-entity scale. The TypeScript
+ * baseline ships in `@soulcraft/brainy`; cor registers this native version,
+ * which delegates every value-level operation to the Rust `NativeColumnStore`
+ * (`native/src/column_store`). The Rust engine owns the sorted-segment k-way
+ * merge, roaring-bitmap filters, and zero-copy mmap reads; this module owns the
+ * async storage bridge (segment persistence, manifests, deleted bitmaps) and
+ * the value typing/coercion that mirrors brainy exactly.
+ *
+ * Why cor needs this: cor's `MetadataIndexManager` does its own native
+ * metadata filtering, but brainy's *unfiltered* `find({ orderBy })` path calls
+ * `metadataIndex.columnStore.sortTopK(...)` directly. Without a `.columnStore`
+ * on cor's metadata index, that path throws. So the column store's real jobs
+ * here are POPULATE (as entities are indexed), PERSIST (segments + manifests),
+ * and SORT (`sortTopK`). `filter`/`rangeQuery`/`getFilterValues` are implemented
+ * for interface completeness and parity with the TS baseline.
+ *
+ * Storage is billion-scale and mmap-friendly: each flushed segment is written as
+ * a raw binary blob (NOT base64) via the storage adapter's `saveBinaryBlob`, and
+ * loaded zero-copy via `getBinaryBlobPath` + `loadSegmentFromPath` when a local
+ * path exists, falling back to `loadBinaryBlob` + `loadSegmentFromBuffer` for
+ * remote adapters with no local path. Manifests are small JSON files.
+ *
+ * Design decision: value typing/coercion is replicated from brainy's
+ * `ColumnStore.inferValueType`/`normalizeValue` so cor produces identical
+ * `.cidx` segment value types and ordering. See ADR-001 for the native string
+ * collation contract (code-point order, matching `compareCodePoints`).
+ */
+import type { StorageAdapter } from '@soulcraft/brainy';
+import type { ColumnStoreProvider, EntityIdMapperLike } from './columnStoreTypes.js';
+import { ValueType } from './columnStoreTypes.js';
+import { RoaringBitmap32 } from '../native/NativeRoaringBitmap32.js';
+/**
+ * @description The subset of cor's `MmapFileSystemStorage` that the column
+ * store needs for billion-scale raw segment persistence. cor builds against
+ * published `@soulcraft/brainy`, whose `StorageAdapter` type does not yet
+ * declare these binary-blob methods, so we narrow the adapter to this local
+ * interface at runtime. A storage that lacks `saveBinaryBlob` cannot persist
+ * mmap-able segments — that is a hard error, never a silent base64 fallback.
+ */
+export interface BinaryBlobStorage {
+    /** Persist a raw binary blob under a logical key. The adapter appends its own extension. */
+    saveBinaryBlob(key: string, data: Buffer): Promise<void>;
+    /** Load a previously persisted blob, or null if absent. */
+    loadBinaryBlob(key: string): Promise<Buffer | null>;
+    /** Delete a persisted blob. No-op if absent. */
+    deleteBinaryBlob(key: string): Promise<void>;
+    /** Filesystem path to a blob for zero-copy mmap, or null for non-local adapters. */
+    getBinaryBlobPath(key: string): string | null;
+}
+/**
+ * NativeColumnStore — Rust-accelerated `ColumnStoreProvider`.
+ *
+ * Holds one native `NativeColumnStore` instance plus the TS-side state the Rust
+ * engine does not persist: per-field manifests, value types, deleted bitmaps,
+ * and the storage + id mapper handed in at `init`.
+ *
+ * @example
+ * const store = new NativeColumnStore()
+ * await store.init(storage, idMapper)
+ * store.addEntity(42, { createdAt: Date.now(), name: 'alpha', score: 1.5 })
+ * await store.flush()
+ * const newest = await store.sortTopK('createdAt', 'desc', 10) // int ids
+ */
+export declare class NativeColumnStore implements ColumnStoreProvider {
+    /**
+     * The Rust engine. Owns tail buffers, segments, sort/filter/range. Mutable
+     * because {@link reset} replaces it with a fresh instance to discard all
+     * native state (the napi class exposes no in-place clear).
+     */
+    private native;
+    /** Storage adapter narrowed to the binary-blob + object-path surface. */
+    private storage;
+    /** Per-field manifests tracking persisted segments. */
+    private readonly manifests;
+    /** Per-field value type, inferred from the first value seen (matches brainy). */
+    private readonly fieldTypes;
+    /**
+     * Per-field deleted-entity bitmaps. Entity int ids here are excluded from all
+     * native queries (the Rust engine also tracks them via `deleteEntity`); the
+     * TS copy is what gets persisted to `DELETED.bin` and re-applied on init.
+     */
+    private readonly deletedEntities;
+    /**
+     * Create a native column store. Call {@link init} before queries that need
+     * storage (loading persisted segments). Writes work immediately — the Rust
+     * tail buffer holds them in memory until {@link flush}.
+     */
+    constructor();
+    /**
+     * @description Initialize the store: narrow storage, discover persisted field
+     * manifests under `_column_index/`, load each field's segments into the Rust
+     * engine (zero-copy mmap when a local path exists), and re-apply persisted
+     * deleted bitmaps.
+     * @param storage - The brainy storage adapter (must support binary blobs).
+     * @param _idMapper - The shared entity id mapper (part of the provider
+     *   contract). This native implementation returns int ids directly from
+     *   sort/filter/range and lets cor's metadata index own the canonical
+     *   UUID ↔ int mapping, so the mapper is accepted but not retained here.
+     * @throws If the storage adapter lacks the binary-blob primitives.
+     */
+    init(storage: StorageAdapter, _idMapper: EntityIdMapperLike): Promise<void>;
+    /**
+     * @description Index an entity's field values into the Rust tail buffers.
+     * Synchronous, matching the provider contract. Value typing mirrors brainy:
+     * integer number → numeric (i64, rounded); non-integer number → float (f64);
+     * boolean → numeric 0/1; string → string. Arrays add one entry per element.
+     * `null`/`undefined` values are skipped.
+     * @param entityIntId - The entity's u32 id (from the shared id mapper).
+     * @param fields - Map of field name → value (or array of values).
+     */
+    addEntity(entityIntId: number, fields: Record<string, unknown>): void;
+    /**
+     * @description Tombstone an entity across every known field. The Rust engine
+     * excludes it from queries immediately; the TS bitmap is persisted on flush so
+     * the tombstone survives a reopen.
+     * @param entityIntId - The entity's u32 id.
+     */
+    removeEntity(entityIntId: number): void;
+    /**
+     * @description Point filter: entity int ids whose `field` equals `value`.
+     * Routes to the numeric/float/string Rust filter by the field's value type.
+     * @param field - Field to filter.
+     * @param value - Exact value to match.
+     * @returns Roaring bitmap of matching entity int ids.
+     */
+    filter(field: string, value: unknown): Promise<RoaringBitmap32>;
+    /**
+     * @description Decode a native column-store filter/range result into a
+     * `RoaringBitmap32`. Cor 3.0 Piece E switched the native `filter*` /
+     * `rangeQuery*` output to a croaring `Treemap` (Roaring64) portable bitmap,
+     * whose container layout is NOT interchangeable with the Roaring32 `Bitmap`
+     * portable format — decoding the Treemap buffer via `RoaringBitmap32` yields
+     * an empty set (croaring rejects the foreign cookie and returns empty). We
+     * therefore decode through `NativeRoaringBitmap64` and narrow each entity int
+     * to a `number`, lossless for the column store's entity-int range (≤ u32::MAX
+     * in every shipping deployment) and consistent with {@link sortTopK}'s
+     * `Number()` narrowing. U64 brains past that range use brainy 8.0's
+     * u64-capable wrapper.
+     * @param buf - A Roaring64 (Treemap) portable buffer from the native engine.
+     * @returns The matching entity int ids as a `RoaringBitmap32`.
+     */
+    private decodeFilterResult;
+    /**
+     * @description Range filter: entity int ids whose `field` is within
+     * `[min, max]` (inclusive). Open bounds fall back to the type's extreme so a
+     * one-sided range still works. Routes by the field's value type.
+     * @param field - Field to filter.
+     * @param min - Lower bound, or undefined for no lower bound.
+     * @param max - Upper bound, or undefined for no upper bound.
+     * @returns Roaring bitmap of matching entity int ids.
+     */
+    rangeQuery(field: string, min?: unknown, max?: unknown): Promise<RoaringBitmap32>;
+    /**
+     * @description Sort top-K: the K entity int ids with the highest (`desc`) or
+     * lowest (`asc`) values in `field`, in sort order. This is the method brainy's
+     * unfiltered `orderBy` path calls. The Rust engine performs the k-way merge
+     * across sorted segments + the (per-query sorted) tail buffer and excludes
+     * tombstoned ids.
+     * @param field - Field to sort by.
+     * @param order - 'asc' or 'desc'.
+     * @param k - Maximum number of ids to return.
+     * @returns Sorted array of entity int ids.
+     */
+    sortTopK(field: string, order: 'asc' | 'desc', k: number): Promise<number[]>;
+    /**
+     * @description Filtered sort top-K: {@link sortTopK} restricted to ids present
+     * in `filterBitmap`. Implemented as sort-then-intersect: the native sort is
+     * pulled deep enough that the post-filter still yields up to K ids, then the
+     * filter bitmap is applied in order.
+     * @param filterBitmap - Candidate entity int ids.
+     * @param field - Field to sort by.
+     * @param order - 'asc' or 'desc'.
+     * @param k - Maximum number of ids to return.
+     * @returns Sorted array of entity int ids, all present in `filterBitmap`.
+     */
+    filteredSortTopK(filterBitmap: RoaringBitmap32, field: string, order: 'asc' | 'desc', k: number): Promise<number[]>;
+    /**
+     * @description All distinct values for a field, as strings in ascending order
+     * (numeric by value, string by code point), backed by the Rust engine's
+     * segment + tail scan.
+     * @param field - Field name.
+     * @returns Distinct values stringified the way brainy renders them.
+     */
+    getFilterValues(field: string): Promise<string[]>;
+    /**
+     * @description Whether a field has any indexed data (a persisted segment or
+     * buffered tail entry) in the Rust engine.
+     * @param field - Field name.
+     * @returns True if the field is queryable.
+     */
+    hasField(field: string): boolean;
+    /**
+     * @description Flush every field's tail buffer to a new L0 `.cidx` segment,
+     * persist the segment as a raw binary blob, update the manifest, and persist
+     * the deleted bitmap. No-op if {@link init} has not run (no storage to write).
+     */
+    flush(): Promise<void>;
+    /**
+     * @description Flush and release all resources. Persists pending writes first.
+     */
+    close(): Promise<void>;
+    /**
+     * @description Discard all column-store state — in-memory AND persisted — and
+     * start fresh. Called when the owning metadata index rebuilds or clears: a
+     * rebuild reassigns entity int ids from scratch, so any persisted `.cidx`
+     * segments (which encode the OLD ids) would point at the wrong entities. This
+     * drops the native engine (a fresh instance discards all tail buffers and
+     * loaded segments), clears the manifests/value-types/deleted bitmaps, and
+     * deletes every persisted file under `_column_index/`. The store remains
+     * initialized and ready: the rebuild's subsequent `addEntity` calls repopulate
+     * it consistently with the new id assignments.
+     */
+    reset(): Promise<void>;
+    /**
+     * @description Add a single value to a field, inferring + recording the value
+     * type on first write and routing to the matching Rust tail buffer.
+     * @param field - Field name.
+     * @param value - A non-null primitive value.
+     * @param entityIntId - The entity's u32 id.
+     */
+    private addValue;
+    /**
+     * @description Flush a single field: drain its Rust tail buffer to a `.cidx`
+     * segment buffer, persist it as a binary blob, register it in the manifest,
+     * then persist the manifest and (if any) the deleted bitmap.
+     * @param field - Field to flush.
+     */
+    private flushField;
+    /**
+     * @description Load all of a field's persisted segments into the Rust engine.
+     * Used on init, where no segment buffer is held in hand, so each segment is
+     * loaded by mmap path or read from storage.
+     * @param field - Field name.
+     * @param manifest - The field's loaded manifest.
+     */
+    private loadFieldSegments;
+    /**
+     * @description Load one persisted segment into the Rust engine, preferring
+     * zero-copy mmap (`loadSegmentFromPath`) when the adapter exposes a local
+     * path. Falls back to `fallbackBuffer` if provided (the just-flushed bytes),
+     * otherwise reads the blob from storage. Used both on init (reload all
+     * segments) and on flush (re-attach the segment a drained tail just produced).
+     * @param field - Field name.
+     * @param segId - Segment id.
+     * @param level - Segment compaction level.
+     * @param fallbackBuffer - Segment bytes already in memory (flush path), if any.
+     */
+    private loadSegmentIntoEngine;
+    /**
+     * @description Load and re-apply a field's persisted deleted bitmap by calling
+     * `deleteEntity` for each id, so tombstones survive a reopen.
+     * @param field - Field name.
+     */
+    private loadDeletedBitmap;
+    /**
+     * @description Coerce a JS value to the i64 numeric column representation,
+     * matching brainy's `normalizeValue` for `Number`/`Boolean`.
+     * @param value - The raw value.
+     * @param type - The target value type (`Number` or `Boolean`).
+     * @returns The integer value, or undefined if not coercible.
+     */
+    private toNumeric;
+    /**
+     * @description Coerce a JS value to the f64 float column representation,
+     * matching brainy's `normalizeValue` for `Float`.
+     * @param value - The raw value.
+     * @returns The float value, or undefined if not coercible.
+     */
+    private toFloat;
+}
+/**
+ * @description Infer the column `ValueType` from a JS value, identical to
+ * brainy's `ColumnStore.inferValueType`: boolean → Boolean; integer number →
+ * Number; non-integer number → Float; everything else → String.
+ * @param value - A non-null value.
+ * @returns The inferred value type.
+ */
+export declare function inferValueType(value: unknown): ValueType;
+//# sourceMappingURL=NativeColumnStore.d.ts.map