@soulcraft/cor 3.0.0-rc.1

package/dist/utils/NativeColumnStore.js

@@ -0,0 +1,685 @@
+/**
+ * @module utils/NativeColumnStore
+ * @description Rust-accelerated implementation of brainy's `ColumnStoreProvider`.
+ *
+ * Brainy's unified column store keeps per-field sorted columns for filtering,
+ * range queries, and `orderBy` sorting at billion-entity scale. The TypeScript
+ * baseline ships in `@soulcraft/brainy`; cor registers this native version,
+ * which delegates every value-level operation to the Rust `NativeColumnStore`
+ * (`native/src/column_store`). The Rust engine owns the sorted-segment k-way
+ * merge, roaring-bitmap filters, and zero-copy mmap reads; this module owns the
+ * async storage bridge (segment persistence, manifests, deleted bitmaps) and
+ * the value typing/coercion that mirrors brainy exactly.
+ *
+ * Why cor needs this: cor's `MetadataIndexManager` does its own native
+ * metadata filtering, but brainy's *unfiltered* `find({ orderBy })` path calls
+ * `metadataIndex.columnStore.sortTopK(...)` directly. Without a `.columnStore`
+ * on cor's metadata index, that path throws. So the column store's real jobs
+ * here are POPULATE (as entities are indexed), PERSIST (segments + manifests),
+ * and SORT (`sortTopK`). `filter`/`rangeQuery`/`getFilterValues` are implemented
+ * for interface completeness and parity with the TS baseline.
+ *
+ * Storage is billion-scale and mmap-friendly: each flushed segment is written as
+ * a raw binary blob (NOT base64) via the storage adapter's `saveBinaryBlob`, and
+ * loaded zero-copy via `getBinaryBlobPath` + `loadSegmentFromPath` when a local
+ * path exists, falling back to `loadBinaryBlob` + `loadSegmentFromBuffer` for
+ * remote adapters with no local path. Manifests are small JSON files.
+ *
+ * Design decision: value typing/coercion is replicated from brainy's
+ * `ColumnStore.inferValueType`/`normalizeValue` so cor produces identical
+ * `.cidx` segment value types and ordering. See ADR-001 for the native string
+ * collation contract (code-point order, matching `compareCodePoints`).
+ */
+import { existsSync } from 'node:fs';
+import { prodLog } from '@soulcraft/brainy/internals';
+import { ValueType } from './columnStoreTypes.js';
+import { ColumnManifest } from './ColumnManifest.js';
+import { loadNativeModule } from '../native/index.js';
+import { RoaringBitmap32 } from '../native/NativeRoaringBitmap32.js';
+/**
+ * @description Narrow check that a storage adapter supports the binary-blob
+ * primitives required for mmap segment persistence.
+ * @param storage - The storage adapter brainy handed to the metadata index.
+ * @returns True if all four binary-blob methods are present.
+ */
+function isBinaryBlobStorage(storage) {
+    const s = storage;
+    return (typeof s?.saveBinaryBlob === 'function' &&
+        typeof s?.loadBinaryBlob === 'function' &&
+        typeof s?.deleteBinaryBlob === 'function' &&
+        typeof s?.getBinaryBlobPath === 'function');
+}
+/** Base storage path for all column-index data, matching brainy's TS baseline. */
+const BASE_PATH = '_column_index';
+/**
+ * @description Build the binary-blob storage key for a flushed segment. The key
+ * intentionally carries NO `.cidx`/`.bin` suffix — the adapter appends its own
+ * (`.bin`). The same convention is used on load, so persistence round-trips.
+ * @param field - Field name the segment belongs to.
+ * @param level - LSM compaction level (always 0 for fresh flushes here).
+ * @param segId - Segment id, unique within the field.
+ * @returns Key like `_column_index/createdAt/L0-000001`.
+ */
+function segmentBlobKey(field, level, segId) {
+    return `${BASE_PATH}/${field}/L${level}-${String(segId).padStart(6, '0')}`;
+}
+/**
+ * @description Build the binary-blob storage key for a field's persisted
+ * deleted bitmap. Suffix-free for the same reason as {@link segmentBlobKey}.
+ * @param field - Field name.
+ * @returns Key like `_column_index/createdAt/DELETED`.
+ */
+function deletedBlobKey(field) {
+    return `${BASE_PATH}/${field}/DELETED`;
+}
+/**
+ * NativeColumnStore — Rust-accelerated `ColumnStoreProvider`.
+ *
+ * Holds one native `NativeColumnStore` instance plus the TS-side state the Rust
+ * engine does not persist: per-field manifests, value types, deleted bitmaps,
+ * and the storage + id mapper handed in at `init`.
+ *
+ * @example
+ * const store = new NativeColumnStore()
+ * await store.init(storage, idMapper)
+ * store.addEntity(42, { createdAt: Date.now(), name: 'alpha', score: 1.5 })
+ * await store.flush()
+ * const newest = await store.sortTopK('createdAt', 'desc', 10) // int ids
+ */
+export class NativeColumnStore {
+    /**
+     * The Rust engine. Owns tail buffers, segments, sort/filter/range. Mutable
+     * because {@link reset} replaces it with a fresh instance to discard all
+     * native state (the napi class exposes no in-place clear).
+     */
+    native;
+    /** Storage adapter narrowed to the binary-blob + object-path surface. */
+    storage = null;
+    /** Per-field manifests tracking persisted segments. */
+    manifests = new Map();
+    /** Per-field value type, inferred from the first value seen (matches brainy). */
+    fieldTypes = new Map();
+    /**
+     * Per-field deleted-entity bitmaps. Entity int ids here are excluded from all
+     * native queries (the Rust engine also tracks them via `deleteEntity`); the
+     * TS copy is what gets persisted to `DELETED.bin` and re-applied on init.
+     */
+    deletedEntities = new Map();
+    /**
+     * Create a native column store. Call {@link init} before queries that need
+     * storage (loading persisted segments). Writes work immediately — the Rust
+     * tail buffer holds them in memory until {@link flush}.
+     */
+    constructor() {
+        const bindings = loadNativeModule();
+        this.native = new bindings.NativeColumnStore();
+    }
+    /**
+     * @description Initialize the store: narrow storage, discover persisted field
+     * manifests under `_column_index/`, load each field's segments into the Rust
+     * engine (zero-copy mmap when a local path exists), and re-apply persisted
+     * deleted bitmaps.
+     * @param storage - The brainy storage adapter (must support binary blobs).
+     * @param _idMapper - The shared entity id mapper (part of the provider
+     *   contract). This native implementation returns int ids directly from
+     *   sort/filter/range and lets cor's metadata index own the canonical
+     *   UUID ↔ int mapping, so the mapper is accepted but not retained here.
+     * @throws If the storage adapter lacks the binary-blob primitives.
+     */
+    async init(storage, _idMapper) {
+        if (!isBinaryBlobStorage(storage)) {
+            throw new Error('NativeColumnStore requires a storage adapter with binary-blob support ' +
+                '(saveBinaryBlob/loadBinaryBlob/deleteBinaryBlob/getBinaryBlobPath). ' +
+                'Use cor MmapFileSystemStorage or another adapter implementing BinaryBlobStorage.');
+        }
+        this.storage = storage;
+        // Discover fields by listing manifest files under the column-index prefix.
+        let paths;
+        try {
+            paths = await this.storage.listObjectsUnderPath(BASE_PATH + '/');
+        }
+        catch (error) {
+            // No column index exists yet — a fresh workspace. Not an error.
+            prodLog.debug('NativeColumnStore: no existing column index to load', error);
+            return;
+        }
+        for (const path of paths) {
+            if (!path.endsWith('/MANIFEST.json'))
+                continue;
+            const field = path.replace(BASE_PATH + '/', '').replace('/MANIFEST.json', '');
+            if (!field)
+                continue;
+            // Per-field fault isolation: one field that fails to load (corrupt or
+            // legacy segment, missing blob, parser mismatch) must NOT abort the
+            // discovery of every OTHER field. The previous single try/catch around
+            // the whole loop meant the first failing field silently dropped all
+            // later fields' columns — e.g. `find({ orderBy: 'name' })` returning
+            // `[]` because an earlier field threw before `name` was reached.
+            try {
+                const manifest = new ColumnManifest(field, BASE_PATH);
+                await manifest.load(this.storage);
+                this.manifests.set(field, manifest);
+                this.fieldTypes.set(field, manifest.valueType);
+                await this.loadFieldSegments(field, manifest);
+                await this.loadDeletedBitmap(field);
+            }
+            catch (error) {
+                prodLog.warn(`NativeColumnStore: failed to load column field '${field}' — skipping it; ` +
+                    `other fields are unaffected`, error);
+            }
+        }
+    }
+    /**
+     * @description Index an entity's field values into the Rust tail buffers.
+     * Synchronous, matching the provider contract. Value typing mirrors brainy:
+     * integer number → numeric (i64, rounded); non-integer number → float (f64);
+     * boolean → numeric 0/1; string → string. Arrays add one entry per element.
+     * `null`/`undefined` values are skipped.
+     * @param entityIntId - The entity's u32 id (from the shared id mapper).
+     * @param fields - Map of field name → value (or array of values).
+     */
+    addEntity(entityIntId, fields) {
+        for (const [field, rawValue] of Object.entries(fields)) {
+            if (rawValue === undefined || rawValue === null)
+                continue;
+            if (Array.isArray(rawValue)) {
+                for (const v of rawValue) {
+                    if (v !== undefined && v !== null) {
+                        this.addValue(field, v, entityIntId);
+                    }
+                }
+            }
+            else {
+                this.addValue(field, rawValue, entityIntId);
+            }
+        }
+    }
+    /**
+     * @description Tombstone an entity across every known field. The Rust engine
+     * excludes it from queries immediately; the TS bitmap is persisted on flush so
+     * the tombstone survives a reopen.
+     * @param entityIntId - The entity's u32 id.
+     */
+    removeEntity(entityIntId) {
+        // Tombstone in every field that has data (segments or tail buffer).
+        const fields = new Set([...this.manifests.keys(), ...this.fieldTypes.keys()]);
+        for (const field of fields) {
+            // Cor 3.0 Piece E: napi now accepts BigInt for the full u64
+            // range. The wrapper still surfaces u32 numbers for back-compat;
+            // brains with ids past u32::MAX need brainy 8.0's u64 wrapper.
+            this.native.deleteEntity(field, BigInt(entityIntId));
+            let deleted = this.deletedEntities.get(field);
+            if (!deleted) {
+                deleted = new RoaringBitmap32();
+                this.deletedEntities.set(field, deleted);
+            }
+            deleted.add(entityIntId);
+        }
+    }
+    /**
+     * @description Point filter: entity int ids whose `field` equals `value`.
+     * Routes to the numeric/float/string Rust filter by the field's value type.
+     * @param field - Field to filter.
+     * @param value - Exact value to match.
+     * @returns Roaring bitmap of matching entity int ids.
+     */
+    async filter(field, value) {
+        const type = this.fieldTypes.get(field);
+        if (type === undefined)
+            return new RoaringBitmap32();
+        let buf;
+        switch (type) {
+            case ValueType.Float: {
+                const n = this.toFloat(value);
+                if (n === undefined)
+                    return new RoaringBitmap32();
+                buf = this.native.filterFloat(field, n);
+                break;
+            }
+            case ValueType.String:
+                buf = this.native.filterString(field, String(value));
+                break;
+            case ValueType.Number:
+            case ValueType.Boolean:
+            default: {
+                const n = this.toNumeric(value, type);
+                if (n === undefined)
+                    return new RoaringBitmap32();
+                buf = this.native.filterNumeric(field, n);
+                break;
+            }
+        }
+        return this.decodeFilterResult(buf);
+    }
+    /**
+     * @description Decode a native column-store filter/range result into a
+     * `RoaringBitmap32`. Cor 3.0 Piece E switched the native `filter*` /
+     * `rangeQuery*` output to a croaring `Treemap` (Roaring64) portable bitmap,
+     * whose container layout is NOT interchangeable with the Roaring32 `Bitmap`
+     * portable format — decoding the Treemap buffer via `RoaringBitmap32` yields
+     * an empty set (croaring rejects the foreign cookie and returns empty). We
+     * therefore decode through `NativeRoaringBitmap64` and narrow each entity int
+     * to a `number`, lossless for the column store's entity-int range (≤ u32::MAX
+     * in every shipping deployment) and consistent with {@link sortTopK}'s
+     * `Number()` narrowing. U64 brains past that range use brainy 8.0's
+     * u64-capable wrapper.
+     * @param buf - A Roaring64 (Treemap) portable buffer from the native engine.
+     * @returns The matching entity int ids as a `RoaringBitmap32`.
+     */
+    decodeFilterResult(buf) {
+        const bindings = loadNativeModule();
+        const ints = bindings.NativeRoaringBitmap64.deserialize(buf).toArray();
+        if (ints.length === 0)
+            return new RoaringBitmap32();
+        return new RoaringBitmap32(ints.map((v) => Number(v)));
+    }
+    /**
+     * @description Range filter: entity int ids whose `field` is within
+     * `[min, max]` (inclusive). Open bounds fall back to the type's extreme so a
+     * one-sided range still works. Routes by the field's value type.
+     * @param field - Field to filter.
+     * @param min - Lower bound, or undefined for no lower bound.
+     * @param max - Upper bound, or undefined for no upper bound.
+     * @returns Roaring bitmap of matching entity int ids.
+     */
+    async rangeQuery(field, min, max) {
+        const type = this.fieldTypes.get(field);
+        if (type === undefined)
+            return new RoaringBitmap32();
+        let buf;
+        if (type === ValueType.String) {
+            // String code-point range. Empty string is the code-point minimum; the
+            // max sentinel \u{10FFFF} is the highest scalar value, so an open upper
+            // bound includes every string.
+            const lo = min !== undefined && min !== null ? String(min) : '';
+            const hi = max !== undefined && max !== null ? String(max) : '\u{10FFFF}';
+            buf = this.native.rangeQueryString(field, lo, hi);
+        }
+        else if (type === ValueType.Float) {
+            const lo = min !== undefined && min !== null ? this.toFloat(min) ?? -Infinity : -Infinity;
+            const hi = max !== undefined && max !== null ? this.toFloat(max) ?? Infinity : Infinity;
+            buf = this.native.rangeQueryFloat(field, lo, hi);
+        }
+        else {
+            const lo = min !== undefined && min !== null
+                ? this.toNumeric(min, type) ?? Number.MIN_SAFE_INTEGER
+                : Number.MIN_SAFE_INTEGER;
+            const hi = max !== undefined && max !== null
+                ? this.toNumeric(max, type) ?? Number.MAX_SAFE_INTEGER
+                : Number.MAX_SAFE_INTEGER;
+            buf = this.native.rangeQueryNumeric(field, lo, hi);
+        }
+        return this.decodeFilterResult(buf);
+    }
+    /**
+     * @description Sort top-K: the K entity int ids with the highest (`desc`) or
+     * lowest (`asc`) values in `field`, in sort order. This is the method brainy's
+     * unfiltered `orderBy` path calls. The Rust engine performs the k-way merge
+     * across sorted segments + the (per-query sorted) tail buffer and excludes
+     * tombstoned ids.
+     * @param field - Field to sort by.
+     * @param order - 'asc' or 'desc'.
+     * @param k - Maximum number of ids to return.
+     * @returns Sorted array of entity int ids.
+     */
+    async sortTopK(field, order, k) {
+        if (k <= 0)
+            return [];
+        // Cor 3.0 Piece E: napi returns BigInt[]. The wrapper narrows
+        // to number for back-compat (safe for ids ≤ u32::MAX). U64 brains
+        // need brainy 8.0's u64-capable wrapper.
+        return this.native.sortTopK(field, order, k).map(id => Number(id));
+    }
+    /**
+     * @description Filtered sort top-K: {@link sortTopK} restricted to ids present
+     * in `filterBitmap`. Implemented as sort-then-intersect: the native sort is
+     * pulled deep enough that the post-filter still yields up to K ids, then the
+     * filter bitmap is applied in order.
+     * @param filterBitmap - Candidate entity int ids.
+     * @param field - Field to sort by.
+     * @param order - 'asc' or 'desc'.
+     * @param k - Maximum number of ids to return.
+     * @returns Sorted array of entity int ids, all present in `filterBitmap`.
+     */
+    async filteredSortTopK(filterBitmap, field, order, k) {
+        if (k <= 0 || filterBitmap.size === 0)
+            return [];
+        // Intersecting only the global top-K could drop matches when the filter is
+        // selective and non-matching entities sort ahead of the matches. Sort the
+        // ENTIRE field (a u32::MAX cap returns every entry; the Rust engine clamps
+        // to the field's actual size) and intersect in sort order — correct at any
+        // selectivity. The early break keeps the scan to the first K matches.
+        // Cor 3.0 Piece E: napi returns BigInt[]. Narrow per iteration
+        // so we can use the Number version against RoaringBitmap32.
+        const sorted = this.native.sortTopK(field, order, 0xffffffff);
+        const result = [];
+        for (const idBig of sorted) {
+            const id = Number(idBig);
+            if (filterBitmap.has(id)) {
+                result.push(id);
+                if (result.length >= k)
+                    break;
+            }
+        }
+        return result;
+    }
+    /**
+     * @description All distinct values for a field, as strings in ascending order
+     * (numeric by value, string by code point), backed by the Rust engine's
+     * segment + tail scan.
+     * @param field - Field name.
+     * @returns Distinct values stringified the way brainy renders them.
+     */
+    async getFilterValues(field) {
+        if (!this.native.hasField(field))
+            return [];
+        return this.native.distinctValues(field);
+    }
+    /**
+     * @description Whether a field has any indexed data (a persisted segment or
+     * buffered tail entry) in the Rust engine.
+     * @param field - Field name.
+     * @returns True if the field is queryable.
+     */
+    hasField(field) {
+        return this.native.hasField(field);
+    }
+    /**
+     * @description Flush every field's tail buffer to a new L0 `.cidx` segment,
+     * persist the segment as a raw binary blob, update the manifest, and persist
+     * the deleted bitmap. No-op if {@link init} has not run (no storage to write).
+     */
+    async flush() {
+        if (!this.storage)
+            return;
+        // Snapshot field names: flushing drains the Rust tail buffer for a field, so
+        // iterate over a stable set (tail fields + already-persisted fields).
+        const fields = new Set([...this.fieldTypes.keys(), ...this.manifests.keys()]);
+        for (const field of fields) {
+            await this.flushField(field);
+        }
+    }
+    /**
+     * @description Flush and release all resources. Persists pending writes first.
+     */
+    async close() {
+        if (this.storage)
+            await this.flush();
+        this.manifests.clear();
+        this.fieldTypes.clear();
+        this.deletedEntities.clear();
+        this.storage = null;
+    }
+    /**
+     * @description Discard all column-store state — in-memory AND persisted — and
+     * start fresh. Called when the owning metadata index rebuilds or clears: a
+     * rebuild reassigns entity int ids from scratch, so any persisted `.cidx`
+     * segments (which encode the OLD ids) would point at the wrong entities. This
+     * drops the native engine (a fresh instance discards all tail buffers and
+     * loaded segments), clears the manifests/value-types/deleted bitmaps, and
+     * deletes every persisted file under `_column_index/`. The store remains
+     * initialized and ready: the rebuild's subsequent `addEntity` calls repopulate
+     * it consistently with the new id assignments.
+     */
+    async reset() {
+        // Delete persisted column-index files (manifests + segment/deleted blobs)
+        // before dropping the in-memory manifests we need to locate them.
+        if (this.storage) {
+            for (const [field, manifest] of this.manifests) {
+                try {
+                    await this.storage.deleteObjectFromPath(manifest.manifestPath());
+                }
+                catch {
+                    /* manifest already gone */
+                }
+                for (const seg of manifest.getAllSegments()) {
+                    await this.storage.deleteBinaryBlob(segmentBlobKey(field, seg.level, seg.id));
+                }
+                await this.storage.deleteBinaryBlob(deletedBlobKey(field));
+            }
+        }
+        // Drop all native state by replacing the engine, then clear TS-side state.
+        const bindings = loadNativeModule();
+        this.native = new bindings.NativeColumnStore();
+        this.manifests.clear();
+        this.fieldTypes.clear();
+        this.deletedEntities.clear();
+    }
+    // =========================================================================
+    // Internal helpers
+    // =========================================================================
+    /**
+     * @description Add a single value to a field, inferring + recording the value
+     * type on first write and routing to the matching Rust tail buffer.
+     * @param field - Field name.
+     * @param value - A non-null primitive value.
+     * @param entityIntId - The entity's u32 id.
+     */
+    addValue(field, value, entityIntId) {
+        let type = this.fieldTypes.get(field);
+        if (type === undefined) {
+            type = inferValueType(value);
+            this.fieldTypes.set(field, type);
+            // Ensure a manifest exists so flush has somewhere to record segments.
+            if (!this.manifests.has(field)) {
+                const manifest = new ColumnManifest(field, BASE_PATH);
+                manifest.valueType = type;
+                this.manifests.set(field, manifest);
+            }
+        }
+        // Cor 3.0 Piece E: napi expects BigInt entity ids. Wrapper
+        // bridges from u32 number for back-compat; brainy 8.0 wrapper
+        // will accept BigInt directly.
+        const idBig = BigInt(entityIntId);
+        switch (type) {
+            case ValueType.Float: {
+                const n = this.toFloat(value);
+                if (n !== undefined)
+                    this.native.addFloat(field, n, idBig);
+                break;
+            }
+            case ValueType.String:
+                this.native.addString(field, String(value), idBig);
+                break;
+            case ValueType.Number:
+            case ValueType.Boolean:
+            default: {
+                const n = this.toNumeric(value, type);
+                if (n !== undefined)
+                    this.native.addNumeric(field, n, idBig);
+                break;
+            }
+        }
+    }
+    /**
+     * @description Flush a single field: drain its Rust tail buffer to a `.cidx`
+     * segment buffer, persist it as a binary blob, register it in the manifest,
+     * then persist the manifest and (if any) the deleted bitmap.
+     * @param field - Field to flush.
+     */
+    async flushField(field) {
+        if (!this.storage)
+            return;
+        let manifest = this.manifests.get(field);
+        if (!manifest) {
+            manifest = new ColumnManifest(field, BASE_PATH);
+            const type = this.fieldTypes.get(field);
+            if (type !== undefined)
+                manifest.valueType = type;
+            this.manifests.set(field, manifest);
+        }
+        // Drain the tail buffer to a sorted segment, if it has entries.
+        if (this.native.tailBufferSize(field) > 0) {
+            const segId = manifest.nextSegmentId();
+            const entryCount = this.native.tailBufferSize(field);
+            const segBuffer = this.native.flushTailBuffer(field, 0, segId);
+            if (segBuffer) {
+                const key = segmentBlobKey(field, 0, segId);
+                await this.storage.saveBinaryBlob(key, segBuffer);
+                manifest.addSegment({
+                    id: segId,
+                    level: 0,
+                    // Number of entries drained into this segment. min/max zone-map values
+                    // are derived from the sorted segment by the Rust reader, so 0 is a
+                    // safe placeholder here (cor's metadata index does native
+                    // filtering; the column store's job is populate/persist/sort).
+                    count: entryCount,
+                    minValue: 0,
+                    maxValue: 0,
+                    file: `L0-${String(segId).padStart(6, '0')}.cidx`
+                });
+                // flushTailBuffer DRAINS the Rust tail buffer — the data is now only in
+                // the persisted segment. Reload that segment back into the engine so the
+                // field stays queryable in this same session (sortTopK/filter/range
+                // would otherwise see neither tail nor segment until a reopen). Prefer
+                // zero-copy mmap; fall back to the just-written buffer for remote
+                // adapters with no local path.
+                await this.loadSegmentIntoEngine(field, segId, 0, segBuffer);
+            }
+            else {
+                // Tail reported entries but produced no buffer — roll back the id so the
+                // manifest stays consistent. (Defensive; flushTailBuffer only returns
+                // null for an empty tail.)
+                manifest.rollbackSegmentId();
+            }
+        }
+        // Persist the deleted bitmap alongside the manifest, so tombstones survive.
+        const deleted = this.deletedEntities.get(field);
+        if (deleted && deleted.size > 0) {
+            await this.storage.saveBinaryBlob(deletedBlobKey(field), deleted.serialize('portable'));
+        }
+        await manifest.save(this.storage);
+    }
+    /**
+     * @description Load all of a field's persisted segments into the Rust engine.
+     * Used on init, where no segment buffer is held in hand, so each segment is
+     * loaded by mmap path or read from storage.
+     * @param field - Field name.
+     * @param manifest - The field's loaded manifest.
+     */
+    async loadFieldSegments(field, manifest) {
+        if (!this.storage)
+            return;
+        for (const seg of manifest.getAllSegments()) {
+            await this.loadSegmentIntoEngine(field, seg.id, seg.level);
+        }
+    }
+    /**
+     * @description Load one persisted segment into the Rust engine, preferring
+     * zero-copy mmap (`loadSegmentFromPath`) when the adapter exposes a local
+     * path. Falls back to `fallbackBuffer` if provided (the just-flushed bytes),
+     * otherwise reads the blob from storage. Used both on init (reload all
+     * segments) and on flush (re-attach the segment a drained tail just produced).
+     * @param field - Field name.
+     * @param segId - Segment id.
+     * @param level - Segment compaction level.
+     * @param fallbackBuffer - Segment bytes already in memory (flush path), if any.
+     */
+    async loadSegmentIntoEngine(field, segId, level, fallbackBuffer) {
+        if (!this.storage)
+            return;
+        const key = segmentBlobKey(field, level, segId);
+        const path = this.storage.getBinaryBlobPath(key);
+        // Only mmap when the raw-blob file actually EXISTS. `getBinaryBlobPath`
+        // returns a path string whether or not the file is present, and
+        // `loadSegmentFromPath` does NOT fault on a missing file — it loads a
+        // silently-empty segment. Guard so a missing raw blob falls through to the
+        // buffer path below instead of "succeeding" empty.
+        if (path && existsSync(path)) {
+            try {
+                this.native.loadSegmentFromPath(field, String(segId), path);
+                return;
+            }
+            catch (error) {
+                // Path existed but the file was not mmap-able — e.g. a remote adapter
+                // returning a synthetic path. Fall through to the buffer / legacy path.
+                prodLog.debug(`NativeColumnStore: mmap load failed for ${key}, trying buffer`, error);
+            }
+        }
+        const buf = fallbackBuffer ?? (await this.storage.loadBinaryBlob(key));
+        if (buf) {
+            this.native.loadSegmentFromBuffer(field, String(segId), buf);
+            return;
+        }
+        // No raw blob for this segment. A 2.x→3.0 upgrade does NOT read legacy
+        // column envelopes in place — it rebuilds the index from entities (the
+        // clean-break upgrade path), so a missing raw blob here is simply an
+        // absent segment, not a format to migrate.
+        prodLog.warn(`NativeColumnStore: segment blob missing for ${key}`);
+    }
+    /**
+     * @description Load and re-apply a field's persisted deleted bitmap by calling
+     * `deleteEntity` for each id, so tombstones survive a reopen.
+     * @param field - Field name.
+     */
+    async loadDeletedBitmap(field) {
+        if (!this.storage)
+            return;
+        const buf = await this.storage.loadBinaryBlob(deletedBlobKey(field));
+        if (!buf)
+            return;
+        const bitmap = RoaringBitmap32.deserialize(buf, 'portable');
+        this.deletedEntities.set(field, bitmap);
+        for (const id of bitmap) {
+            // Cor 3.0 Piece E: BigInt at the napi boundary.
+            this.native.deleteEntity(field, BigInt(id));
+        }
+    }
+    /**
+     * @description Coerce a JS value to the i64 numeric column representation,
+     * matching brainy's `normalizeValue` for `Number`/`Boolean`.
+     * @param value - The raw value.
+     * @param type - The target value type (`Number` or `Boolean`).
+     * @returns The integer value, or undefined if not coercible.
+     */
+    toNumeric(value, type) {
+        if (type === ValueType.Boolean) {
+            if (typeof value === 'boolean')
+                return value ? 1 : 0;
+            if (typeof value === 'number')
+                return value ? 1 : 0;
+            return undefined;
+        }
+        // ValueType.Number
+        if (typeof value === 'number')
+            return Math.round(value);
+        if (typeof value === 'boolean')
+            return value ? 1 : 0;
+        if (typeof value === 'string') {
+            const n = Number(value);
+            return Number.isNaN(n) ? undefined : Math.round(n);
+        }
+        return undefined;
+    }
+    /**
+     * @description Coerce a JS value to the f64 float column representation,
+     * matching brainy's `normalizeValue` for `Float`.
+     * @param value - The raw value.
+     * @returns The float value, or undefined if not coercible.
+     */
+    toFloat(value) {
+        if (typeof value === 'number')
+            return value;
+        if (typeof value === 'string') {
+            const n = Number(value);
+            return Number.isNaN(n) ? undefined : n;
+        }
+        return undefined;
+    }
+}
+/**
+ * @description Infer the column `ValueType` from a JS value, identical to
+ * brainy's `ColumnStore.inferValueType`: boolean → Boolean; integer number →
+ * Number; non-integer number → Float; everything else → String.
+ * @param value - A non-null value.
+ * @returns The inferred value type.
+ */
+export function inferValueType(value) {
+    if (typeof value === 'boolean')
+        return ValueType.Boolean;
+    if (typeof value === 'number') {
+        return Number.isInteger(value) ? ValueType.Number : ValueType.Float;
+    }
+    return ValueType.String;
+}
+//# sourceMappingURL=NativeColumnStore.js.map