@soulcraft/cortex 2.7.0 → 2.7.1

package/dist/hnsw/NativeHNSWWrapper.d.ts

@@ -37,6 +37,13 @@ export declare class NativeHNSWWrapper implements HnswProvider {
         storage?: StorageAdapter | null;
         persistMode?: 'immediate' | 'deferred';
     });
+    /**
+     * Resolve the storage root directory for the `_hnsw.bin` mmap file. brainy
+     * >= 7.31.8 exposes a public `rootDirectory` getter; earlier 7.x kept the
+     * value in the (compile-time-protected, runtime-present) `rootDir` field.
+     * Accept either so the mmap fast-path engages on every filesystem brainy.
+     */
+    private resolveRootDir;
     addItem(item: VectorDocument): Promise<string>;
     /**
      * Batch insert — inserts all items in a single native call, then persists

package/dist/hnsw/NativeHNSWWrapper.js

@@ -56,8 +56,11 @@ export class NativeHNSWWrapper {
             ml: this.config.ml,
         };
         this.native = new bindings.NativeHNSWIndex(nativeConfig);
-        // Initialize mmap binary store if storage has a rootDirectory (filesystem storage)
-        const rootDir = this.storage?.rootDirectory;
+        // Initialize mmap binary store if storage exposes its root directory
+        // (filesystem storage). Accept brainy >= 7.31.8's `rootDirectory` getter
+        // OR older 7.x's `rootDir` field, so the mmap fast-path engages on every
+        // filesystem deployment — not only the newest brainy.
+        const rootDir = this.resolveRootDir();
         if (rootDir && bindings.MmapHnswStore) {
             const { join } = require('node:path');
             this.mmapStore = new bindings.MmapHnswStore(join(rootDir, '_hnsw.bin'));
@@ -65,6 +68,16 @@ export class NativeHNSWWrapper {
         prodLog.info(`NativeHNSWWrapper initialized (M=${this.config.M}, ef=${this.config.efSearch}, ` +
             `persist=${this.persistMode}, mmap=${!!this.mmapStore})`);
     }
+    /**
+     * Resolve the storage root directory for the `_hnsw.bin` mmap file. brainy
+     * >= 7.31.8 exposes a public `rootDirectory` getter; earlier 7.x kept the
+     * value in the (compile-time-protected, runtime-present) `rootDir` field.
+     * Accept either so the mmap fast-path engages on every filesystem brainy.
+     */
+    resolveRootDir() {
+        const s = this.storage;
+        return s?.rootDirectory ?? s?.rootDir ?? undefined;
+    }
     // ---------------------------------------------------------------------------
     // Core CRUD
     // ---------------------------------------------------------------------------
@@ -227,7 +240,14 @@ export class NativeHNSWWrapper {
     async flush() {
         if (!this.storage)
             return 0;
-        if (this.dirtyNodes.size === 0 && !this.dirtySystem)
+        // The mmap binary is a FULL snapshot of the in-memory graph, independent of
+        // the per-node dirty set. In 'immediate' mode (brainy's filesystem default)
+        // nothing is ever marked dirty — nodes persist inline on add — so flush()'s
+        // real job there is to write that snapshot. It must NOT be skipped just
+        // because the dirty set is empty; only fast-return when there is genuinely
+        // nothing to write (no dirty data AND no graph to snapshot).
+        const needsMmapSnapshot = !!this.mmapStore && this.native.size() > 0;
+        if (this.dirtyNodes.size === 0 && !this.dirtySystem && !needsMmapSnapshot)
             return 0;
         const startTime = Date.now();
         const nodeCount = this.dirtyNodes.size;
@@ -264,7 +284,7 @@ export class NativeHNSWWrapper {
             try {
                 this.writeMmapFile();
                 // Switch to mmap backend — search now reads from kernel page cache
-                const rootDir = this.storage?.rootDirectory;
+                const rootDir = this.resolveRootDir();
                 if (rootDir) {
                     const { join } = require('node:path');
                     this.native.setMmapBackend(join(rootDir, '_hnsw.bin'));
@@ -350,11 +370,9 @@ export class NativeHNSWWrapper {
     loadFromMmap() {
         if (!this.mmapStore)
             return false;
-        const mmapPath = this.mmapStore.path ?? this.mmapStore.path;
         // Use the native index's mmap backend directly
-        const loaded = this.native.setMmapBackend(this.storage?.rootDirectory
-            ? require('node:path').join(this.storage.rootDirectory, '_hnsw.bin')
-            : '');
+        const rootDir = this.resolveRootDir();
+        const loaded = this.native.setMmapBackend(rootDir ? require('node:path').join(rootDir, '_hnsw.bin') : '');
         if (!loaded)
             return false;
         const size = this.native.size();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/cortex",
-  "version": "2.7.0",
+  "version": "2.7.1",
   "description": "Native Rust acceleration for Brainy — SIMD distance, vector quantization, zero-copy mmap, native embeddings. Free tier for storage, Pro license for compute acceleration.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/dist/hnsw/NativeDiskAnnWrapper.d.ts

@@ -1,161 +0,0 @@
-/**
- * @module hnsw/NativeDiskAnnWrapper
- * @description TypeScript wrapper around cortex's native DiskANN engine
- * that satisfies brainy's `HnswProvider` contract. From brainy's
- * perspective this is interchangeable with `NativeHNSWWrapper` — same
- * `addItem` / `search` / `rebuild` surface — but underneath it drives
- * the billion-scale Vamana + PQ index.
- *
- * @example
- * ```typescript
- * import { BrainyData } from '@soulcraft/brainy'
- * import { register as registerCortex } from '@soulcraft/cortex'
- *
- * const brain = new BrainyData({
- *   storage: { type: 'filesystem', rootDirectory: '/data/idx' }
- * })
- * await registerCortex(brain)
- * await brain.init()  // [brainy] DiskANN engaged (path=..., dim=384)
- *
- * await brain.add({ data: 'native rust acceleration', type: 'concept' })
- * const hits = await brain.search('billion scale ann', 10)
- * ```
- *
- * @example
- * ```typescript
- * // Explicit billion-scale build config
- * const brain = new BrainyData({
- *   storage: { type: 'filesystem', rootDirectory: '/data/idx' },
- *   index: {
- *     type: 'diskann',
- *     diskann: {
- *       pqM: 16,
- *       maxDegree: 64,
- *       searchListSize: 100,
- *       useMmapAdjacency: true,  // required >100M nodes
- *       mmapAdjacencyPath: '/data/scratch/diskann-build.adj'
- *     }
- *   }
- * })
- * ```
- *
- * ## Operating model
- *
- * DiskANN is build-once, query-many by design: the on-disk file
- * embeds the Vamana graph, PQ codebook, codes, and full vectors in a
- * single contiguous mmap-able layout. Dynamic insertions go to a
- * small **delta buffer** that brute-force-searches alongside the main
- * index until the next `rebuild()` folds them in. This matches
- * FreshDiskANN's published online-update model.
- *
- * ## Search path
- *
- * 1. Query the main index via the native DiskANN searcher: PQ-greedy
- *    walk in RAM, full-vector re-rank on the candidate set.
- * 2. Brute-force the delta buffer (typically <0.1% of total size after
- *    a recent rebuild).
- * 3. Merge + sort + truncate to `k`.
- *
- * ## When this wrapper engages
- *
- * Brainy's `wireDiskAnn()` decides at init time whether to instantiate
- * this wrapper or the standard HNSW one. The criteria
- * ([ADR-002](../../docs/ADR-002-diskann-100-percent-rust.md)):
- * - Cortex's `index:diskann` provider is registered (this file).
- * - The storage adapter exposes a local filesystem path
- *   (`getBinaryBlobPath` is the canonical check).
- * - The metadata index has a stable `idMapper` (the cortex 2.4.0 #23
- *   foundation).
- * - `config.index.type !== 'hnsw'` (opt-out path).
- */
-import type { Vector, VectorDocument, DistanceFunction, StorageAdapter } from '@soulcraft/brainy';
-import type { HnswProvider } from '../providerContracts.js';
-export interface DiskAnnIndexConfig {
-    /** Vector dimension (e.g. 384 for all-MiniLM-L6-v2). */
-    dimensions: number;
-    /** Output path for the on-disk DiskANN file. */
-    indexPath: string;
-    /** PQ subspaces. Default 16. dim must be divisible by m. */
-    pqM?: number;
-    /** Centroids per subspace. Default 256 (8-bit codes). */
-    pqKsub?: number;
-    /** Vamana max degree (R). Default 64. */
-    maxDegree?: number;
-    /** Build-time candidate list size (L). Default 100. */
-    searchListSize?: number;
-    /** α-pruning density factor. Default 1.2. */
-    alpha?: number;
-    /** Default search-time candidate list size. `2*k` is a good baseline. */
-    defaultLSearch?: number;
-    /** Default padding factor for re-rank over-fetch. Default 1.2. */
-    defaultPaddingFactor?: number;
-    /** Use a file-backed adjacency during build. Required >~100M nodes. */
-    useMmapAdjacency?: boolean;
-    /** Scratch file path when `useMmapAdjacency` is true. */
-    mmapAdjacencyPath?: string;
-}
-export declare class NativeDiskAnnWrapper implements HnswProvider {
-    private config;
-    private distanceFunction;
-    private storage;
-    private persistMode;
-    /** Live searcher instance — null until the first build. */
-    private native;
-    /** Newly added entries since the last build. Brute-force searched. */
-    private delta;
-    /** Removed entries — filtered out at search time. */
-    private tombstones;
-    /** Bidirectional UUID ↔ slot map for the main index. */
-    private slotByUuid;
-    private uuidBySlot;
-    constructor(config: DiskAnnIndexConfig & {
-        distanceFunction?: DistanceFunction;
-    }, distanceFunction: DistanceFunction, options?: {
-        storage?: StorageAdapter | null;
-        persistMode?: 'immediate' | 'deferred';
-    });
-    /**
-     * Append an entry to the delta buffer. Persisted by the next
-     * `rebuild()` call, which folds the delta into the main index.
-     */
-    addItem(item: VectorDocument): Promise<string>;
-    /**
-     * Mark an entry as removed. Filtered out at search time; physically
-     * removed at the next `rebuild()`.
-     */
-    removeItem(id: string): Promise<boolean>;
-    search(queryVector: Vector, k?: number, filter?: (id: string) => Promise<boolean>, options?: {
-        rerank?: {
-            multiplier: number;
-        };
-        candidateIds?: string[];
-    }): Promise<Array<[string, number]>>;
-    size(): number;
-    clear(): void;
-    /**
-     * Rebuild the main index from scratch: concatenate (current main −
-     * tombstones) ∪ delta, run a full DiskANN build, swap the searcher
-     * atomically.
-     *
-     * At billion-scale this is the expensive operation (hours of build
-     * time). Operators schedule it during off-peak; the delta buffer
-     * absorbs writes in between.
-     */
-    rebuild(options?: {
-        pqM?: number;
-        pqKsub?: number;
-        maxDegree?: number;
-        searchListSize?: number;
-        alpha?: number;
-    }): Promise<void>;
-    /**
-     * Flush the delta buffer to disk. For DiskANN the delta is in-memory
-     * by design (a few MB at most between rebuilds); returns the buffer
-     * size for parity with HNSW's flush contract.
-     */
-    flush(): Promise<number>;
-    getPersistMode(): 'immediate' | 'deferred';
-    private tryOpenExisting;
-    private countMainTombstones;
-}
-//# sourceMappingURL=NativeDiskAnnWrapper.d.ts.map

package/dist/hnsw/NativeDiskAnnWrapper.js

@@ -1,329 +0,0 @@
-/**
- * @module hnsw/NativeDiskAnnWrapper
- * @description TypeScript wrapper around cortex's native DiskANN engine
- * that satisfies brainy's `HnswProvider` contract. From brainy's
- * perspective this is interchangeable with `NativeHNSWWrapper` — same
- * `addItem` / `search` / `rebuild` surface — but underneath it drives
- * the billion-scale Vamana + PQ index.
- *
- * @example
- * ```typescript
- * import { BrainyData } from '@soulcraft/brainy'
- * import { register as registerCortex } from '@soulcraft/cortex'
- *
- * const brain = new BrainyData({
- *   storage: { type: 'filesystem', rootDirectory: '/data/idx' }
- * })
- * await registerCortex(brain)
- * await brain.init()  // [brainy] DiskANN engaged (path=..., dim=384)
- *
- * await brain.add({ data: 'native rust acceleration', type: 'concept' })
- * const hits = await brain.search('billion scale ann', 10)
- * ```
- *
- * @example
- * ```typescript
- * // Explicit billion-scale build config
- * const brain = new BrainyData({
- *   storage: { type: 'filesystem', rootDirectory: '/data/idx' },
- *   index: {
- *     type: 'diskann',
- *     diskann: {
- *       pqM: 16,
- *       maxDegree: 64,
- *       searchListSize: 100,
- *       useMmapAdjacency: true,  // required >100M nodes
- *       mmapAdjacencyPath: '/data/scratch/diskann-build.adj'
- *     }
- *   }
- * })
- * ```
- *
- * ## Operating model
- *
- * DiskANN is build-once, query-many by design: the on-disk file
- * embeds the Vamana graph, PQ codebook, codes, and full vectors in a
- * single contiguous mmap-able layout. Dynamic insertions go to a
- * small **delta buffer** that brute-force-searches alongside the main
- * index until the next `rebuild()` folds them in. This matches
- * FreshDiskANN's published online-update model.
- *
- * ## Search path
- *
- * 1. Query the main index via the native DiskANN searcher: PQ-greedy
- *    walk in RAM, full-vector re-rank on the candidate set.
- * 2. Brute-force the delta buffer (typically <0.1% of total size after
- *    a recent rebuild).
- * 3. Merge + sort + truncate to `k`.
- *
- * ## When this wrapper engages
- *
- * Brainy's `wireDiskAnn()` decides at init time whether to instantiate
- * this wrapper or the standard HNSW one. The criteria
- * ([ADR-002](../../docs/ADR-002-diskann-100-percent-rust.md)):
- * - Cortex's `index:diskann` provider is registered (this file).
- * - The storage adapter exposes a local filesystem path
- *   (`getBinaryBlobPath` is the canonical check).
- * - The metadata index has a stable `idMapper` (the cortex 2.4.0 #23
- *   foundation).
- * - `config.index.type !== 'hnsw'` (opt-out path).
- */
-import { loadNativeModule } from '../native/index.js';
-import { prodLog } from '@soulcraft/brainy/internals';
-const DEFAULTS = {
-    pqM: 16,
-    pqKsub: 256,
-    maxDegree: 64,
-    searchListSize: 100,
-    alpha: 1.2,
-    defaultLSearch: 100,
-    defaultPaddingFactor: 1.2,
-    useMmapAdjacency: false,
-};
-export class NativeDiskAnnWrapper {
-    config;
-    distanceFunction;
-    storage;
-    persistMode;
-    /** Live searcher instance — null until the first build. */
-    native = null;
-    /** Newly added entries since the last build. Brute-force searched. */
-    delta = new Map();
-    /** Removed entries — filtered out at search time. */
-    tombstones = new Set();
-    /** Bidirectional UUID ↔ slot map for the main index. */
-    slotByUuid = new Map();
-    uuidBySlot = new Map();
-    constructor(config, distanceFunction, options = {}) {
-        this.config = { ...DEFAULTS, ...config };
-        this.distanceFunction = distanceFunction;
-        this.storage = options.storage ?? null;
-        this.persistMode = options.persistMode ?? 'immediate';
-        // Try to open an existing file. If absent, the index stays
-        // empty until the first rebuild() flushes the delta buffer.
-        this.tryOpenExisting();
-    }
-    /**
-     * Append an entry to the delta buffer. Persisted by the next
-     * `rebuild()` call, which folds the delta into the main index.
-     */
-    async addItem(item) {
-        if (this.tombstones.has(item.id)) {
-            this.tombstones.delete(item.id);
-        }
-        this.delta.set(item.id, item.vector);
-        return item.id;
-    }
-    /**
-     * Mark an entry as removed. Filtered out at search time; physically
-     * removed at the next `rebuild()`.
-     */
-    async removeItem(id) {
-        const inDelta = this.delta.delete(id);
-        const inMain = this.slotByUuid.has(id);
-        if (inMain)
-            this.tombstones.add(id);
-        return inDelta || inMain;
-    }
-    async search(queryVector, k = 10, filter, options) {
-        const lSearch = Math.max(this.config.defaultLSearch, k * 2);
-        const padding = options?.rerank?.multiplier ?? this.config.defaultPaddingFactor;
-        // 1. Main-index PQ-greedy walk (returns slot ids).
-        const mainHits = this.native
-            ? this.native.search(Array.from(queryVector), k * 2, // over-fetch so filter / tombstone losses don't starve final result
-            lSearch, padding)
-            : [];
-        // 2. Hydrate slot → uuid; drop tombstoned + filter-rejected.
-        const merged = [];
-        for (const hit of mainHits) {
-            const uuid = this.uuidBySlot.get(hit.slot);
-            if (!uuid)
-                continue;
-            if (this.tombstones.has(uuid))
-                continue;
-            if (filter && !(await filter(uuid)))
-                continue;
-            merged.push([uuid, hit.distance]);
-        }
-        // 3. Brute-force the delta buffer.
-        for (const [id, v] of this.delta) {
-            if (filter && !(await filter(id)))
-                continue;
-            const d = this.distanceFunction(queryVector, v);
-            merged.push([id, d]);
-        }
-        // 4. Sort ascending by distance, truncate to k.
-        merged.sort((a, b) => a[1] - b[1]);
-        return merged.slice(0, k);
-    }
-    size() {
-        const mainSize = this.native ? this.native.size() : 0;
-        return (mainSize +
-            this.delta.size -
-            // Tombstones from the main index reduce effective size.
-            this.countMainTombstones());
-    }
-    clear() {
-        this.delta.clear();
-        this.tombstones.clear();
-        this.slotByUuid.clear();
-        this.uuidBySlot.clear();
-        this.native = null;
-    }
-    /**
-     * Rebuild the main index from scratch: concatenate (current main −
-     * tombstones) ∪ delta, run a full DiskANN build, swap the searcher
-     * atomically.
-     *
-     * At billion-scale this is the expensive operation (hours of build
-     * time). Operators schedule it during off-peak; the delta buffer
-     * absorbs writes in between.
-     */
-    async rebuild(options) {
-        const bindings = loadNativeModule();
-        // napi-rs exports the class as `NativeDiskAnn` (PascalCase
-        // normalization of the Rust ident `NativeDiskANN`). The TS type
-        // alias `NativeDiskANN = NativeDiskAnn` in `native/index.d.ts` is
-        // for backwards-compat in *types* only — at runtime there's a
-        // single export under the napi-normalized name.
-        const NativeDiskANN = bindings.NativeDiskAnn;
-        if (!NativeDiskANN) {
-            throw new Error('NativeDiskANN binding missing — rebuild requires the cortex native module');
-        }
-        // Build the new logical slot ordering: (live old slots) + (delta).
-        // **Critical for billion-scale correctness**: the old vectors stay
-        // mmap'd inside the native module — we only pass slot IDs across
-        // the FFI boundary, not the vector data itself. At 1B × 1536 × 4
-        // bytes = ~6 TB this is the difference between "rebuild works" and
-        // "rebuild OOMs."
-        const liveOldSlots = [];
-        const newUuids = [];
-        if (this.native) {
-            // Iterate in slot order so the new index's first n_live slots
-            // mirror the OLD index's surviving subset in deterministic order.
-            // We deliberately iterate by sorted slot id rather than uuidBySlot
-            // insertion order — sorting keeps the Vamana entry point stable
-            // and the on-disk vector section's locality similar to the
-            // pre-rebuild file (less page-cache turnover during the post-
-            // rebuild warm-up).
-            const sortedSlots = Array.from(this.uuidBySlot.keys()).sort((a, b) => a - b);
-            for (const slot of sortedSlots) {
-                const uuid = this.uuidBySlot.get(slot);
-                if (this.tombstones.has(uuid))
-                    continue;
-                liveOldSlots.push(slot);
-                newUuids.push(uuid);
-            }
-        }
-        const dim = this.config.dimensions;
-        const deltaCount = this.delta.size;
-        let deltaBuf = null;
-        if (deltaCount > 0) {
-            deltaBuf = new Float32Array(deltaCount * dim);
-            let idx = 0;
-            for (const [uuid, vector] of this.delta) {
-                if (vector.length !== dim) {
-                    throw new Error(`NativeDiskAnnWrapper.rebuild: vector dim ${vector.length} ≠ index dim ${dim}`);
-                }
-                deltaBuf.set(vector, idx * dim);
-                newUuids.push(uuid);
-                idx++;
-            }
-        }
-        if (liveOldSlots.length + deltaCount === 0) {
-            prodLog?.warn?.('NativeDiskAnnWrapper.rebuild: nothing to build');
-            return;
-        }
-        const totalCount = liveOldSlots.length + deltaCount;
-        const cfg = {
-            vamana: {
-                maxDegree: options?.maxDegree ?? this.config.maxDegree,
-                searchListSize: options?.searchListSize ?? this.config.searchListSize,
-                alpha: options?.alpha ?? this.config.alpha,
-                seed: BigInt(0xd15ca4440ffff00dn),
-                parallel: true,
-                parallelBatch: 64,
-            },
-            pq: {
-                m: options?.pqM ?? this.config.pqM,
-                ksub: options?.pqKsub ?? this.config.pqKsub,
-                iterations: 25,
-                trainingSample: Math.min(200_000, totalCount),
-            },
-            adjacency: this.config.useMmapAdjacency
-                ? {
-                    kind: 'mmap',
-                    mmapPath: this.config.mmapAdjacencyPath ?? `${this.config.indexPath}.adj`,
-                }
-                : { kind: 'ram' },
-        };
-        const newNative = NativeDiskANN.rebuildFromExisting({
-            existingPath: this.native ? this.config.indexPath : undefined,
-            liveOldSlots,
-            deltaVectors: deltaBuf != null
-                ? Buffer.from(deltaBuf.buffer, deltaBuf.byteOffset, deltaBuf.byteLength)
-                : undefined,
-            deltaCount,
-            dim,
-            outputPath: this.config.indexPath,
-            cfg,
-        });
-        // Rebuild the bidirectional UUID↔slot maps from `newUuids`. New
-        // slot `i` corresponds to `newUuids[i]` — this matches the napi
-        // layout invariant (live old slots first, delta tail second).
-        const newSlotByUuid = new Map();
-        const newUuidBySlot = new Map();
-        for (let i = 0; i < newUuids.length; i++) {
-            newSlotByUuid.set(newUuids[i], i);
-            newUuidBySlot.set(i, newUuids[i]);
-        }
-        // Atomic swap.
-        this.native = newNative;
-        this.slotByUuid = newSlotByUuid;
-        this.uuidBySlot = newUuidBySlot;
-        this.delta.clear();
-        this.tombstones.clear();
-    }
-    /**
-     * Flush the delta buffer to disk. For DiskANN the delta is in-memory
-     * by design (a few MB at most between rebuilds); returns the buffer
-     * size for parity with HNSW's flush contract.
-     */
-    async flush() {
-        return this.delta.size;
-    }
-    getPersistMode() {
-        return this.persistMode;
-    }
-    tryOpenExisting() {
-        try {
-            const bindings = loadNativeModule();
-            // napi-rs exports the class as `NativeDiskAnn` (PascalCase
-            // normalization of the Rust ident `NativeDiskANN`). The TS type
-            // alias `NativeDiskANN = NativeDiskAnn` in `native/index.d.ts` is
-            // for backwards-compat in *types* only — at runtime there's a
-            // single export under the napi-normalized name.
-            const NativeDiskANN = bindings.NativeDiskAnn;
-            if (!NativeDiskANN)
-                return;
-            this.native = NativeDiskANN.openExisting(this.config.indexPath);
-            // Populate slot maps from the storage adapter — these are persisted
-            // alongside the index file in production. For 35c we read from a
-            // sibling `.slots.json` that rebuild() writes.
-            // (Stub for now; the real path lands when storage integration ships.)
-        }
-        catch {
-            // No existing file — index stays empty until first rebuild().
-            this.native = null;
-        }
-    }
-    countMainTombstones() {
-        let n = 0;
-        for (const uuid of this.tombstones) {
-            if (this.slotByUuid.has(uuid))
-                n++;
-        }
-        return n;
-    }
-}
-//# sourceMappingURL=NativeDiskAnnWrapper.js.map

package/dist/utils/nativeBinaryEntityIdMapper.d.ts

@@ -1,194 +0,0 @@
-/**
- * @module utils/nativeBinaryEntityIdMapper
- * @description TypeScript wrapper around cortex's native binary
- * `BinaryIdMapper`. Implements brainy's `EntityIdMapperProvider` so the
- * mmap-backed billion-scale mapper is a drop-in for the existing
- * JSON-persisted one.
- *
- * ## When this engages
- *
- * The cortex plugin registers this wrapper as the `'entityIdMapper'`
- * provider when the storage adapter exposes `getBinaryBlobPath()` (i.e.
- * filesystem-backed storage with cortex's 2.4.0 #2 mmap-vector layer).
- * Cloud-storage adapters fall back to the JSON variant
- * (`NativeEntityIdMapperWrapper`) since they have no local-path concept.
- *
- * ## UUID format conversion
- *
- * Brainy passes UUIDs as strings (typically the canonical 36-char
- * `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`). The native side works in
- * 16-byte Buffers. This wrapper converts at the boundary. Non-canonical
- * UUID strings (any other 32-hex-digit form) are also accepted.
- *
- * ## Concurrency
- *
- * `getOrAssign` is atomic across concurrent callers for the same UUID
- * (256 sharded per-UUID mutexes in the native layer). Lookups are
- * lock-free. The wrapper holds no JS-side mutable state besides the
- * native handle.
- *
- * ## IdSpace (Piece 10)
- *
- * The wrapper supports two entity-int wire widths:
- *
- * - `'u32'` (default): cortex 2.x compatible — JS `number` throughout,
- *   capped at 4.29 B entities. Persists in the legacy `int_to_uuid.bin`
- *   v1 header.
- * - `'u64'`: opt-in via `idSpace: 'u64'`. The native layer's `number`
- *   surface throws in this mode, so the wrapper transparently routes
- *   the `EntityIdMapperProvider` methods through the BigInt napi
- *   siblings and converts BigInt → number at the boundary. Entity ints
- *   above `Number.MAX_SAFE_INTEGER` (2^53 - 1 = ~9 PB of entities)
- *   throw a clear `EntityIdSpaceExceeded` error so callers get a loud
- *   failure rather than a silent precision loss.
- *
- * The `getOrAssignBig` / `getIntBig` / `getUuidBig` / `sizeBig`
- * sibling methods are available on both modes and always return
- * `bigint` losslessly — use them in u64-aware code paths that need
- * the full u64 range.
- */
-import type { StorageAdapter } from '@soulcraft/brainy';
-import type { EntityIdMapperProvider } from '../providerContracts.js';
-export interface NativeBinaryEntityIdMapperOptions {
-    /** Storage adapter — required for binary blob path resolution. */
-    storage: StorageAdapter;
-    /**
-     * Override the relative path under storage for the uuid_to_int file.
-     * Default `_id_mapper/uuid_to_int.mkv`.
-     */
-    uuidToIntKey?: string;
-    /**
-     * Override the relative path under storage for the int_to_uuid file.
-     * Default `_id_mapper/int_to_uuid.bin`.
-     */
-    intToUuidKey?: string;
-    /** Sparse file size for int_to_uuid. Default 32 GB. */
-    intToUuidSize?: bigint;
-    /** Sparse file size for uuid_to_int. Default 32 GB. */
-    uuidToIntSize?: bigint;
-    /** Bucket capacity in the MmapKv. Default 16. */
-    bucketCapacity?: number;
-    /** Maximum extendible-hash directory depth. Default 28. */
-    maxGlobalDepth?: number;
-    /**
-     * Entity-int wire width. `'u32'` (default) caps at 4.29 B entities
-     * and is cortex 2.x compatible. `'u64'` opts into the Piece 10 U64
-     * IdSpace — required when targeting >4.29 B entities. The
-     * `EntityIdMapperProvider` `number`-typed methods still work in U64
-     * mode by routing through the BigInt napi siblings; entity ints
-     * above `Number.MAX_SAFE_INTEGER` (2^53 - 1) throw an explicit
-     * `EntityIdSpaceExceeded` error.
-     *
-     * Ignored on open when the underlying file's header disagrees: the
-     * on-disk format wins and any mismatch is surfaced as a hard error.
-     */
-    idSpace?: 'u32' | 'u64';
-}
-/**
- * Thrown when a U64-mode mapper allocates or returns an entity int
- * above `Number.MAX_SAFE_INTEGER` (2^53 - 1). At this point a JS
- * `number` can no longer represent the value losslessly; callers must
- * switch to the BigInt sibling methods (`getOrAssignBig`,
- * `getIntBig`, `getUuidBig`).
- */
-export declare class EntityIdSpaceExceeded extends Error {
-    /** The u64 entity int that exceeded the safe-integer ceiling. */
-    readonly value: bigint;
-    /** The method that was called (`'getOrAssign'`, `'getInt'`, etc.). */
-    readonly method: string;
-    constructor(method: string, value: bigint);
-}
-/**
- * Drop-in `EntityIdMapperProvider` backed by the native `BinaryIdMapper`.
- *
- * @example
- * ```typescript
- * const mapper = new NativeBinaryEntityIdMapperWrapper({ storage })
- * await mapper.init()
- * const intId = mapper.getOrAssign('12345678-1234-5678-1234-567812345678')
- * const uuid = mapper.getUuid(intId)
- * ```
- */
-export declare class NativeBinaryEntityIdMapperWrapper implements EntityIdMapperProvider {
-    private storage;
-    private uuidToIntKey;
-    private intToUuidKey;
-    private intToUuidSize;
-    private uuidToIntSize;
-    private bucketCapacity;
-    private maxGlobalDepth;
-    private requestedIdSpace;
-    /**
-     * The actual IdSpace of the open mapper, sourced from the native
-     * binding's `idSpace()` reflection after `init()`. The on-disk
-     * header wins over `requestedIdSpace` (which may be ignored at
-     * `openExisting` time).
-     */
-    private resolvedIdSpace;
-    private native;
-    private initialized;
-    constructor(options: NativeBinaryEntityIdMapperOptions);
-    init(): Promise<void>;
-    /**
-     * Report the mapper's actual IdSpace mode. Returns `'u32'` before
-     * `init()` (the default the wrapper assumes); after init, returns the
-     * mode reported by the native binding (which is authoritative).
-     */
-    getIdSpace(): 'u32' | 'u64';
-    /**
-     * Allocate or retrieve the entity int for `uuid`. Returns a JS
-     * `number`. In U64 mode, routes through the BigInt sibling and
-     * throws {@link EntityIdSpaceExceeded} if the allocated int exceeds
-     * `Number.MAX_SAFE_INTEGER` — at that point the caller MUST switch
-     * to `getOrAssignBig` for the full u64 range.
-     */
-    getOrAssign(uuid: string): number;
-    /**
-     * Look up the UUID for `intId`. Accepts a JS `number` — in U64 mode
-     * this is a lossy conversion above 2^53; use {@link getUuidBig} for
-     * the full u64 range.
-     */
-    getUuid(intId: number): string | undefined;
-    /**
-     * Look up the entity int for `uuid`. Returns a JS `number`. In U64
-     * mode throws {@link EntityIdSpaceExceeded} if the int exceeds
-     * `Number.MAX_SAFE_INTEGER`.
-     */
-    getInt(uuid: string): number | undefined;
-    remove(uuid: string): boolean;
-    flush(): Promise<void>;
-    clear(): Promise<void>;
-    /**
-     * Materialise every live int id into a JS `number[]`. **U32 mode
-     * only.** U64 mode throws — the native binding refuses to allocate
-     * a giant JS array at the scale a U64 brain implies. Iterate via
-     * the BigInt sibling iterator (TBD — a follow-up surfaces it on the
-     * wrapper) for U64 brains.
-     */
-    getAllIntIds(): number[];
-    intsIterableToUuids(ints: Iterable<number>): string[];
-    get size(): number;
-    /**
-     * Allocate or retrieve the entity int for `uuid` as a `bigint`.
-     * Lossless across the full u64 range; safe to call in either mode.
-     */
-    getOrAssignBig(uuid: string): bigint;
-    /** Look up the entity int for `uuid` as a `bigint`. */
-    getIntBig(uuid: string): bigint | undefined;
-    /** Look up the UUID for `int` (passed as a `bigint`). */
-    getUuidBig(int: bigint): string | undefined;
-    /** Live (non-tombstone) entry count as a `bigint`. */
-    sizeBig(): bigint;
-    /** Largest int ever assigned + 1, as a `bigint`. */
-    nextIntBig(): bigint;
-    /**
-     * Encode a UUID string into a 16-byte Buffer. Accepts canonical
-     * 36-char form (with hyphens) or any 32-hex-digit form. Throws on
-     * malformed input.
-     */
-    private encode;
-    /** Decode a 16-byte Buffer back to canonical UUID string. */
-    private decode;
-    private ensure;
-}
-//# sourceMappingURL=nativeBinaryEntityIdMapper.d.ts.map

package/dist/utils/nativeBinaryEntityIdMapper.js

@@ -1,358 +0,0 @@
-/**
- * @module utils/nativeBinaryEntityIdMapper
- * @description TypeScript wrapper around cortex's native binary
- * `BinaryIdMapper`. Implements brainy's `EntityIdMapperProvider` so the
- * mmap-backed billion-scale mapper is a drop-in for the existing
- * JSON-persisted one.
- *
- * ## When this engages
- *
- * The cortex plugin registers this wrapper as the `'entityIdMapper'`
- * provider when the storage adapter exposes `getBinaryBlobPath()` (i.e.
- * filesystem-backed storage with cortex's 2.4.0 #2 mmap-vector layer).
- * Cloud-storage adapters fall back to the JSON variant
- * (`NativeEntityIdMapperWrapper`) since they have no local-path concept.
- *
- * ## UUID format conversion
- *
- * Brainy passes UUIDs as strings (typically the canonical 36-char
- * `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`). The native side works in
- * 16-byte Buffers. This wrapper converts at the boundary. Non-canonical
- * UUID strings (any other 32-hex-digit form) are also accepted.
- *
- * ## Concurrency
- *
- * `getOrAssign` is atomic across concurrent callers for the same UUID
- * (256 sharded per-UUID mutexes in the native layer). Lookups are
- * lock-free. The wrapper holds no JS-side mutable state besides the
- * native handle.
- *
- * ## IdSpace (Piece 10)
- *
- * The wrapper supports two entity-int wire widths:
- *
- * - `'u32'` (default): cortex 2.x compatible — JS `number` throughout,
- *   capped at 4.29 B entities. Persists in the legacy `int_to_uuid.bin`
- *   v1 header.
- * - `'u64'`: opt-in via `idSpace: 'u64'`. The native layer's `number`
- *   surface throws in this mode, so the wrapper transparently routes
- *   the `EntityIdMapperProvider` methods through the BigInt napi
- *   siblings and converts BigInt → number at the boundary. Entity ints
- *   above `Number.MAX_SAFE_INTEGER` (2^53 - 1 = ~9 PB of entities)
- *   throw a clear `EntityIdSpaceExceeded` error so callers get a loud
- *   failure rather than a silent precision loss.
- *
- * The `getOrAssignBig` / `getIntBig` / `getUuidBig` / `sizeBig`
- * sibling methods are available on both modes and always return
- * `bigint` losslessly — use them in u64-aware code paths that need
- * the full u64 range.
- */
-import { existsSync } from 'node:fs';
-import { loadNativeModule } from '../native/index.js';
-import { prodLog } from '@soulcraft/brainy/internals';
-const UUID_BYTES = 16;
-/**
- * Thrown when a U64-mode mapper allocates or returns an entity int
- * above `Number.MAX_SAFE_INTEGER` (2^53 - 1). At this point a JS
- * `number` can no longer represent the value losslessly; callers must
- * switch to the BigInt sibling methods (`getOrAssignBig`,
- * `getIntBig`, `getUuidBig`).
- */
-export class EntityIdSpaceExceeded extends Error {
-    /** The u64 entity int that exceeded the safe-integer ceiling. */
-    value;
-    /** The method that was called (`'getOrAssign'`, `'getInt'`, etc.). */
-    method;
-    constructor(method, value) {
-        super(`${method}: entity int ${value} exceeds Number.MAX_SAFE_INTEGER ` +
-            `(2^53 - 1). Switch to the BigInt sibling method (${method}Big) ` +
-            `for entity ints above 9.007 PB.`);
-        this.name = 'EntityIdSpaceExceeded';
-        this.method = method;
-        this.value = value;
-    }
-}
-const MAX_SAFE_INTEGER_BIG = BigInt(Number.MAX_SAFE_INTEGER);
-/**
- * Convert a `bigint` entity int to a JS `number`. Throws
- * {@link EntityIdSpaceExceeded} if the value exceeds the JS safe-integer
- * range.
- */
-function bigToSafeNumber(value, method) {
-    if (value > MAX_SAFE_INTEGER_BIG) {
-        throw new EntityIdSpaceExceeded(method, value);
-    }
-    return Number(value);
-}
-const DEFAULT_UUID_TO_INT_KEY = '_id_mapper/uuid_to_int.mkv';
-const DEFAULT_INT_TO_UUID_KEY = '_id_mapper/int_to_uuid.bin';
-/**
- * Drop-in `EntityIdMapperProvider` backed by the native `BinaryIdMapper`.
- *
- * @example
- * ```typescript
- * const mapper = new NativeBinaryEntityIdMapperWrapper({ storage })
- * await mapper.init()
- * const intId = mapper.getOrAssign('12345678-1234-5678-1234-567812345678')
- * const uuid = mapper.getUuid(intId)
- * ```
- */
-export class NativeBinaryEntityIdMapperWrapper {
-    storage;
-    uuidToIntKey;
-    intToUuidKey;
-    intToUuidSize;
-    uuidToIntSize;
-    bucketCapacity;
-    maxGlobalDepth;
-    requestedIdSpace;
-    /**
-     * The actual IdSpace of the open mapper, sourced from the native
-     * binding's `idSpace()` reflection after `init()`. The on-disk
-     * header wins over `requestedIdSpace` (which may be ignored at
-     * `openExisting` time).
-     */
-    resolvedIdSpace = 'u32';
-    native = null;
-    initialized = false;
-    constructor(options) {
-        this.storage = options.storage;
-        this.uuidToIntKey = options.uuidToIntKey ?? DEFAULT_UUID_TO_INT_KEY;
-        this.intToUuidKey = options.intToUuidKey ?? DEFAULT_INT_TO_UUID_KEY;
-        this.intToUuidSize = options.intToUuidSize ?? BigInt(32) * BigInt(1024) ** BigInt(3);
-        this.uuidToIntSize = options.uuidToIntSize ?? BigInt(32) * BigInt(1024) ** BigInt(3);
-        this.bucketCapacity = options.bucketCapacity ?? 16;
-        this.maxGlobalDepth = options.maxGlobalDepth ?? 28;
-        this.requestedIdSpace = options.idSpace ?? 'u32';
-    }
-    async init() {
-        if (this.initialized)
-            return;
-        const storage = this.storage;
-        if (!storage.getBinaryBlobPath) {
-            throw new Error('NativeBinaryEntityIdMapperWrapper requires a storage adapter that ' +
-                'exposes getBinaryBlobPath() (filesystem-backed). For cloud adapters, ' +
-                'use NativeEntityIdMapperWrapper (JSON variant) instead.');
-        }
-        const uuidToIntPath = storage.getBinaryBlobPath(this.uuidToIntKey);
-        const intToUuidPath = storage.getBinaryBlobPath(this.intToUuidKey);
-        if (!uuidToIntPath || !intToUuidPath) {
-            throw new Error(`NativeBinaryEntityIdMapperWrapper: getBinaryBlobPath returned null for ` +
-                `${this.uuidToIntKey} or ${this.intToUuidKey}`);
-        }
-        const bindings = loadNativeModule();
-        const NativeBinaryIdMapper = bindings.NativeBinaryIdMapper;
-        if (!NativeBinaryIdMapper) {
-            throw new Error('NativeBinaryIdMapper binding missing from cortex native module — ' +
-                'this build of cortex is older than the BinaryIdMapper feature');
-        }
-        const config = {
-            uuidToIntPath,
-            intToUuidPath,
-            intToUuidSize: this.intToUuidSize,
-            uuidToIntSize: this.uuidToIntSize,
-            bucketCapacity: this.bucketCapacity,
-            maxGlobalDepth: this.maxGlobalDepth,
-            idSpace: this.requestedIdSpace,
-        };
-        // Explicitly distinguish "fresh install" from "existing files".
-        // Both files must exist together (paired write semantics) — a
-        // half-present state is corruption from a crash between file
-        // creations and is surfaced as an error rather than silently
-        // recreated.
-        const uuidFileExists = existsSync(uuidToIntPath);
-        const intFileExists = existsSync(intToUuidPath);
-        if (uuidFileExists && intFileExists) {
-            this.native = NativeBinaryIdMapper.openExisting(config);
-        }
-        else if (!uuidFileExists && !intFileExists) {
-            this.native = NativeBinaryIdMapper.create(config);
-        }
-        else {
-            throw new Error(`NativeBinaryEntityIdMapperWrapper: half-present file pair — ` +
-                `${this.uuidToIntKey} ${uuidFileExists ? 'exists' : 'missing'}, ` +
-                `${this.intToUuidKey} ${intFileExists ? 'exists' : 'missing'}. ` +
-                `Refusing to silently recreate; investigate manually.`);
-        }
-        // Reflect the on-disk IdSpace — authoritative over the requested
-        // value when openExisting opens a file with a different mode.
-        this.resolvedIdSpace = this.native.idSpace();
-        this.initialized = true;
-        if (prodLog?.debug) {
-            prodLog.debug(`[cortex] BinaryIdMapper wired: paths=[${uuidToIntPath}, ${intToUuidPath}], idSpace=${this.resolvedIdSpace}`);
-        }
-    }
-    /**
-     * Report the mapper's actual IdSpace mode. Returns `'u32'` before
-     * `init()` (the default the wrapper assumes); after init, returns the
-     * mode reported by the native binding (which is authoritative).
-     */
-    getIdSpace() {
-        return this.resolvedIdSpace;
-    }
-    // -- EntityIdMapperProvider surface (number-typed, brainy contract) --
-    /**
-     * Allocate or retrieve the entity int for `uuid`. Returns a JS
-     * `number`. In U64 mode, routes through the BigInt sibling and
-     * throws {@link EntityIdSpaceExceeded} if the allocated int exceeds
-     * `Number.MAX_SAFE_INTEGER` — at that point the caller MUST switch
-     * to `getOrAssignBig` for the full u64 range.
-     */
-    getOrAssign(uuid) {
-        const native = this.ensure();
-        if (this.resolvedIdSpace === 'u64') {
-            return bigToSafeNumber(native.getOrAssignBig(this.encode(uuid)), 'getOrAssign');
-        }
-        return native.getOrAssign(this.encode(uuid));
-    }
-    /**
-     * Look up the UUID for `intId`. Accepts a JS `number` — in U64 mode
-     * this is a lossy conversion above 2^53; use {@link getUuidBig} for
-     * the full u64 range.
-     */
-    getUuid(intId) {
-        const native = this.ensure();
-        const buf = this.resolvedIdSpace === 'u64'
-            ? native.getUuidBig(BigInt(intId))
-            : native.getUuid(intId);
-        if (!buf)
-            return undefined;
-        return this.decode(buf);
-    }
-    /**
-     * Look up the entity int for `uuid`. Returns a JS `number`. In U64
-     * mode throws {@link EntityIdSpaceExceeded} if the int exceeds
-     * `Number.MAX_SAFE_INTEGER`.
-     */
-    getInt(uuid) {
-        const native = this.ensure();
-        if (this.resolvedIdSpace === 'u64') {
-            const big = native.getIntBig(this.encode(uuid));
-            return big == null ? undefined : bigToSafeNumber(big, 'getInt');
-        }
-        const out = native.getInt(this.encode(uuid));
-        return out == null ? undefined : out;
-    }
-    remove(uuid) {
-        const native = this.ensure();
-        return native.remove(this.encode(uuid));
-    }
-    async flush() {
-        const native = this.ensure();
-        native.flush();
-    }
-    async clear() {
-        // Reset by recreating the files. Atomicity caveat: any concurrent
-        // reader holds a stale mmap. Brainy calls clear() during clear()
-        // operations that already block other access; this is fine.
-        this.initialized = false;
-        this.native = null;
-        await this.init();
-    }
-    /**
-     * Materialise every live int id into a JS `number[]`. **U32 mode
-     * only.** U64 mode throws — the native binding refuses to allocate
-     * a giant JS array at the scale a U64 brain implies. Iterate via
-     * the BigInt sibling iterator (TBD — a follow-up surfaces it on the
-     * wrapper) for U64 brains.
-     */
-    getAllIntIds() {
-        const native = this.ensure();
-        if (this.resolvedIdSpace === 'u64') {
-            throw new Error('getAllIntIds: not supported in U64 mode — the materialised ' +
-                'number[] would risk OOM at billion scale. Use the BigInt ' +
-                'streaming iterator on the underlying native binding instead.');
-        }
-        return native.getAllIntIds();
-    }
-    intsIterableToUuids(ints) {
-        const native = this.ensure();
-        const u64 = this.resolvedIdSpace === 'u64';
-        const out = [];
-        for (const i of ints) {
-            const buf = u64 ? native.getUuidBig(BigInt(i)) : native.getUuid(i);
-            if (buf)
-                out.push(this.decode(buf));
-        }
-        return out;
-    }
-    get size() {
-        if (!this.initialized || !this.native)
-            return 0;
-        if (this.resolvedIdSpace === 'u64') {
-            return bigToSafeNumber(this.native.sizeBig(), 'size');
-        }
-        return this.native.size();
-    }
-    // -- BigInt sibling surface (u64-safe, works in both modes) -------
-    /**
-     * Allocate or retrieve the entity int for `uuid` as a `bigint`.
-     * Lossless across the full u64 range; safe to call in either mode.
-     */
-    getOrAssignBig(uuid) {
-        const native = this.ensure();
-        return native.getOrAssignBig(this.encode(uuid));
-    }
-    /** Look up the entity int for `uuid` as a `bigint`. */
-    getIntBig(uuid) {
-        const native = this.ensure();
-        const out = native.getIntBig(this.encode(uuid));
-        return out == null ? undefined : out;
-    }
-    /** Look up the UUID for `int` (passed as a `bigint`). */
-    getUuidBig(int) {
-        const native = this.ensure();
-        const buf = native.getUuidBig(int);
-        if (!buf)
-            return undefined;
-        return this.decode(buf);
-    }
-    /** Live (non-tombstone) entry count as a `bigint`. */
-    sizeBig() {
-        if (!this.initialized || !this.native)
-            return 0n;
-        return this.native.sizeBig();
-    }
-    /** Largest int ever assigned + 1, as a `bigint`. */
-    nextIntBig() {
-        return this.ensure().nextIntBig();
-    }
-    // ---------------------------------------------------------------
-    // UUID string ↔ Buffer conversion
-    // ---------------------------------------------------------------
-    /**
-     * Encode a UUID string into a 16-byte Buffer. Accepts canonical
-     * 36-char form (with hyphens) or any 32-hex-digit form. Throws on
-     * malformed input.
-     */
-    encode(uuid) {
-        const hex = uuid.replace(/-/g, '').toLowerCase();
-        if (hex.length !== 32 || !/^[0-9a-f]{32}$/.test(hex)) {
-            throw new Error(`NativeBinaryEntityIdMapperWrapper: invalid UUID string "${uuid}"`);
-        }
-        return Buffer.from(hex, 'hex');
-    }
-    /** Decode a 16-byte Buffer back to canonical UUID string. */
-    decode(buf) {
-        if (buf.length !== UUID_BYTES) {
-            throw new Error(`NativeBinaryEntityIdMapperWrapper: native returned ${buf.length}-byte uuid (expected ${UUID_BYTES})`);
-        }
-        const hex = buf.toString('hex');
-        return (hex.slice(0, 8) +
-            '-' +
-            hex.slice(8, 12) +
-            '-' +
-            hex.slice(12, 16) +
-            '-' +
-            hex.slice(16, 20) +
-            '-' +
-            hex.slice(20, 32));
-    }
-    ensure() {
-        if (!this.initialized || !this.native) {
-            throw new Error('NativeBinaryEntityIdMapperWrapper: call init() before any operation');
-        }
-        return this.native;
-    }
-}
-//# sourceMappingURL=nativeBinaryEntityIdMapper.js.map