@soulcraft/cortex 2.5.1 → 2.7.0

package/dist/hnsw/NativeDiskAnnWrapper.js

@@ -181,40 +181,60 @@ export class NativeDiskAnnWrapper {
      */
     async rebuild(options) {
         const bindings = loadNativeModule();
-        const NativeDiskANN = bindings.NativeDiskANN;
+        // napi-rs exports the class as `NativeDiskAnn` (PascalCase
+        // normalization of the Rust ident `NativeDiskANN`). The TS type
+        // alias `NativeDiskANN = NativeDiskAnn` in `native/index.d.ts` is
+        // for backwards-compat in *types* only — at runtime there's a
+        // single export under the napi-normalized name.
+        const NativeDiskANN = bindings.NativeDiskAnn;
         if (!NativeDiskANN) {
             throw new Error('NativeDiskANN binding missing — rebuild requires the cortex native module');
         }
-        // Collect the surviving vector set: main minus tombstones, plus delta.
-        const allVectors = [];
+        // Build the new logical slot ordering: (live old slots) + (delta).
+        // **Critical for billion-scale correctness**: the old vectors stay
+        // mmap'd inside the native module — we only pass slot IDs across
+        // the FFI boundary, not the vector data itself. At 1B × 1536 × 4
+        // bytes = ~6 TB this is the difference between "rebuild works" and
+        // "rebuild OOMs."
+        const liveOldSlots = [];
+        const newUuids = [];
         if (this.native) {
-            // Iterate current main index. The native side doesn't expose a
-            // vector iterator yet (35c follow-up), so we replay the
-            // delta+tombstones model: callers building from scratch should
-            // pass a fresh storage source. For now: rebuild from delta only.
-            // TODO once NativeDiskANN.iterAll() lands, fold the main index
-            //      into allVectors here.
+            // Iterate in slot order so the new index's first n_live slots
+            // mirror the OLD index's surviving subset in deterministic order.
+            // We deliberately iterate by sorted slot id rather than uuidBySlot
+            // insertion order — sorting keeps the Vamana entry point stable
+            // and the on-disk vector section's locality similar to the
+            // pre-rebuild file (less page-cache turnover during the post-
+            // rebuild warm-up).
+            const sortedSlots = Array.from(this.uuidBySlot.keys()).sort((a, b) => a - b);
+            for (const slot of sortedSlots) {
+                const uuid = this.uuidBySlot.get(slot);
+                if (this.tombstones.has(uuid))
+                    continue;
+                liveOldSlots.push(slot);
+                newUuids.push(uuid);
+            }
         }
-        for (const [id, vector] of this.delta) {
-            allVectors.push({ id, vector });
+        const dim = this.config.dimensions;
+        const deltaCount = this.delta.size;
+        let deltaBuf = null;
+        if (deltaCount > 0) {
+            deltaBuf = new Float32Array(deltaCount * dim);
+            let idx = 0;
+            for (const [uuid, vector] of this.delta) {
+                if (vector.length !== dim) {
+                    throw new Error(`NativeDiskAnnWrapper.rebuild: vector dim ${vector.length} ≠ index dim ${dim}`);
+                }
+                deltaBuf.set(vector, idx * dim);
+                newUuids.push(uuid);
+                idx++;
+            }
         }
-        if (allVectors.length === 0) {
+        if (liveOldSlots.length + deltaCount === 0) {
             prodLog?.warn?.('NativeDiskAnnWrapper.rebuild: nothing to build');
             return;
         }
-        const dim = this.config.dimensions;
-        const buf = new Float32Array(allVectors.length * dim);
-        const newSlotByUuid = new Map();
-        const newUuidBySlot = new Map();
-        for (let i = 0; i < allVectors.length; i++) {
-            const v = allVectors[i].vector;
-            if (v.length !== dim) {
-                throw new Error(`NativeDiskAnnWrapper.rebuild: vector dim ${v.length} ≠ index dim ${dim}`);
-            }
-            buf.set(v, i * dim);
-            newSlotByUuid.set(allVectors[i].id, i);
-            newUuidBySlot.set(i, allVectors[i].id);
-        }
+        const totalCount = liveOldSlots.length + deltaCount;
         const cfg = {
             vamana: {
                 maxDegree: options?.maxDegree ?? this.config.maxDegree,
@@ -228,7 +248,7 @@ export class NativeDiskAnnWrapper {
                 m: options?.pqM ?? this.config.pqM,
                 ksub: options?.pqKsub ?? this.config.pqKsub,
                 iterations: 25,
-                trainingSample: Math.min(200_000, allVectors.length),
+                trainingSample: Math.min(200_000, totalCount),
             },
             adjacency: this.config.useMmapAdjacency
                 ? {
@@ -237,9 +257,27 @@ export class NativeDiskAnnWrapper {
                 }
                 : { kind: 'ram' },
         };
-        const newNative = NativeDiskANN.build(Buffer.from(buf.buffer, buf.byteOffset, buf.byteLength), dim, this.config.indexPath, cfg);
-        // Atomic swap: replace the searcher + the slot maps, drop tombstones
-        // (they're already applied — the rebuilt set excludes them).
+        const newNative = NativeDiskANN.rebuildFromExisting({
+            existingPath: this.native ? this.config.indexPath : undefined,
+            liveOldSlots,
+            deltaVectors: deltaBuf != null
+                ? Buffer.from(deltaBuf.buffer, deltaBuf.byteOffset, deltaBuf.byteLength)
+                : undefined,
+            deltaCount,
+            dim,
+            outputPath: this.config.indexPath,
+            cfg,
+        });
+        // Rebuild the bidirectional UUID↔slot maps from `newUuids`. New
+        // slot `i` corresponds to `newUuids[i]` — this matches the napi
+        // layout invariant (live old slots first, delta tail second).
+        const newSlotByUuid = new Map();
+        const newUuidBySlot = new Map();
+        for (let i = 0; i < newUuids.length; i++) {
+            newSlotByUuid.set(newUuids[i], i);
+            newUuidBySlot.set(i, newUuids[i]);
+        }
+        // Atomic swap.
         this.native = newNative;
         this.slotByUuid = newSlotByUuid;
         this.uuidBySlot = newUuidBySlot;
@@ -260,7 +298,12 @@ export class NativeDiskAnnWrapper {
     tryOpenExisting() {
         try {
             const bindings = loadNativeModule();
-            const NativeDiskANN = bindings.NativeDiskANN;
+            // napi-rs exports the class as `NativeDiskAnn` (PascalCase
+            // normalization of the Rust ident `NativeDiskANN`). The TS type
+            // alias `NativeDiskANN = NativeDiskAnn` in `native/index.d.ts` is
+            // for backwards-compat in *types* only — at runtime there's a
+            // single export under the napi-normalized name.
+            const NativeDiskANN = bindings.NativeDiskAnn;
             if (!NativeDiskANN)
                 return;
             this.native = NativeDiskANN.openExisting(this.config.indexPath);

package/dist/utils/nativeBinaryEntityIdMapper.d.ts

@@ -26,6 +26,26 @@
  * (256 sharded per-UUID mutexes in the native layer). Lookups are
  * lock-free. The wrapper holds no JS-side mutable state besides the
  * native handle.
+ *
+ * ## IdSpace (Piece 10)
+ *
+ * The wrapper supports two entity-int wire widths:
+ *
+ * - `'u32'` (default): cortex 2.x compatible — JS `number` throughout,
+ *   capped at 4.29 B entities. Persists in the legacy `int_to_uuid.bin`
+ *   v1 header.
+ * - `'u64'`: opt-in via `idSpace: 'u64'`. The native layer's `number`
+ *   surface throws in this mode, so the wrapper transparently routes
+ *   the `EntityIdMapperProvider` methods through the BigInt napi
+ *   siblings and converts BigInt → number at the boundary. Entity ints
+ *   above `Number.MAX_SAFE_INTEGER` (2^53 - 1 = ~9 PB of entities)
+ *   throw a clear `EntityIdSpaceExceeded` error so callers get a loud
+ *   failure rather than a silent precision loss.
+ *
+ * The `getOrAssignBig` / `getIntBig` / `getUuidBig` / `sizeBig`
+ * sibling methods are available on both modes and always return
+ * `bigint` losslessly — use them in u64-aware code paths that need
+ * the full u64 range.
  */
 import type { StorageAdapter } from '@soulcraft/brainy';
 import type { EntityIdMapperProvider } from '../providerContracts.js';
@@ -50,6 +70,33 @@ export interface NativeBinaryEntityIdMapperOptions {
     bucketCapacity?: number;
     /** Maximum extendible-hash directory depth. Default 28. */
     maxGlobalDepth?: number;
+    /**
+     * Entity-int wire width. `'u32'` (default) caps at 4.29 B entities
+     * and is cortex 2.x compatible. `'u64'` opts into the Piece 10 U64
+     * IdSpace — required when targeting >4.29 B entities. The
+     * `EntityIdMapperProvider` `number`-typed methods still work in U64
+     * mode by routing through the BigInt napi siblings; entity ints
+     * above `Number.MAX_SAFE_INTEGER` (2^53 - 1) throw an explicit
+     * `EntityIdSpaceExceeded` error.
+     *
+     * Ignored on open when the underlying file's header disagrees: the
+     * on-disk format wins and any mismatch is surfaced as a hard error.
+     */
+    idSpace?: 'u32' | 'u64';
+}
+/**
+ * Thrown when a U64-mode mapper allocates or returns an entity int
+ * above `Number.MAX_SAFE_INTEGER` (2^53 - 1). At this point a JS
+ * `number` can no longer represent the value losslessly; callers must
+ * switch to the BigInt sibling methods (`getOrAssignBig`,
+ * `getIntBig`, `getUuidBig`).
+ */
+export declare class EntityIdSpaceExceeded extends Error {
+    /** The u64 entity int that exceeded the safe-integer ceiling. */
+    readonly value: bigint;
+    /** The method that was called (`'getOrAssign'`, `'getInt'`, etc.). */
+    readonly method: string;
+    constructor(method: string, value: bigint);
 }
 /**
  * Drop-in `EntityIdMapperProvider` backed by the native `BinaryIdMapper`.
@@ -70,19 +117,70 @@ export declare class NativeBinaryEntityIdMapperWrapper implements EntityIdMapper
     private uuidToIntSize;
     private bucketCapacity;
     private maxGlobalDepth;
+    private requestedIdSpace;
+    /**
+     * The actual IdSpace of the open mapper, sourced from the native
+     * binding's `idSpace()` reflection after `init()`. The on-disk
+     * header wins over `requestedIdSpace` (which may be ignored at
+     * `openExisting` time).
+     */
+    private resolvedIdSpace;
     private native;
     private initialized;
     constructor(options: NativeBinaryEntityIdMapperOptions);
     init(): Promise<void>;
+    /**
+     * Report the mapper's actual IdSpace mode. Returns `'u32'` before
+     * `init()` (the default the wrapper assumes); after init, returns the
+     * mode reported by the native binding (which is authoritative).
+     */
+    getIdSpace(): 'u32' | 'u64';
+    /**
+     * Allocate or retrieve the entity int for `uuid`. Returns a JS
+     * `number`. In U64 mode, routes through the BigInt sibling and
+     * throws {@link EntityIdSpaceExceeded} if the allocated int exceeds
+     * `Number.MAX_SAFE_INTEGER` — at that point the caller MUST switch
+     * to `getOrAssignBig` for the full u64 range.
+     */
     getOrAssign(uuid: string): number;
+    /**
+     * Look up the UUID for `intId`. Accepts a JS `number` — in U64 mode
+     * this is a lossy conversion above 2^53; use {@link getUuidBig} for
+     * the full u64 range.
+     */
     getUuid(intId: number): string | undefined;
+    /**
+     * Look up the entity int for `uuid`. Returns a JS `number`. In U64
+     * mode throws {@link EntityIdSpaceExceeded} if the int exceeds
+     * `Number.MAX_SAFE_INTEGER`.
+     */
     getInt(uuid: string): number | undefined;
     remove(uuid: string): boolean;
     flush(): Promise<void>;
     clear(): Promise<void>;
+    /**
+     * Materialise every live int id into a JS `number[]`. **U32 mode
+     * only.** U64 mode throws — the native binding refuses to allocate
+     * a giant JS array at the scale a U64 brain implies. Iterate via
+     * the BigInt sibling iterator (TBD — a follow-up surfaces it on the
+     * wrapper) for U64 brains.
+     */
     getAllIntIds(): number[];
     intsIterableToUuids(ints: Iterable<number>): string[];
     get size(): number;
+    /**
+     * Allocate or retrieve the entity int for `uuid` as a `bigint`.
+     * Lossless across the full u64 range; safe to call in either mode.
+     */
+    getOrAssignBig(uuid: string): bigint;
+    /** Look up the entity int for `uuid` as a `bigint`. */
+    getIntBig(uuid: string): bigint | undefined;
+    /** Look up the UUID for `int` (passed as a `bigint`). */
+    getUuidBig(int: bigint): string | undefined;
+    /** Live (non-tombstone) entry count as a `bigint`. */
+    sizeBig(): bigint;
+    /** Largest int ever assigned + 1, as a `bigint`. */
+    nextIntBig(): bigint;
     /**
      * Encode a UUID string into a 16-byte Buffer. Accepts canonical
      * 36-char form (with hyphens) or any 32-hex-digit form. Throws on

package/dist/utils/nativeBinaryEntityIdMapper.js

@@ -26,11 +26,64 @@
  * (256 sharded per-UUID mutexes in the native layer). Lookups are
  * lock-free. The wrapper holds no JS-side mutable state besides the
  * native handle.
+ *
+ * ## IdSpace (Piece 10)
+ *
+ * The wrapper supports two entity-int wire widths:
+ *
+ * - `'u32'` (default): cortex 2.x compatible — JS `number` throughout,
+ *   capped at 4.29 B entities. Persists in the legacy `int_to_uuid.bin`
+ *   v1 header.
+ * - `'u64'`: opt-in via `idSpace: 'u64'`. The native layer's `number`
+ *   surface throws in this mode, so the wrapper transparently routes
+ *   the `EntityIdMapperProvider` methods through the BigInt napi
+ *   siblings and converts BigInt → number at the boundary. Entity ints
+ *   above `Number.MAX_SAFE_INTEGER` (2^53 - 1 = ~9 PB of entities)
+ *   throw a clear `EntityIdSpaceExceeded` error so callers get a loud
+ *   failure rather than a silent precision loss.
+ *
+ * The `getOrAssignBig` / `getIntBig` / `getUuidBig` / `sizeBig`
+ * sibling methods are available on both modes and always return
+ * `bigint` losslessly — use them in u64-aware code paths that need
+ * the full u64 range.
  */
 import { existsSync } from 'node:fs';
 import { loadNativeModule } from '../native/index.js';
 import { prodLog } from '@soulcraft/brainy/internals';
 const UUID_BYTES = 16;
+/**
+ * Thrown when a U64-mode mapper allocates or returns an entity int
+ * above `Number.MAX_SAFE_INTEGER` (2^53 - 1). At this point a JS
+ * `number` can no longer represent the value losslessly; callers must
+ * switch to the BigInt sibling methods (`getOrAssignBig`,
+ * `getIntBig`, `getUuidBig`).
+ */
+export class EntityIdSpaceExceeded extends Error {
+    /** The u64 entity int that exceeded the safe-integer ceiling. */
+    value;
+    /** The method that was called (`'getOrAssign'`, `'getInt'`, etc.). */
+    method;
+    constructor(method, value) {
+        super(`${method}: entity int ${value} exceeds Number.MAX_SAFE_INTEGER ` +
+            `(2^53 - 1). Switch to the BigInt sibling method (${method}Big) ` +
+            `for entity ints above 9.007 PB.`);
+        this.name = 'EntityIdSpaceExceeded';
+        this.method = method;
+        this.value = value;
+    }
+}
+const MAX_SAFE_INTEGER_BIG = BigInt(Number.MAX_SAFE_INTEGER);
+/**
+ * Convert a `bigint` entity int to a JS `number`. Throws
+ * {@link EntityIdSpaceExceeded} if the value exceeds the JS safe-integer
+ * range.
+ */
+function bigToSafeNumber(value, method) {
+    if (value > MAX_SAFE_INTEGER_BIG) {
+        throw new EntityIdSpaceExceeded(method, value);
+    }
+    return Number(value);
+}
 const DEFAULT_UUID_TO_INT_KEY = '_id_mapper/uuid_to_int.mkv';
 const DEFAULT_INT_TO_UUID_KEY = '_id_mapper/int_to_uuid.bin';
 /**
@@ -52,6 +105,14 @@ export class NativeBinaryEntityIdMapperWrapper {
     uuidToIntSize;
     bucketCapacity;
     maxGlobalDepth;
+    requestedIdSpace;
+    /**
+     * The actual IdSpace of the open mapper, sourced from the native
+     * binding's `idSpace()` reflection after `init()`. The on-disk
+     * header wins over `requestedIdSpace` (which may be ignored at
+     * `openExisting` time).
+     */
+    resolvedIdSpace = 'u32';
     native = null;
     initialized = false;
     constructor(options) {
@@ -62,6 +123,7 @@ export class NativeBinaryEntityIdMapperWrapper {
         this.uuidToIntSize = options.uuidToIntSize ?? BigInt(32) * BigInt(1024) ** BigInt(3);
         this.bucketCapacity = options.bucketCapacity ?? 16;
         this.maxGlobalDepth = options.maxGlobalDepth ?? 28;
+        this.requestedIdSpace = options.idSpace ?? 'u32';
     }
     async init() {
         if (this.initialized)
@@ -91,6 +153,7 @@ export class NativeBinaryEntityIdMapperWrapper {
             uuidToIntSize: this.uuidToIntSize,
             bucketCapacity: this.bucketCapacity,
             maxGlobalDepth: this.maxGlobalDepth,
+            idSpace: this.requestedIdSpace,
         };
         // Explicitly distinguish "fresh install" from "existing files".
         // Both files must exist together (paired write semantics) — a
@@ -111,24 +174,62 @@ export class NativeBinaryEntityIdMapperWrapper {
                 `${this.intToUuidKey} ${intFileExists ? 'exists' : 'missing'}. ` +
                 `Refusing to silently recreate; investigate manually.`);
         }
+        // Reflect the on-disk IdSpace — authoritative over the requested
+        // value when openExisting opens a file with a different mode.
+        this.resolvedIdSpace = this.native.idSpace();
         this.initialized = true;
         if (prodLog?.debug) {
-            prodLog.debug(`[cortex] BinaryIdMapper wired: paths=[${uuidToIntPath}, ${intToUuidPath}]`);
+            prodLog.debug(`[cortex] BinaryIdMapper wired: paths=[${uuidToIntPath}, ${intToUuidPath}], idSpace=${this.resolvedIdSpace}`);
         }
     }
+    /**
+     * Report the mapper's actual IdSpace mode. Returns `'u32'` before
+     * `init()` (the default the wrapper assumes); after init, returns the
+     * mode reported by the native binding (which is authoritative).
+     */
+    getIdSpace() {
+        return this.resolvedIdSpace;
+    }
+    // -- EntityIdMapperProvider surface (number-typed, brainy contract) --
+    /**
+     * Allocate or retrieve the entity int for `uuid`. Returns a JS
+     * `number`. In U64 mode, routes through the BigInt sibling and
+     * throws {@link EntityIdSpaceExceeded} if the allocated int exceeds
+     * `Number.MAX_SAFE_INTEGER` — at that point the caller MUST switch
+     * to `getOrAssignBig` for the full u64 range.
+     */
     getOrAssign(uuid) {
         const native = this.ensure();
+        if (this.resolvedIdSpace === 'u64') {
+            return bigToSafeNumber(native.getOrAssignBig(this.encode(uuid)), 'getOrAssign');
+        }
         return native.getOrAssign(this.encode(uuid));
     }
+    /**
+     * Look up the UUID for `intId`. Accepts a JS `number` — in U64 mode
+     * this is a lossy conversion above 2^53; use {@link getUuidBig} for
+     * the full u64 range.
+     */
     getUuid(intId) {
         const native = this.ensure();
-        const buf = native.getUuid(intId);
+        const buf = this.resolvedIdSpace === 'u64'
+            ? native.getUuidBig(BigInt(intId))
+            : native.getUuid(intId);
         if (!buf)
             return undefined;
         return this.decode(buf);
     }
+    /**
+     * Look up the entity int for `uuid`. Returns a JS `number`. In U64
+     * mode throws {@link EntityIdSpaceExceeded} if the int exceeds
+     * `Number.MAX_SAFE_INTEGER`.
+     */
     getInt(uuid) {
         const native = this.ensure();
+        if (this.resolvedIdSpace === 'u64') {
+            const big = native.getIntBig(this.encode(uuid));
+            return big == null ? undefined : bigToSafeNumber(big, 'getInt');
+        }
         const out = native.getInt(this.encode(uuid));
         return out == null ? undefined : out;
     }
@@ -148,15 +249,28 @@ export class NativeBinaryEntityIdMapperWrapper {
         this.native = null;
         await this.init();
     }
+    /**
+     * Materialise every live int id into a JS `number[]`. **U32 mode
+     * only.** U64 mode throws — the native binding refuses to allocate
+     * a giant JS array at the scale a U64 brain implies. Iterate via
+     * the BigInt sibling iterator (TBD — a follow-up surfaces it on the
+     * wrapper) for U64 brains.
+     */
     getAllIntIds() {
         const native = this.ensure();
+        if (this.resolvedIdSpace === 'u64') {
+            throw new Error('getAllIntIds: not supported in U64 mode — the materialised ' +
+                'number[] would risk OOM at billion scale. Use the BigInt ' +
+                'streaming iterator on the underlying native binding instead.');
+        }
         return native.getAllIntIds();
     }
     intsIterableToUuids(ints) {
         const native = this.ensure();
+        const u64 = this.resolvedIdSpace === 'u64';
         const out = [];
         for (const i of ints) {
-            const buf = native.getUuid(i);
+            const buf = u64 ? native.getUuidBig(BigInt(i)) : native.getUuid(i);
             if (buf)
                 out.push(this.decode(buf));
         }
@@ -165,8 +279,44 @@ export class NativeBinaryEntityIdMapperWrapper {
     get size() {
         if (!this.initialized || !this.native)
             return 0;
+        if (this.resolvedIdSpace === 'u64') {
+            return bigToSafeNumber(this.native.sizeBig(), 'size');
+        }
         return this.native.size();
     }
+    // -- BigInt sibling surface (u64-safe, works in both modes) -------
+    /**
+     * Allocate or retrieve the entity int for `uuid` as a `bigint`.
+     * Lossless across the full u64 range; safe to call in either mode.
+     */
+    getOrAssignBig(uuid) {
+        const native = this.ensure();
+        return native.getOrAssignBig(this.encode(uuid));
+    }
+    /** Look up the entity int for `uuid` as a `bigint`. */
+    getIntBig(uuid) {
+        const native = this.ensure();
+        const out = native.getIntBig(this.encode(uuid));
+        return out == null ? undefined : out;
+    }
+    /** Look up the UUID for `int` (passed as a `bigint`). */
+    getUuidBig(int) {
+        const native = this.ensure();
+        const buf = native.getUuidBig(int);
+        if (!buf)
+            return undefined;
+        return this.decode(buf);
+    }
+    /** Live (non-tombstone) entry count as a `bigint`. */
+    sizeBig() {
+        if (!this.initialized || !this.native)
+            return 0n;
+        return this.native.sizeBig();
+    }
+    /** Largest int ever assigned + 1, as a `bigint`. */
+    nextIntBig() {
+        return this.ensure().nextIntBig();
+    }
     // ---------------------------------------------------------------
     // UUID string ↔ Buffer conversion
     // ---------------------------------------------------------------

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/cortex",
-  "version": "2.5.1",
+  "version": "2.7.0",
   "description": "Native Rust acceleration for Brainy — SIMD distance, vector quantization, zero-copy mmap, native embeddings. Free tier for storage, Pro license for compute acceleration.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -66,11 +66,11 @@
     "LICENSE"
   ],
   "peerDependencies": {
-    "@soulcraft/brainy": ">=7.28.0"
+    "@soulcraft/brainy": ">=7.30.0"
   },
   "devDependencies": {
     "@napi-rs/cli": "^3.0.0",
-    "@soulcraft/brainy": "^7.28.0",
+    "@soulcraft/brainy": "7.30.0",
     "@types/node": "^22.0.0",
     "tsx": "^4.21.0",
     "typescript": "^5.9.3",