@soulcraft/cortex 2.5.0 → 2.6.0

package/dist/hnsw/NativeDiskAnnWrapper.d.ts

@@ -0,0 +1,161 @@
+/**
+ * @module hnsw/NativeDiskAnnWrapper
+ * @description TypeScript wrapper around cortex's native DiskANN engine
+ * that satisfies brainy's `HnswProvider` contract. From brainy's
+ * perspective this is interchangeable with `NativeHNSWWrapper` — same
+ * `addItem` / `search` / `rebuild` surface — but underneath it drives
+ * the billion-scale Vamana + PQ index.
+ *
+ * @example
+ * ```typescript
+ * import { BrainyData } from '@soulcraft/brainy'
+ * import { register as registerCortex } from '@soulcraft/cortex'
+ *
+ * const brain = new BrainyData({
+ *   storage: { type: 'filesystem', rootDirectory: '/data/idx' }
+ * })
+ * await registerCortex(brain)
+ * await brain.init()  // [brainy] DiskANN engaged (path=..., dim=384)
+ *
+ * await brain.add({ data: 'native rust acceleration', type: 'concept' })
+ * const hits = await brain.search('billion scale ann', 10)
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Explicit billion-scale build config
+ * const brain = new BrainyData({
+ *   storage: { type: 'filesystem', rootDirectory: '/data/idx' },
+ *   index: {
+ *     type: 'diskann',
+ *     diskann: {
+ *       pqM: 16,
+ *       maxDegree: 64,
+ *       searchListSize: 100,
+ *       useMmapAdjacency: true,  // required >100M nodes
+ *       mmapAdjacencyPath: '/data/scratch/diskann-build.adj'
+ *     }
+ *   }
+ * })
+ * ```
+ *
+ * ## Operating model
+ *
+ * DiskANN is build-once, query-many by design: the on-disk file
+ * embeds the Vamana graph, PQ codebook, codes, and full vectors in a
+ * single contiguous mmap-able layout. Dynamic insertions go to a
+ * small **delta buffer** that brute-force-searches alongside the main
+ * index until the next `rebuild()` folds them in. This matches
+ * FreshDiskANN's published online-update model.
+ *
+ * ## Search path
+ *
+ * 1. Query the main index via the native DiskANN searcher: PQ-greedy
+ *    walk in RAM, full-vector re-rank on the candidate set.
+ * 2. Brute-force the delta buffer (typically <0.1% of total size after
+ *    a recent rebuild).
+ * 3. Merge + sort + truncate to `k`.
+ *
+ * ## When this wrapper engages
+ *
+ * Brainy's `wireDiskAnn()` decides at init time whether to instantiate
+ * this wrapper or the standard HNSW one. The criteria
+ * ([ADR-002](../../docs/ADR-002-diskann-100-percent-rust.md)):
+ * - Cortex's `index:diskann` provider is registered (this file).
+ * - The storage adapter exposes a local filesystem path
+ *   (`getBinaryBlobPath` is the canonical check).
+ * - The metadata index has a stable `idMapper` (the cortex 2.4.0 #23
+ *   foundation).
+ * - `config.index.type !== 'hnsw'` (opt-out path).
+ */
+import type { Vector, VectorDocument, DistanceFunction, StorageAdapter } from '@soulcraft/brainy';
+import type { HnswProvider } from '../providerContracts.js';
+export interface DiskAnnIndexConfig {
+    /** Vector dimension (e.g. 384 for all-MiniLM-L6-v2). */
+    dimensions: number;
+    /** Output path for the on-disk DiskANN file. */
+    indexPath: string;
+    /** PQ subspaces. Default 16. dim must be divisible by m. */
+    pqM?: number;
+    /** Centroids per subspace. Default 256 (8-bit codes). */
+    pqKsub?: number;
+    /** Vamana max degree (R). Default 64. */
+    maxDegree?: number;
+    /** Build-time candidate list size (L). Default 100. */
+    searchListSize?: number;
+    /** α-pruning density factor. Default 1.2. */
+    alpha?: number;
+    /** Default search-time candidate list size. `2*k` is a good baseline. */
+    defaultLSearch?: number;
+    /** Default padding factor for re-rank over-fetch. Default 1.2. */
+    defaultPaddingFactor?: number;
+    /** Use a file-backed adjacency during build. Required >~100M nodes. */
+    useMmapAdjacency?: boolean;
+    /** Scratch file path when `useMmapAdjacency` is true. */
+    mmapAdjacencyPath?: string;
+}
+export declare class NativeDiskAnnWrapper implements HnswProvider {
+    private config;
+    private distanceFunction;
+    private storage;
+    private persistMode;
+    /** Live searcher instance — null until the first build. */
+    private native;
+    /** Newly added entries since the last build. Brute-force searched. */
+    private delta;
+    /** Removed entries — filtered out at search time. */
+    private tombstones;
+    /** Bidirectional UUID ↔ slot map for the main index. */
+    private slotByUuid;
+    private uuidBySlot;
+    constructor(config: DiskAnnIndexConfig & {
+        distanceFunction?: DistanceFunction;
+    }, distanceFunction: DistanceFunction, options?: {
+        storage?: StorageAdapter | null;
+        persistMode?: 'immediate' | 'deferred';
+    });
+    /**
+     * Append an entry to the delta buffer. Persisted by the next
+     * `rebuild()` call, which folds the delta into the main index.
+     */
+    addItem(item: VectorDocument): Promise<string>;
+    /**
+     * Mark an entry as removed. Filtered out at search time; physically
+     * removed at the next `rebuild()`.
+     */
+    removeItem(id: string): Promise<boolean>;
+    search(queryVector: Vector, k?: number, filter?: (id: string) => Promise<boolean>, options?: {
+        rerank?: {
+            multiplier: number;
+        };
+        candidateIds?: string[];
+    }): Promise<Array<[string, number]>>;
+    size(): number;
+    clear(): void;
+    /**
+     * Rebuild the main index from scratch: concatenate (current main −
+     * tombstones) ∪ delta, run a full DiskANN build, swap the searcher
+     * atomically.
+     *
+     * At billion-scale this is the expensive operation (hours of build
+     * time). Operators schedule it during off-peak; the delta buffer
+     * absorbs writes in between.
+     */
+    rebuild(options?: {
+        pqM?: number;
+        pqKsub?: number;
+        maxDegree?: number;
+        searchListSize?: number;
+        alpha?: number;
+    }): Promise<void>;
+    /**
+     * Flush the delta buffer to disk. For DiskANN the delta is in-memory
+     * by design (a few MB at most between rebuilds); returns the buffer
+     * size for parity with HNSW's flush contract.
+     */
+    flush(): Promise<number>;
+    getPersistMode(): 'immediate' | 'deferred';
+    private tryOpenExisting;
+    private countMainTombstones;
+}
+//# sourceMappingURL=NativeDiskAnnWrapper.d.ts.map

package/dist/hnsw/NativeDiskAnnWrapper.js

@@ -0,0 +1,329 @@
+/**
+ * @module hnsw/NativeDiskAnnWrapper
+ * @description TypeScript wrapper around cortex's native DiskANN engine
+ * that satisfies brainy's `HnswProvider` contract. From brainy's
+ * perspective this is interchangeable with `NativeHNSWWrapper` — same
+ * `addItem` / `search` / `rebuild` surface — but underneath it drives
+ * the billion-scale Vamana + PQ index.
+ *
+ * @example
+ * ```typescript
+ * import { BrainyData } from '@soulcraft/brainy'
+ * import { register as registerCortex } from '@soulcraft/cortex'
+ *
+ * const brain = new BrainyData({
+ *   storage: { type: 'filesystem', rootDirectory: '/data/idx' }
+ * })
+ * await registerCortex(brain)
+ * await brain.init()  // [brainy] DiskANN engaged (path=..., dim=384)
+ *
+ * await brain.add({ data: 'native rust acceleration', type: 'concept' })
+ * const hits = await brain.search('billion scale ann', 10)
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Explicit billion-scale build config
+ * const brain = new BrainyData({
+ *   storage: { type: 'filesystem', rootDirectory: '/data/idx' },
+ *   index: {
+ *     type: 'diskann',
+ *     diskann: {
+ *       pqM: 16,
+ *       maxDegree: 64,
+ *       searchListSize: 100,
+ *       useMmapAdjacency: true,  // required >100M nodes
+ *       mmapAdjacencyPath: '/data/scratch/diskann-build.adj'
+ *     }
+ *   }
+ * })
+ * ```
+ *
+ * ## Operating model
+ *
+ * DiskANN is build-once, query-many by design: the on-disk file
+ * embeds the Vamana graph, PQ codebook, codes, and full vectors in a
+ * single contiguous mmap-able layout. Dynamic insertions go to a
+ * small **delta buffer** that brute-force-searches alongside the main
+ * index until the next `rebuild()` folds them in. This matches
+ * FreshDiskANN's published online-update model.
+ *
+ * ## Search path
+ *
+ * 1. Query the main index via the native DiskANN searcher: PQ-greedy
+ *    walk in RAM, full-vector re-rank on the candidate set.
+ * 2. Brute-force the delta buffer (typically <0.1% of total size after
+ *    a recent rebuild).
+ * 3. Merge + sort + truncate to `k`.
+ *
+ * ## When this wrapper engages
+ *
+ * Brainy's `wireDiskAnn()` decides at init time whether to instantiate
+ * this wrapper or the standard HNSW one. The criteria
+ * ([ADR-002](../../docs/ADR-002-diskann-100-percent-rust.md)):
+ * - Cortex's `index:diskann` provider is registered (this file).
+ * - The storage adapter exposes a local filesystem path
+ *   (`getBinaryBlobPath` is the canonical check).
+ * - The metadata index has a stable `idMapper` (the cortex 2.4.0 #23
+ *   foundation).
+ * - `config.index.type !== 'hnsw'` (opt-out path).
+ */
+import { loadNativeModule } from '../native/index.js';
+import { prodLog } from '@soulcraft/brainy/internals';
+const DEFAULTS = {
+    pqM: 16,
+    pqKsub: 256,
+    maxDegree: 64,
+    searchListSize: 100,
+    alpha: 1.2,
+    defaultLSearch: 100,
+    defaultPaddingFactor: 1.2,
+    useMmapAdjacency: false,
+};
+export class NativeDiskAnnWrapper {
+    config;
+    distanceFunction;
+    storage;
+    persistMode;
+    /** Live searcher instance — null until the first build. */
+    native = null;
+    /** Newly added entries since the last build. Brute-force searched. */
+    delta = new Map();
+    /** Removed entries — filtered out at search time. */
+    tombstones = new Set();
+    /** Bidirectional UUID ↔ slot map for the main index. */
+    slotByUuid = new Map();
+    uuidBySlot = new Map();
+    constructor(config, distanceFunction, options = {}) {
+        this.config = { ...DEFAULTS, ...config };
+        this.distanceFunction = distanceFunction;
+        this.storage = options.storage ?? null;
+        this.persistMode = options.persistMode ?? 'immediate';
+        // Try to open an existing file. If absent, the index stays
+        // empty until the first rebuild() flushes the delta buffer.
+        this.tryOpenExisting();
+    }
+    /**
+     * Append an entry to the delta buffer. Persisted by the next
+     * `rebuild()` call, which folds the delta into the main index.
+     */
+    async addItem(item) {
+        if (this.tombstones.has(item.id)) {
+            this.tombstones.delete(item.id);
+        }
+        this.delta.set(item.id, item.vector);
+        return item.id;
+    }
+    /**
+     * Mark an entry as removed. Filtered out at search time; physically
+     * removed at the next `rebuild()`.
+     */
+    async removeItem(id) {
+        const inDelta = this.delta.delete(id);
+        const inMain = this.slotByUuid.has(id);
+        if (inMain)
+            this.tombstones.add(id);
+        return inDelta || inMain;
+    }
+    async search(queryVector, k = 10, filter, options) {
+        const lSearch = Math.max(this.config.defaultLSearch, k * 2);
+        const padding = options?.rerank?.multiplier ?? this.config.defaultPaddingFactor;
+        // 1. Main-index PQ-greedy walk (returns slot ids).
+        const mainHits = this.native
+            ? this.native.search(Array.from(queryVector), k * 2, // over-fetch so filter / tombstone losses don't starve final result
+            lSearch, padding)
+            : [];
+        // 2. Hydrate slot → uuid; drop tombstoned + filter-rejected.
+        const merged = [];
+        for (const hit of mainHits) {
+            const uuid = this.uuidBySlot.get(hit.slot);
+            if (!uuid)
+                continue;
+            if (this.tombstones.has(uuid))
+                continue;
+            if (filter && !(await filter(uuid)))
+                continue;
+            merged.push([uuid, hit.distance]);
+        }
+        // 3. Brute-force the delta buffer.
+        for (const [id, v] of this.delta) {
+            if (filter && !(await filter(id)))
+                continue;
+            const d = this.distanceFunction(queryVector, v);
+            merged.push([id, d]);
+        }
+        // 4. Sort ascending by distance, truncate to k.
+        merged.sort((a, b) => a[1] - b[1]);
+        return merged.slice(0, k);
+    }
+    size() {
+        const mainSize = this.native ? this.native.size() : 0;
+        return (mainSize +
+            this.delta.size -
+            // Tombstones from the main index reduce effective size.
+            this.countMainTombstones());
+    }
+    clear() {
+        this.delta.clear();
+        this.tombstones.clear();
+        this.slotByUuid.clear();
+        this.uuidBySlot.clear();
+        this.native = null;
+    }
+    /**
+     * Rebuild the main index from scratch: concatenate (current main −
+     * tombstones) ∪ delta, run a full DiskANN build, swap the searcher
+     * atomically.
+     *
+     * At billion-scale this is the expensive operation (hours of build
+     * time). Operators schedule it during off-peak; the delta buffer
+     * absorbs writes in between.
+     */
+    async rebuild(options) {
+        const bindings = loadNativeModule();
+        // napi-rs exports the class as `NativeDiskAnn` (PascalCase
+        // normalization of the Rust ident `NativeDiskANN`). The TS type
+        // alias `NativeDiskANN = NativeDiskAnn` in `native/index.d.ts` is
+        // for backwards-compat in *types* only — at runtime there's a
+        // single export under the napi-normalized name.
+        const NativeDiskANN = bindings.NativeDiskAnn;
+        if (!NativeDiskANN) {
+            throw new Error('NativeDiskANN binding missing — rebuild requires the cortex native module');
+        }
+        // Build the new logical slot ordering: (live old slots) + (delta).
+        // **Critical for billion-scale correctness**: the old vectors stay
+        // mmap'd inside the native module — we only pass slot IDs across
+        // the FFI boundary, not the vector data itself. At 1B × 1536 × 4
+        // bytes = ~6 TB this is the difference between "rebuild works" and
+        // "rebuild OOMs."
+        const liveOldSlots = [];
+        const newUuids = [];
+        if (this.native) {
+            // Iterate in slot order so the new index's first n_live slots
+            // mirror the OLD index's surviving subset in deterministic order.
+            // We deliberately iterate by sorted slot id rather than uuidBySlot
+            // insertion order — sorting keeps the Vamana entry point stable
+            // and the on-disk vector section's locality similar to the
+            // pre-rebuild file (less page-cache turnover during the post-
+            // rebuild warm-up).
+            const sortedSlots = Array.from(this.uuidBySlot.keys()).sort((a, b) => a - b);
+            for (const slot of sortedSlots) {
+                const uuid = this.uuidBySlot.get(slot);
+                if (this.tombstones.has(uuid))
+                    continue;
+                liveOldSlots.push(slot);
+                newUuids.push(uuid);
+            }
+        }
+        const dim = this.config.dimensions;
+        const deltaCount = this.delta.size;
+        let deltaBuf = null;
+        if (deltaCount > 0) {
+            deltaBuf = new Float32Array(deltaCount * dim);
+            let idx = 0;
+            for (const [uuid, vector] of this.delta) {
+                if (vector.length !== dim) {
+                    throw new Error(`NativeDiskAnnWrapper.rebuild: vector dim ${vector.length} ≠ index dim ${dim}`);
+                }
+                deltaBuf.set(vector, idx * dim);
+                newUuids.push(uuid);
+                idx++;
+            }
+        }
+        if (liveOldSlots.length + deltaCount === 0) {
+            prodLog?.warn?.('NativeDiskAnnWrapper.rebuild: nothing to build');
+            return;
+        }
+        const totalCount = liveOldSlots.length + deltaCount;
+        const cfg = {
+            vamana: {
+                maxDegree: options?.maxDegree ?? this.config.maxDegree,
+                searchListSize: options?.searchListSize ?? this.config.searchListSize,
+                alpha: options?.alpha ?? this.config.alpha,
+                seed: BigInt(0xd15ca4440ffff00dn),
+                parallel: true,
+                parallelBatch: 64,
+            },
+            pq: {
+                m: options?.pqM ?? this.config.pqM,
+                ksub: options?.pqKsub ?? this.config.pqKsub,
+                iterations: 25,
+                trainingSample: Math.min(200_000, totalCount),
+            },
+            adjacency: this.config.useMmapAdjacency
+                ? {
+                    kind: 'mmap',
+                    mmapPath: this.config.mmapAdjacencyPath ?? `${this.config.indexPath}.adj`,
+                }
+                : { kind: 'ram' },
+        };
+        const newNative = NativeDiskANN.rebuildFromExisting({
+            existingPath: this.native ? this.config.indexPath : undefined,
+            liveOldSlots,
+            deltaVectors: deltaBuf != null
+                ? Buffer.from(deltaBuf.buffer, deltaBuf.byteOffset, deltaBuf.byteLength)
+                : undefined,
+            deltaCount,
+            dim,
+            outputPath: this.config.indexPath,
+            cfg,
+        });
+        // Rebuild the bidirectional UUID↔slot maps from `newUuids`. New
+        // slot `i` corresponds to `newUuids[i]` — this matches the napi
+        // layout invariant (live old slots first, delta tail second).
+        const newSlotByUuid = new Map();
+        const newUuidBySlot = new Map();
+        for (let i = 0; i < newUuids.length; i++) {
+            newSlotByUuid.set(newUuids[i], i);
+            newUuidBySlot.set(i, newUuids[i]);
+        }
+        // Atomic swap.
+        this.native = newNative;
+        this.slotByUuid = newSlotByUuid;
+        this.uuidBySlot = newUuidBySlot;
+        this.delta.clear();
+        this.tombstones.clear();
+    }
+    /**
+     * Flush the delta buffer to disk. For DiskANN the delta is in-memory
+     * by design (a few MB at most between rebuilds); returns the buffer
+     * size for parity with HNSW's flush contract.
+     */
+    async flush() {
+        return this.delta.size;
+    }
+    getPersistMode() {
+        return this.persistMode;
+    }
+    tryOpenExisting() {
+        try {
+            const bindings = loadNativeModule();
+            // napi-rs exports the class as `NativeDiskAnn` (PascalCase
+            // normalization of the Rust ident `NativeDiskANN`). The TS type
+            // alias `NativeDiskANN = NativeDiskAnn` in `native/index.d.ts` is
+            // for backwards-compat in *types* only — at runtime there's a
+            // single export under the napi-normalized name.
+            const NativeDiskANN = bindings.NativeDiskAnn;
+            if (!NativeDiskANN)
+                return;
+            this.native = NativeDiskANN.openExisting(this.config.indexPath);
+            // Populate slot maps from the storage adapter — these are persisted
+            // alongside the index file in production. For 35c we read from a
+            // sibling `.slots.json` that rebuild() writes.
+            // (Stub for now; the real path lands when storage integration ships.)
+        }
+        catch {
+            // No existing file — index stays empty until first rebuild().
+            this.native = null;
+        }
+    }
+    countMainTombstones() {
+        let n = 0;
+        for (const uuid of this.tombstones) {
+            if (this.slotByUuid.has(uuid))
+                n++;
+        }
+        return n;
+    }
+}
+//# sourceMappingURL=NativeDiskAnnWrapper.js.map

package/dist/hnsw/NativeHNSWWrapper.d.ts

@@ -30,6 +30,7 @@ export declare class NativeHNSWWrapper implements HnswProvider {
     private unifiedCache;
     private cowEnabled;
     private mmapStore;
+    private connectionsCodec;
     constructor(config: (Partial<HNSWConfig> & {
         distanceFunction?: DistanceFunction;
     }) | undefined, distanceFunction: DistanceFunction, options?: {
@@ -83,6 +84,35 @@ export declare class NativeHNSWWrapper implements HnswProvider {
     enableCOW(parent: NativeHNSWWrapper): void;
     setUseParallelization(useParallelization: boolean): void;
     getUseParallelization(): boolean;
+    /**
+     * @description Accept (or detach) the brainy `ConnectionsCodec`. Brainy 7.27+
+     * calls this unconditionally during init from `wireConnectionsCodec()` when
+     * the `graph:compression` provider is registered (which cortex always
+     * supplies via `native.encodeConnections`/`decodeConnections`).
+     *
+     * Cortex's native HNSW serializes connections through its own path —
+     * `addItemFull` returns `nodeData` written directly via `storage.saveHNSWData`
+     * (and the mmap binary backend when available). It never routes through
+     * brainy's JS-side `persistNodeConnections`/`restoreNodeConnections`, which
+     * is where the codec is consumed. The codec is therefore unreachable from
+     * this wrapper.
+     *
+     * We accept the call (so brainy's init succeeds) and store the reference for
+     * introspection/parity. We do NOT re-encode connections through the codec on
+     * top of the native format — that would double-encode (waste CPU) or replace
+     * the native format with a strictly less efficient one (waste perf). Brainy
+     * treats the method as feature-detected/optional on third-party providers,
+     * so a storing acceptor is the contract-correct behaviour.
+     *
+     * @param codec - The `ConnectionsCodec` instance, or `null` to detach.
+     */
+    setConnectionsCodec(codec: unknown): void;
+    /**
+     * @description Read back the currently-attached `ConnectionsCodec`, or null.
+     * Exposed for parity tests + future inspection; cortex itself does not
+     * consult this value on the read/write path.
+     */
+    getConnectionsCodec(): unknown;
     size(): number;
     clear(): void;
     getEntryPointId(): string | null;

package/dist/hnsw/NativeHNSWWrapper.js

@@ -38,6 +38,10 @@ export class NativeHNSWWrapper {
     cowEnabled = false;
     // Mmap binary HNSW store (Phase 4 — optional, used when storage has rootDirectory)
     mmapStore = null;
+    // Brainy ConnectionsCodec (brainy >= 7.27 `wireConnectionsCodec`). Stored for
+    // introspection but not consulted on the read/write path — see
+    // `setConnectionsCodec` below for the architectural rationale.
+    connectionsCodec = null;
     constructor(config = {}, distanceFunction, options = {}) {
         this.config = { ...DEFAULT_CONFIG, ...config };
         this.distanceFunction = distanceFunction;
@@ -485,6 +489,39 @@ export class NativeHNSWWrapper {
     getUseParallelization() {
         return this.useParallelization;
     }
+    /**
+     * @description Accept (or detach) the brainy `ConnectionsCodec`. Brainy 7.27+
+     * calls this unconditionally during init from `wireConnectionsCodec()` when
+     * the `graph:compression` provider is registered (which cortex always
+     * supplies via `native.encodeConnections`/`decodeConnections`).
+     *
+     * Cortex's native HNSW serializes connections through its own path —
+     * `addItemFull` returns `nodeData` written directly via `storage.saveHNSWData`
+     * (and the mmap binary backend when available). It never routes through
+     * brainy's JS-side `persistNodeConnections`/`restoreNodeConnections`, which
+     * is where the codec is consumed. The codec is therefore unreachable from
+     * this wrapper.
+     *
+     * We accept the call (so brainy's init succeeds) and store the reference for
+     * introspection/parity. We do NOT re-encode connections through the codec on
+     * top of the native format — that would double-encode (waste CPU) or replace
+     * the native format with a strictly less efficient one (waste perf). Brainy
+     * treats the method as feature-detected/optional on third-party providers,
+     * so a storing acceptor is the contract-correct behaviour.
+     *
+     * @param codec - The `ConnectionsCodec` instance, or `null` to detach.
+     */
+    setConnectionsCodec(codec) {
+        this.connectionsCodec = codec;
+    }
+    /**
+     * @description Read back the currently-attached `ConnectionsCodec`, or null.
+     * Exposed for parity tests + future inspection; cortex itself does not
+     * consult this value on the read/write path.
+     */
+    getConnectionsCodec() {
+        return this.connectionsCodec;
+    }
     // ---------------------------------------------------------------------------
     // Info / Introspection
     // ---------------------------------------------------------------------------